The following pull request was submitted through Github. It can be accessed and reviewed at: https://github.com/lxc/lxd/pull/6283
This e-mail was sent by the LXC bot, direct replies will not reach the author unless they happen to be subscribed to this list. === Description (from pull-request) ===
From aa2ec7b359ae05d86711f145c97e29d3030dd642 Mon Sep 17 00:00:00 2001 From: Thomas Hipp <thomas.h...@canonical.com> Date: Mon, 7 Oct 2019 14:33:37 +0200 Subject: [PATCH 1/2] lxd: Move backup to separate package This moves relevant backup code into its own package. Signed-off-by: Thomas Hipp <thomas.h...@canonical.com> --- lxd/backup.go | 237 +++++--------------------------------- lxd/backup/backup.go | 233 +++++++++++++++++++++++++++++++++++++ lxd/container.go | 3 +- lxd/container_backup.go | 8 +- lxd/container_lxc.go | 7 +- lxd/containers_post.go | 5 +- lxd/instance_interface.go | 3 +- lxd/storage.go | 5 +- lxd/storage_btrfs.go | 45 +++----- lxd/storage_ceph.go | 38 +----- lxd/storage_ceph_utils.go | 3 +- lxd/storage_cephfs.go | 5 +- lxd/storage_dir.go | 40 ++----- lxd/storage_lvm.go | 36 ++---- lxd/storage_mock.go | 5 +- lxd/storage_zfs.go | 51 +++----- 16 files changed, 338 insertions(+), 386 deletions(-) create mode 100644 lxd/backup/backup.go diff --git a/lxd/backup.go b/lxd/backup.go index 4ebd4240a4..ce235b9989 100644 --- a/lxd/backup.go +++ b/lxd/backup.go @@ -1,33 +1,30 @@ package main import ( - "archive/tar" "fmt" - "io" + "io/ioutil" "os" - "os/exec" "path/filepath" - "strings" "time" "context" "gopkg.in/yaml.v2" + "github.com/lxc/lxd/lxd/backup" "github.com/lxc/lxd/lxd/cluster" "github.com/lxc/lxd/lxd/db" "github.com/lxc/lxd/lxd/operations" "github.com/lxc/lxd/lxd/state" "github.com/lxc/lxd/lxd/task" "github.com/lxc/lxd/shared" - "github.com/lxc/lxd/shared/api" log "github.com/lxc/lxd/shared/log15" "github.com/lxc/lxd/shared/logger" "github.com/pkg/errors" ) // Load a backup from the database -func backupLoadByName(s *state.State, project, name string) (*backup, error) { +func backupLoadByName(s *state.State, project, name string) (*backup.Backup, error) { // Get the backup database record args, err := s.Cluster.ContainerGetBackup(project, name) if err != nil { @@ -40,17 +37,7 @@ func backupLoadByName(s *state.State, project, name string) (*backup, error) { return nil, errors.Wrap(err, "Load container from database") } - // Return the backup struct - return &backup{ - state: s, - instance: instance, - id: args.ID, - name: name, - creationDate: args.CreationDate, - expiryDate: args.ExpiryDate, - instanceOnly: args.InstanceOnly, - optimizedStorage: args.OptimizedStorage, - }, nil + return backup.NewBackup(s, instance, name, args), nil } // Create a new backup @@ -71,168 +58,36 @@ func backupCreate(s *state.State, args db.InstanceBackupArgs, sourceContainer In return errors.Wrap(err, "Load backup object") } - // Now create the empty snapshot - err = sourceContainer.Storage().ContainerBackupCreate(*b, sourceContainer) - if err != nil { - s.Cluster.ContainerBackupRemove(args.Name) - return errors.Wrap(err, "Backup storage") - } - - return nil -} - -// backup represents a container backup -type backup struct { - state *state.State - instance Instance - - // Properties - id int - name string - creationDate time.Time - expiryDate time.Time - instanceOnly bool - optimizedStorage bool -} - -type backupInfo struct { - Project string `json:"project" yaml:"project"` - Name string `json:"name" yaml:"name"` - Backend string `json:"backend" yaml:"backend"` - Privileged bool `json:"privileged" yaml:"privileged"` - Pool string `json:"pool" yaml:"pool"` - Snapshots []string `json:"snapshots,omitempty" yaml:"snapshots,omitempty"` - HasBinaryFormat bool `json:"-" yaml:"-"` -} - -// Rename renames a container backup -func (b *backup) Rename(newName string) error { - oldBackupPath := shared.VarPath("backups", b.name) - newBackupPath := shared.VarPath("backups", newName) - - // Create the new backup path - backupsPath := shared.VarPath("backups", b.instance.Name()) - if !shared.PathExists(backupsPath) { - err := os.MkdirAll(backupsPath, 0700) - if err != nil { - return err - } - } - - // Rename the backup directory - err := os.Rename(oldBackupPath, newBackupPath) + ourStart, err := sourceContainer.StorageStart() if err != nil { return err } - - // Check if we can remove the container directory - empty, _ := shared.PathIsEmpty(backupsPath) - if empty { - err := os.Remove(backupsPath) - if err != nil { - return err - } + if ourStart { + defer sourceContainer.StorageStop() } - // Rename the database record - err = b.state.Cluster.ContainerBackupRename(b.name, newName) + // Create a temporary path for the backup + tmpPath, err := ioutil.TempDir(shared.VarPath("backups"), "lxd_backup_") if err != nil { return err } + defer os.RemoveAll(tmpPath) - return nil -} - -// Delete removes an instance backup -func (b *backup) Delete() error { - return doBackupDelete(b.state, b.name, b.instance.Name()) -} - -func (b *backup) Render() *api.InstanceBackup { - return &api.InstanceBackup{ - Name: strings.SplitN(b.name, "/", 2)[1], - CreatedAt: b.creationDate, - ExpiresAt: b.expiryDate, - InstanceOnly: b.instanceOnly, - ContainerOnly: b.instanceOnly, - OptimizedStorage: b.optimizedStorage, - } -} - -func backupGetInfo(r io.ReadSeeker) (*backupInfo, error) { - var tr *tar.Reader - result := backupInfo{} - hasBinaryFormat := false - hasIndexFile := false - - // Extract - r.Seek(0, 0) - _, _, unpacker, err := shared.DetectCompressionFile(r) + // Now create the empty snapshot + err = sourceContainer.Storage().ContainerBackupCreate(tmpPath, *b, sourceContainer) if err != nil { - return nil, err - } - r.Seek(0, 0) - - if unpacker == nil { - return nil, fmt.Errorf("Unsupported backup compression") - } - - if len(unpacker) > 0 { - cmd := exec.Command(unpacker[0], unpacker[1:]...) - cmd.Stdin = r - - stdout, err := cmd.StdoutPipe() - if err != nil { - return nil, err - } - defer stdout.Close() - - err = cmd.Start() - if err != nil { - return nil, err - } - defer cmd.Wait() - - tr = tar.NewReader(stdout) - } else { - tr = tar.NewReader(r) - } - - for { - hdr, err := tr.Next() - if err == io.EOF { - break // End of archive - } - if err != nil { - return nil, err - } - - if hdr.Name == "backup/index.yaml" { - err = yaml.NewDecoder(tr).Decode(&result) - if err != nil { - return nil, err - } - - hasIndexFile = true - } - - if hdr.Name == "backup/container.bin" { - hasBinaryFormat = true - } - } - - if !hasIndexFile { - return nil, fmt.Errorf("Backup is missing index.yaml") + s.Cluster.ContainerBackupRemove(args.Name) + return errors.Wrap(err, "Backup storage") } - result.HasBinaryFormat = hasBinaryFormat - return &result, nil + // Pack the backup + return backupCreateTarball(s, tmpPath, *b, sourceContainer) } // fixBackupStoragePool changes the pool information in the backup.yaml. This // is done only if the provided pool doesn't exist. In this case, the pool of // the default profile will be used. -func backupFixStoragePool(c *db.Cluster, b backupInfo, useDefaultPool bool) error { +func backupFixStoragePool(c *db.Cluster, b backup.Info, useDefaultPool bool) error { var poolName string if useDefaultPool { @@ -323,23 +178,23 @@ func backupFixStoragePool(c *db.Cluster, b backupInfo, useDefaultPool bool) erro return nil } -func backupCreateTarball(s *state.State, path string, backup backup) error { +func backupCreateTarball(s *state.State, path string, b backup.Backup, c Instance) error { // Create the index - pool, err := backup.instance.StoragePool() + pool, err := c.StoragePool() if err != nil { return err } - indexFile := backupInfo{ - Name: backup.instance.Name(), - Backend: backup.instance.Storage().GetStorageTypeName(), - Privileged: backup.instance.IsPrivileged(), + indexFile := backup.Info{ + Name: c.Name(), + Backend: c.Storage().GetStorageTypeName(), + Privileged: c.IsPrivileged(), Pool: pool, Snapshots: []string{}, } - if !backup.instanceOnly { - snaps, err := backup.instance.Snapshots() + if !b.InstanceOnly() { + snaps, err := c.Snapshots() if err != nil { return err } @@ -367,7 +222,7 @@ func backupCreateTarball(s *state.State, path string, backup backup) error { } // Create the target path if needed - backupsPath := shared.VarPath("backups", backup.instance.Name()) + backupsPath := shared.VarPath("backups", c.Name()) if !shared.PathExists(backupsPath) { err := os.MkdirAll(backupsPath, 0700) if err != nil { @@ -376,7 +231,7 @@ func backupCreateTarball(s *state.State, path string, backup backup) error { } // Create the tarball - backupPath := shared.VarPath("backups", backup.name) + backupPath := shared.VarPath("backups", b.Name()) success := false defer func() { if success { @@ -489,43 +344,13 @@ func pruneExpiredContainerBackups(ctx context.Context, d *Daemon) error { return errors.Wrap(err, "Unable to retrieve the list of expired container backups") } - for _, backup := range backups { - containerName, _, _ := shared.ContainerGetParentAndSnapshotName(backup) - err := doBackupDelete(d.State(), backup, containerName) - if err != nil { - return errors.Wrapf(err, "Error deleting container backup %s", backup) - } - } - - return nil -} - -func doBackupDelete(s *state.State, backupName, containerName string) error { - backupPath := shared.VarPath("backups", backupName) - - // Delete the on-disk data - if shared.PathExists(backupPath) { - err := os.RemoveAll(backupPath) - if err != nil { - return err - } - } - - // Check if we can remove the container directory - backupsPath := shared.VarPath("backups", containerName) - empty, _ := shared.PathIsEmpty(backupsPath) - if empty { - err := os.Remove(backupsPath) + for _, b := range backups { + containerName, _, _ := shared.ContainerGetParentAndSnapshotName(b) + err := backup.DoBackupDelete(d.State(), b, containerName) if err != nil { - return err + return errors.Wrapf(err, "Error deleting container backup %s", b) } } - // Remove the database record - err := s.Cluster.ContainerBackupRemove(backupName) - if err != nil { - return err - } - return nil } diff --git a/lxd/backup/backup.go b/lxd/backup/backup.go new file mode 100644 index 0000000000..1e694f2f4d --- /dev/null +++ b/lxd/backup/backup.go @@ -0,0 +1,233 @@ +package backup + +import ( + "archive/tar" + "fmt" + "io" + "os" + "os/exec" + "strings" + "time" + + "github.com/lxc/lxd/lxd/db" + "github.com/lxc/lxd/lxd/state" + "github.com/lxc/lxd/shared" + "github.com/lxc/lxd/shared/api" + "gopkg.in/yaml.v2" +) + +// Info represents exported backup information. +type Info struct { + Project string `json:"project" yaml:"project"` + Name string `json:"name" yaml:"name"` + Backend string `json:"backend" yaml:"backend"` + Privileged bool `json:"privileged" yaml:"privileged"` + Pool string `json:"pool" yaml:"pool"` + Snapshots []string `json:"snapshots,omitempty" yaml:"snapshots,omitempty"` + HasBinaryFormat bool `json:"-" yaml:"-"` +} + +// GetInfo extracts backup information from a given ReadSeeker. +func GetInfo(r io.ReadSeeker) (*Info, error) { + var tr *tar.Reader + result := Info{} + hasBinaryFormat := false + hasIndexFile := false + + // Extract + r.Seek(0, 0) + _, _, unpacker, err := shared.DetectCompressionFile(r) + if err != nil { + return nil, err + } + r.Seek(0, 0) + + if unpacker == nil { + return nil, fmt.Errorf("Unsupported backup compression") + } + + if len(unpacker) > 0 { + cmd := exec.Command(unpacker[0], unpacker[1:]...) + cmd.Stdin = r + + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, err + } + defer stdout.Close() + + err = cmd.Start() + if err != nil { + return nil, err + } + defer cmd.Wait() + + tr = tar.NewReader(stdout) + } else { + tr = tar.NewReader(r) + } + + for { + hdr, err := tr.Next() + if err == io.EOF { + break // End of archive + } + if err != nil { + return nil, err + } + + if hdr.Name == "backup/index.yaml" { + err = yaml.NewDecoder(tr).Decode(&result) + if err != nil { + return nil, err + } + + hasIndexFile = true + } + + if hdr.Name == "backup/container.bin" { + hasBinaryFormat = true + } + } + + if !hasIndexFile { + return nil, fmt.Errorf("Backup is missing index.yaml") + } + + result.HasBinaryFormat = hasBinaryFormat + return &result, nil +} + +type instance interface { + Name() string +} + +// Backup represents a container backup +type Backup struct { + state *state.State + instance instance + + // Properties + id int + name string + creationDate time.Time + expiryDate time.Time + instanceOnly bool + optimizedStorage bool +} + +// NewBackup returns a new Backup struct. +func NewBackup(s *state.State, instance instance, name string, args db.InstanceBackupArgs) *Backup { + return &Backup{ + state: s, + instance: instance, + id: args.ID, + name: name, + creationDate: args.CreationDate, + expiryDate: args.ExpiryDate, + instanceOnly: args.InstanceOnly, + optimizedStorage: args.OptimizedStorage, + } +} + +// InstanceOnly returns whether only the instance itself is to be backed up. +func (b *Backup) InstanceOnly() bool { + return b.instanceOnly +} + +// Name returns the name of the backup. +func (b *Backup) Name() string { + return b.name +} + +// OptimizedStorage returns whether the backup is to be performed using +// optimization supported by the storage driver. +func (b *Backup) OptimizedStorage() bool { + return b.optimizedStorage +} + +// Rename renames a container backup +func (b *Backup) Rename(newName string) error { + oldBackupPath := shared.VarPath("backups", b.name) + newBackupPath := shared.VarPath("backups", newName) + + // Create the new backup path + backupsPath := shared.VarPath("backups", b.instance.Name()) + if !shared.PathExists(backupsPath) { + err := os.MkdirAll(backupsPath, 0700) + if err != nil { + return err + } + } + + // Rename the backup directory + err := os.Rename(oldBackupPath, newBackupPath) + if err != nil { + return err + } + + // Check if we can remove the container directory + empty, _ := shared.PathIsEmpty(backupsPath) + if empty { + err := os.Remove(backupsPath) + if err != nil { + return err + } + } + + // Rename the database record + err = b.state.Cluster.ContainerBackupRename(b.name, newName) + if err != nil { + return err + } + + return nil +} + +// Delete removes an instance backup +func (b *Backup) Delete() error { + return DoBackupDelete(b.state, b.name, b.instance.Name()) +} + +// Render returns an InstanceBackup struct of the backup. +func (b *Backup) Render() *api.InstanceBackup { + return &api.InstanceBackup{ + Name: strings.SplitN(b.name, "/", 2)[1], + CreatedAt: b.creationDate, + ExpiresAt: b.expiryDate, + InstanceOnly: b.instanceOnly, + ContainerOnly: b.instanceOnly, + OptimizedStorage: b.optimizedStorage, + } +} + +// DoBackupDelete deletes a backup. +func DoBackupDelete(s *state.State, backupName, containerName string) error { + backupPath := shared.VarPath("backups", backupName) + + // Delete the on-disk data + if shared.PathExists(backupPath) { + err := os.RemoveAll(backupPath) + if err != nil { + return err + } + } + + // Check if we can remove the container directory + backupsPath := shared.VarPath("backups", containerName) + empty, _ := shared.PathIsEmpty(backupsPath) + if empty { + err := os.Remove(backupsPath) + if err != nil { + return err + } + } + + // Remove the database record + err := s.Cluster.ContainerBackupRemove(backupName) + if err != nil { + return err + } + + return nil +} diff --git a/lxd/container.go b/lxd/container.go index dce1e8f191..6162e1a67c 100644 --- a/lxd/container.go +++ b/lxd/container.go @@ -16,6 +16,7 @@ import ( cron "gopkg.in/robfig/cron.v2" "github.com/flosch/pongo2" + "github.com/lxc/lxd/lxd/backup" "github.com/lxc/lxd/lxd/cluster" "github.com/lxc/lxd/lxd/db" "github.com/lxc/lxd/lxd/device" @@ -264,7 +265,7 @@ func containerCreateAsEmpty(d *Daemon, args db.InstanceArgs) (container, error) return c, nil } -func containerCreateFromBackup(s *state.State, info backupInfo, data io.ReadSeeker, +func containerCreateFromBackup(s *state.State, info backup.Info, data io.ReadSeeker, customPool bool) (storage, error) { var pool storage var fixBackupFile = false diff --git a/lxd/container_backup.go b/lxd/container_backup.go index 4e016d7af0..064eac09ba 100644 --- a/lxd/container_backup.go +++ b/lxd/container_backup.go @@ -55,7 +55,7 @@ func containerBackupsGet(d *Daemon, r *http.Request) response.Response { for _, backup := range backups { if !recursion { url := fmt.Sprintf("/%s/containers/%s/backups/%s", - version.APIVersion, cname, strings.Split(backup.name, "/")[1]) + version.APIVersion, cname, strings.Split(backup.Name(), "/")[1]) resultString = append(resultString, url) } else { render := backup.Render() @@ -131,11 +131,11 @@ func containerBackupsPost(d *Daemon, r *http.Request) response.Response { for _, backup := range backups { // Ignore backups not containing base - if !strings.HasPrefix(backup.name, base) { + if !strings.HasPrefix(backup.Name(), base) { continue } - substr := backup.name[length:] + substr := backup.Name()[length:] var num int count, err := fmt.Sscanf(substr, "%d", &num) if err != nil || count != 1 { @@ -347,7 +347,7 @@ func containerBackupExportGet(d *Daemon, r *http.Request) response.Response { } ent := response.FileResponseEntry{ - Path: shared.VarPath("backups", backup.name), + Path: shared.VarPath("backups", backup.Name()), } return response.FileResponse(r, []response.FileResponseEntry{ent}, nil, false) diff --git a/lxd/container_lxc.go b/lxd/container_lxc.go index f2c7195b69..9dc090281d 100644 --- a/lxd/container_lxc.go +++ b/lxd/container_lxc.go @@ -26,6 +26,7 @@ import ( yaml "gopkg.in/yaml.v2" "github.com/lxc/lxd/lxd/apparmor" + "github.com/lxc/lxd/lxd/backup" "github.com/lxc/lxd/lxd/cluster" "github.com/lxc/lxd/lxd/db" "github.com/lxc/lxd/lxd/db/query" @@ -3287,7 +3288,7 @@ func (c *containerLXC) Snapshots() ([]Instance, error) { return instances, nil } -func (c *containerLXC) Backups() ([]backup, error) { +func (c *containerLXC) Backups() ([]backup.Backup, error) { // Get all the backups backupNames, err := c.state.Cluster.ContainerGetBackups(c.project, c.name) if err != nil { @@ -3295,7 +3296,7 @@ func (c *containerLXC) Backups() ([]backup, error) { } // Build the backup list - backups := []backup{} + backups := []backup.Backup{} for _, backupName := range backupNames { backup, err := backupLoadByName(c.state, c.project, backupName) if err != nil { @@ -3709,7 +3710,7 @@ func (c *containerLXC) Rename(newName string) error { } for _, backup := range backups { - backupName := strings.Split(backup.name, "/")[1] + backupName := strings.Split(backup.Name(), "/")[1] newName := fmt.Sprintf("%s/%s", newName, backupName) err = backup.Rename(newName) diff --git a/lxd/containers_post.go b/lxd/containers_post.go index 5c79e675d2..a19d53aa0c 100644 --- a/lxd/containers_post.go +++ b/lxd/containers_post.go @@ -13,10 +13,11 @@ import ( "os" "strings" - "github.com/dustinkirkland/golang-petname" + petname "github.com/dustinkirkland/golang-petname" "github.com/gorilla/websocket" "github.com/pkg/errors" + "github.com/lxc/lxd/lxd/backup" "github.com/lxc/lxd/lxd/cluster" "github.com/lxc/lxd/lxd/db" deviceConfig "github.com/lxc/lxd/lxd/device/config" @@ -634,7 +635,7 @@ func createFromBackup(d *Daemon, project string, data io.Reader, pool string) re // Parse the backup information f.Seek(0, 0) - bInfo, err := backupGetInfo(f) + bInfo, err := backup.GetInfo(f) if err != nil { f.Close() return response.BadRequest(err) diff --git a/lxd/instance_interface.go b/lxd/instance_interface.go index 80af10ce13..b6dbaa3022 100644 --- a/lxd/instance_interface.go +++ b/lxd/instance_interface.go @@ -6,6 +6,7 @@ import ( "os/exec" "time" + "github.com/lxc/lxd/lxd/backup" "github.com/lxc/lxd/lxd/db" "github.com/lxc/lxd/lxd/device" deviceConfig "github.com/lxc/lxd/lxd/device/config" @@ -29,7 +30,7 @@ type Instance interface { // Snapshots & migration & backups Restore(source Instance, stateful bool) error Snapshots() ([]Instance, error) - Backups() ([]backup, error) + Backups() ([]backup.Backup, error) // Config handling Rename(newName string) error diff --git a/lxd/storage.go b/lxd/storage.go index ece8dc2812..4f5a1d6289 100644 --- a/lxd/storage.go +++ b/lxd/storage.go @@ -11,6 +11,7 @@ import ( "github.com/gorilla/websocket" "github.com/pkg/errors" + "github.com/lxc/lxd/lxd/backup" "github.com/lxc/lxd/lxd/db" "github.com/lxc/lxd/lxd/device" "github.com/lxc/lxd/lxd/instance/instancetype" @@ -212,8 +213,8 @@ type storage interface { ContainerSnapshotStart(c Instance) (bool, error) ContainerSnapshotStop(c Instance) (bool, error) - ContainerBackupCreate(backup backup, sourceContainer Instance) error - ContainerBackupLoad(info backupInfo, data io.ReadSeeker, tarArgs []string) error + ContainerBackupCreate(path string, backup backup.Backup, sourceContainer Instance) error + ContainerBackupLoad(info backup.Info, data io.ReadSeeker, tarArgs []string) error // For use in migrating snapshots. ContainerSnapshotCreateEmpty(c Instance) error diff --git a/lxd/storage_btrfs.go b/lxd/storage_btrfs.go index 3220b10af2..daa64b6138 100644 --- a/lxd/storage_btrfs.go +++ b/lxd/storage_btrfs.go @@ -16,6 +16,7 @@ import ( "github.com/pkg/errors" "golang.org/x/sys/unix" + "github.com/lxc/lxd/lxd/backup" "github.com/lxc/lxd/lxd/migration" "github.com/lxc/lxd/lxd/operations" "github.com/lxc/lxd/lxd/project" @@ -1616,10 +1617,10 @@ func (s *storageBtrfs) doBtrfsBackup(cur string, prev string, target string) err return err } -func (s *storageBtrfs) doContainerBackupCreateOptimized(tmpPath string, backup backup, source Instance) error { +func (s *storageBtrfs) doContainerBackupCreateOptimized(tmpPath string, backup backup.Backup, source Instance) error { // Handle snapshots finalParent := "" - if !backup.instanceOnly { + if !backup.InstanceOnly() { snapshotsPath := fmt.Sprintf("%s/snapshots", tmpPath) // Retrieve the snapshots @@ -1689,7 +1690,7 @@ func (s *storageBtrfs) doContainerBackupCreateOptimized(tmpPath string, backup b return nil } -func (s *storageBtrfs) doContainerBackupCreateVanilla(tmpPath string, backup backup, source Instance) error { +func (s *storageBtrfs) doContainerBackupCreateVanilla(tmpPath string, backup backup.Backup, source Instance) error { // Prepare for rsync rsync := func(oldPath string, newPath string, bwlimit string) error { output, err := rsyncLocalCopy(oldPath, newPath, bwlimit, true) @@ -1703,7 +1704,7 @@ func (s *storageBtrfs) doContainerBackupCreateVanilla(tmpPath string, backup bac bwlimit := s.pool.Config["rsync.bwlimit"] // Handle snapshots - if !backup.instanceOnly { + if !backup.InstanceOnly() { snapshotsPath := fmt.Sprintf("%s/snapshots", tmpPath) // Retrieve the snapshots @@ -1772,46 +1773,26 @@ func (s *storageBtrfs) doContainerBackupCreateVanilla(tmpPath string, backup bac return nil } -func (s *storageBtrfs) ContainerBackupCreate(backup backup, source Instance) error { - // Start storage - ourStart, err := source.StorageStart() - if err != nil { - return err - } - if ourStart { - defer source.StorageStop() - } - - // Create a temporary path for the backup - tmpPath, err := ioutil.TempDir(shared.VarPath("backups"), "lxd_backup_") - if err != nil { - return err - } - defer os.RemoveAll(tmpPath) +func (s *storageBtrfs) ContainerBackupCreate(path string, backup backup.Backup, source Instance) error { + var err error // Generate the actual backup - if backup.optimizedStorage { - err = s.doContainerBackupCreateOptimized(tmpPath, backup, source) + if backup.OptimizedStorage() { + err = s.doContainerBackupCreateOptimized(path, backup, source) if err != nil { return err } } else { - err := s.doContainerBackupCreateVanilla(tmpPath, backup, source) + err := s.doContainerBackupCreateVanilla(path, backup, source) if err != nil { return err } } - // Pack the backup - err = backupCreateTarball(s.s, tmpPath, backup) - if err != nil { - return err - } - return nil } -func (s *storageBtrfs) doContainerBackupLoadOptimized(info backupInfo, data io.ReadSeeker, tarArgs []string) error { +func (s *storageBtrfs) doContainerBackupLoadOptimized(info backup.Info, data io.ReadSeeker, tarArgs []string) error { containerName, _, _ := shared.ContainerGetParentAndSnapshotName(info.Name) containerMntPoint := driver.GetContainerMountPoint("default", s.pool.Name, "") @@ -1909,7 +1890,7 @@ func (s *storageBtrfs) doContainerBackupLoadOptimized(info backupInfo, data io.R return nil } -func (s *storageBtrfs) doContainerBackupLoadVanilla(info backupInfo, data io.ReadSeeker, tarArgs []string) error { +func (s *storageBtrfs) doContainerBackupLoadVanilla(info backup.Info, data io.ReadSeeker, tarArgs []string) error { // create the main container err := s.doContainerCreate(info.Project, info.Name, info.Privileged) if err != nil { @@ -1964,7 +1945,7 @@ func (s *storageBtrfs) doContainerBackupLoadVanilla(info backupInfo, data io.Rea return nil } -func (s *storageBtrfs) ContainerBackupLoad(info backupInfo, data io.ReadSeeker, tarArgs []string) error { +func (s *storageBtrfs) ContainerBackupLoad(info backup.Info, data io.ReadSeeker, tarArgs []string) error { logger.Debugf("Loading BTRFS storage volume for backup \"%s\" on storage pool \"%s\"", info.Name, s.pool.Name) if info.HasBinaryFormat { diff --git a/lxd/storage_ceph.go b/lxd/storage_ceph.go index bede743bd3..61c032a8ad 100644 --- a/lxd/storage_ceph.go +++ b/lxd/storage_ceph.go @@ -14,6 +14,7 @@ import ( "github.com/pkg/errors" "golang.org/x/sys/unix" + "github.com/lxc/lxd/lxd/backup" "github.com/lxc/lxd/lxd/db" "github.com/lxc/lxd/lxd/migration" "github.com/lxc/lxd/lxd/operations" @@ -1890,50 +1891,23 @@ func (s *storageCeph) ContainerSnapshotCreateEmpty(c Instance) error { return nil } -func (s *storageCeph) ContainerBackupCreate(backup backup, source Instance) error { - // Start storage - ourStart, err := source.StorageStart() - if err != nil { - return err - } - if ourStart { - defer source.StorageStop() - } - - // Create a temporary path for the backup - tmpPath, err := ioutil.TempDir(shared.VarPath("backups"), "lxd_backup_") - if err != nil { - return err - } - defer os.RemoveAll(tmpPath) - +func (s *storageCeph) ContainerBackupCreate(path string, backup backup.Backup, source Instance) error { // Generate the actual backup - if !backup.instanceOnly { + if !backup.InstanceOnly() { snapshots, err := source.Snapshots() if err != nil { return err } for _, snap := range snapshots { - err := s.cephRBDVolumeBackupCreate(tmpPath, backup, snap) + err := s.cephRBDVolumeBackupCreate(path, backup, snap) if err != nil { return err } } } - err = s.cephRBDVolumeBackupCreate(tmpPath, backup, source) - if err != nil { - return err - } - - // Pack the backup - err = backupCreateTarball(s.s, tmpPath, backup) - if err != nil { - return err - } - - return nil + return s.cephRBDVolumeBackupCreate(path, backup, source) } // This function recreates an rbd container including its snapshots. It @@ -1942,7 +1916,7 @@ func (s *storageCeph) ContainerBackupCreate(backup backup, source Instance) erro // - for each snapshot dump the contents into the empty storage volume and // after each dump take a snapshot of the rbd storage volume // - dump the container contents into the rbd storage volume. -func (s *storageCeph) ContainerBackupLoad(info backupInfo, data io.ReadSeeker, tarArgs []string) error { +func (s *storageCeph) ContainerBackupLoad(info backup.Info, data io.ReadSeeker, tarArgs []string) error { // create the main container err := s.doContainerCreate(info.Project, info.Name, info.Privileged) if err != nil { diff --git a/lxd/storage_ceph_utils.go b/lxd/storage_ceph_utils.go index 4d5c20222e..174a62315c 100644 --- a/lxd/storage_ceph_utils.go +++ b/lxd/storage_ceph_utils.go @@ -14,6 +14,7 @@ import ( "github.com/pborman/uuid" "golang.org/x/sys/unix" + "github.com/lxc/lxd/lxd/backup" "github.com/lxc/lxd/lxd/db" "github.com/lxc/lxd/lxd/project" driver "github.com/lxc/lxd/lxd/storage" @@ -1584,7 +1585,7 @@ func (s *storageCeph) cephRBDVolumeDumpToFile(sourceVolumeName string, file stri } // cephRBDVolumeBackupCreate creates a backup of a container or snapshot. -func (s *storageCeph) cephRBDVolumeBackupCreate(tmpPath string, backup backup, source Instance) error { +func (s *storageCeph) cephRBDVolumeBackupCreate(tmpPath string, backup backup.Backup, source Instance) error { sourceIsSnapshot := source.IsSnapshot() sourceContainerName := source.Name() sourceContainerOnlyName := project.Prefix(source.Project(), sourceContainerName) diff --git a/lxd/storage_cephfs.go b/lxd/storage_cephfs.go index ed3f219ece..89810318b4 100644 --- a/lxd/storage_cephfs.go +++ b/lxd/storage_cephfs.go @@ -13,6 +13,7 @@ import ( "github.com/gorilla/websocket" "github.com/pkg/errors" + "github.com/lxc/lxd/lxd/backup" "github.com/lxc/lxd/lxd/migration" "github.com/lxc/lxd/lxd/operations" "github.com/lxc/lxd/lxd/state" @@ -693,11 +694,11 @@ func (s *storageCephFs) ContainerSnapshotStop(container Instance) (bool, error) return false, fmt.Errorf("CEPHFS cannot be used for containers") } -func (s *storageCephFs) ContainerBackupCreate(backup backup, source Instance) error { +func (s *storageCephFs) ContainerBackupCreate(path string, backup backup.Backup, source Instance) error { return fmt.Errorf("CEPHFS cannot be used for containers") } -func (s *storageCephFs) ContainerBackupLoad(info backupInfo, data io.ReadSeeker, tarArgs []string) error { +func (s *storageCephFs) ContainerBackupLoad(info backup.Info, data io.ReadSeeker, tarArgs []string) error { return fmt.Errorf("CEPHFS cannot be used for containers") } diff --git a/lxd/storage_dir.go b/lxd/storage_dir.go index c24eecaae4..825485eedf 100644 --- a/lxd/storage_dir.go +++ b/lxd/storage_dir.go @@ -3,7 +3,6 @@ package main import ( "fmt" "io" - "io/ioutil" "os" "path/filepath" "strings" @@ -12,6 +11,7 @@ import ( "github.com/pkg/errors" "golang.org/x/sys/unix" + "github.com/lxc/lxd/lxd/backup" "github.com/lxc/lxd/lxd/migration" "github.com/lxc/lxd/lxd/operations" "github.com/lxc/lxd/lxd/project" @@ -1122,23 +1122,7 @@ func (s *storageDir) ContainerSnapshotStop(container Instance) (bool, error) { return true, nil } -func (s *storageDir) ContainerBackupCreate(backup backup, source Instance) error { - // Start storage - ourStart, err := source.StorageStart() - if err != nil { - return err - } - if ourStart { - defer source.StorageStop() - } - - // Create a temporary path for the backup - tmpPath, err := ioutil.TempDir(shared.VarPath("backups"), "lxd_backup_") - if err != nil { - return err - } - defer os.RemoveAll(tmpPath) - +func (s *storageDir) ContainerBackupCreate(path string, backup backup.Backup, source Instance) error { // Prepare for rsync rsync := func(oldPath string, newPath string, bwlimit string) error { output, err := rsyncLocalCopy(oldPath, newPath, bwlimit, true) @@ -1152,8 +1136,8 @@ func (s *storageDir) ContainerBackupCreate(backup backup, source Instance) error bwlimit := s.pool.Config["rsync.bwlimit"] // Handle snapshots - if !backup.instanceOnly { - snapshotsPath := fmt.Sprintf("%s/snapshots", tmpPath) + if !backup.InstanceOnly() { + snapshotsPath := fmt.Sprintf("%s/snapshots", path) // Retrieve the snapshots snapshots, err := source.Snapshots() @@ -1195,22 +1179,12 @@ func (s *storageDir) ContainerBackupCreate(backup backup, source Instance) error } // Copy the container - containerPath := fmt.Sprintf("%s/container", tmpPath) - err = rsync(source.Path(), containerPath, bwlimit) - if err != nil { - return err - } + containerPath := fmt.Sprintf("%s/container", path) - // Pack the backup - err = backupCreateTarball(s.s, tmpPath, backup) - if err != nil { - return err - } - - return nil + return rsync(source.Path(), containerPath, bwlimit) } -func (s *storageDir) ContainerBackupLoad(info backupInfo, data io.ReadSeeker, tarArgs []string) error { +func (s *storageDir) ContainerBackupLoad(info backup.Info, data io.ReadSeeker, tarArgs []string) error { _, err := s.StoragePoolMount() if err != nil { return err diff --git a/lxd/storage_lvm.go b/lxd/storage_lvm.go index d41704a8bf..bdb211368e 100644 --- a/lxd/storage_lvm.go +++ b/lxd/storage_lvm.go @@ -3,7 +3,6 @@ package main import ( "fmt" "io" - "io/ioutil" "os" "path/filepath" "strconv" @@ -13,6 +12,7 @@ import ( "github.com/pborman/uuid" "github.com/pkg/errors" + "github.com/lxc/lxd/lxd/backup" "github.com/lxc/lxd/lxd/migration" "github.com/lxc/lxd/lxd/operations" "github.com/lxc/lxd/lxd/project" @@ -1662,25 +1662,9 @@ func (s *storageLvm) ContainerSnapshotCreateEmpty(snapshotContainer Instance) er return nil } -func (s *storageLvm) ContainerBackupCreate(backup backup, source Instance) error { +func (s *storageLvm) ContainerBackupCreate(path string, backup backup.Backup, source Instance) error { poolName := s.getOnDiskPoolName() - // Start storage - ourStart, err := source.StorageStart() - if err != nil { - return err - } - if ourStart { - defer source.StorageStop() - } - - // Create a temporary path for the backup - tmpPath, err := ioutil.TempDir(shared.VarPath("backups"), "lxd_backup_") - if err != nil { - return err - } - defer os.RemoveAll(tmpPath) - // Prepare for rsync rsync := func(oldPath string, newPath string, bwlimit string) error { output, err := rsyncLocalCopy(oldPath, newPath, bwlimit, true) @@ -1694,8 +1678,8 @@ func (s *storageLvm) ContainerBackupCreate(backup backup, source Instance) error bwlimit := s.pool.Config["rsync.bwlimit"] // Handle snapshots - if !backup.instanceOnly { - snapshotsPath := fmt.Sprintf("%s/snapshots", tmpPath) + if !backup.InstanceOnly() { + snapshotsPath := fmt.Sprintf("%s/snapshots", path) // Retrieve the snapshots snapshots, err := source.Snapshots() @@ -1734,7 +1718,7 @@ func (s *storageLvm) ContainerBackupCreate(backup backup, source Instance) error // Make a temporary snapshot of the container sourceLvmDatasetSnapshot := fmt.Sprintf("snapshot-%s", uuid.NewRandom().String()) tmpContainerMntPoint := driver.GetContainerMountPoint(source.Project(), s.pool.Name, sourceLvmDatasetSnapshot) - err = os.MkdirAll(tmpContainerMntPoint, 0700) + err := os.MkdirAll(tmpContainerMntPoint, 0700) if err != nil { return err } @@ -1756,23 +1740,17 @@ func (s *storageLvm) ContainerBackupCreate(backup backup, source Instance) error } // Copy the container - containerPath := fmt.Sprintf("%s/container", tmpPath) + containerPath := fmt.Sprintf("%s/container", path) err = rsync(tmpContainerMntPoint, containerPath, bwlimit) s.umount(source.Project(), sourceLvmDatasetSnapshot, "") if err != nil { return err } - // Pack the backup - err = backupCreateTarball(s.s, tmpPath, backup) - if err != nil { - return err - } - return nil } -func (s *storageLvm) ContainerBackupLoad(info backupInfo, data io.ReadSeeker, tarArgs []string) error { +func (s *storageLvm) ContainerBackupLoad(info backup.Info, data io.ReadSeeker, tarArgs []string) error { containerPath, err := s.doContainerBackupLoad(info.Project, info.Name, info.Privileged, false) if err != nil { return err diff --git a/lxd/storage_mock.go b/lxd/storage_mock.go index d3ab28b093..7b9c2d1d80 100644 --- a/lxd/storage_mock.go +++ b/lxd/storage_mock.go @@ -5,6 +5,7 @@ import ( "github.com/gorilla/websocket" + "github.com/lxc/lxd/lxd/backup" "github.com/lxc/lxd/lxd/migration" "github.com/lxc/lxd/lxd/operations" "github.com/lxc/lxd/lxd/state" @@ -176,11 +177,11 @@ func (s *storageMock) ContainerSnapshotCreateEmpty(snapshotContainer Instance) e return nil } -func (s *storageMock) ContainerBackupCreate(backup backup, sourceContainer Instance) error { +func (s *storageMock) ContainerBackupCreate(path string, backup backup.Backup, sourceContainer Instance) error { return nil } -func (s *storageMock) ContainerBackupLoad(info backupInfo, data io.ReadSeeker, tarArgs []string) error { +func (s *storageMock) ContainerBackupLoad(info backup.Info, data io.ReadSeeker, tarArgs []string) error { return nil } diff --git a/lxd/storage_zfs.go b/lxd/storage_zfs.go index b0bf9c9df9..03bcd510d7 100644 --- a/lxd/storage_zfs.go +++ b/lxd/storage_zfs.go @@ -14,6 +14,7 @@ import ( "github.com/pkg/errors" "golang.org/x/sys/unix" + "github.com/lxc/lxd/lxd/backup" "github.com/lxc/lxd/lxd/migration" "github.com/lxc/lxd/lxd/operations" "github.com/lxc/lxd/lxd/project" @@ -1839,7 +1840,7 @@ func (s *storageZfs) ContainerSnapshotCreateEmpty(snapshotContainer Instance) er return nil } -func (s *storageZfs) doContainerOnlyBackup(tmpPath string, backup backup, source Instance) error { +func (s *storageZfs) doContainerOnlyBackup(tmpPath string, backup backup.Backup, source Instance) error { sourceIsSnapshot := source.IsSnapshot() poolName := s.getOnDiskPoolName() @@ -1887,7 +1888,7 @@ func (s *storageZfs) doContainerOnlyBackup(tmpPath string, backup backup, source return nil } -func (s *storageZfs) doSnapshotBackup(tmpPath string, backup backup, source Instance, parentSnapshot string) error { +func (s *storageZfs) doSnapshotBackup(tmpPath string, backup backup.Backup, source Instance, parentSnapshot string) error { sourceName := source.Name() snapshotsPath := fmt.Sprintf("%s/snapshots", tmpPath) @@ -1919,14 +1920,14 @@ func (s *storageZfs) doSnapshotBackup(tmpPath string, backup backup, source Inst return zfsSendCmd.Run() } -func (s *storageZfs) doContainerBackupCreateOptimized(tmpPath string, backup backup, source Instance) error { +func (s *storageZfs) doContainerBackupCreateOptimized(tmpPath string, backup backup.Backup, source Instance) error { // Handle snapshots snapshots, err := source.Snapshots() if err != nil { return err } - if backup.instanceOnly || len(snapshots) == 0 { + if backup.InstanceOnly() || len(snapshots) == 0 { err = s.doContainerOnlyBackup(tmpPath, backup, source) } else { prev := "" @@ -1988,7 +1989,7 @@ func (s *storageZfs) doContainerBackupCreateOptimized(tmpPath string, backup bac return nil } -func (s *storageZfs) doContainerBackupCreateVanilla(tmpPath string, backup backup, source Instance) error { +func (s *storageZfs) doContainerBackupCreateVanilla(tmpPath string, backup backup.Backup, source Instance) error { // Prepare for rsync rsync := func(oldPath string, newPath string, bwlimit string) error { output, err := rsyncLocalCopy(oldPath, newPath, bwlimit, true) @@ -2000,10 +2001,10 @@ func (s *storageZfs) doContainerBackupCreateVanilla(tmpPath string, backup backu } bwlimit := s.pool.Config["rsync.bwlimit"] - projectName := backup.instance.Project() + projectName := source.Project() // Handle snapshots - if !backup.instanceOnly { + if !backup.InstanceOnly() { snapshotsPath := fmt.Sprintf("%s/snapshots", tmpPath) // Retrieve the snapshots @@ -2091,46 +2092,24 @@ func (s *storageZfs) doContainerBackupCreateVanilla(tmpPath string, backup backu return nil } -func (s *storageZfs) ContainerBackupCreate(backup backup, source Instance) error { - // Start storage - ourStart, err := source.StorageStart() - if err != nil { - return err - } - if ourStart { - defer source.StorageStop() - } - - // Create a temporary path for the backup - tmpPath, err := ioutil.TempDir(shared.VarPath("backups"), "lxd_backup_") - if err != nil { - return err - } - defer os.RemoveAll(tmpPath) - +func (s *storageZfs) ContainerBackupCreate(path string, backup backup.Backup, source Instance) error { // Generate the actual backup - if backup.optimizedStorage { - err = s.doContainerBackupCreateOptimized(tmpPath, backup, source) + if backup.OptimizedStorage() { + err := s.doContainerBackupCreateOptimized(path, backup, source) if err != nil { return errors.Wrap(err, "Optimized backup") } } else { - err = s.doContainerBackupCreateVanilla(tmpPath, backup, source) + err := s.doContainerBackupCreateVanilla(path, backup, source) if err != nil { return errors.Wrap(err, "Vanilla backup") } } - // Pack the backup - err = backupCreateTarball(s.s, tmpPath, backup) - if err != nil { - return err - } - return nil } -func (s *storageZfs) doContainerBackupLoadOptimized(info backupInfo, data io.ReadSeeker, tarArgs []string) error { +func (s *storageZfs) doContainerBackupLoadOptimized(info backup.Info, data io.ReadSeeker, tarArgs []string) error { containerName, _, _ := shared.ContainerGetParentAndSnapshotName(info.Name) containerMntPoint := driver.GetContainerMountPoint(info.Project, s.pool.Name, containerName) err := driver.CreateContainerMountpoint(containerMntPoint, driver.ContainerPath(info.Name, false), info.Privileged) @@ -2240,7 +2219,7 @@ func (s *storageZfs) doContainerBackupLoadOptimized(info backupInfo, data io.Rea return nil } -func (s *storageZfs) doContainerBackupLoadVanilla(info backupInfo, data io.ReadSeeker, tarArgs []string) error { +func (s *storageZfs) doContainerBackupLoadVanilla(info backup.Info, data io.ReadSeeker, tarArgs []string) error { // create the main container err := s.doContainerCreate(info.Project, info.Name, info.Privileged) if err != nil { @@ -2302,7 +2281,7 @@ func (s *storageZfs) doContainerBackupLoadVanilla(info backupInfo, data io.ReadS return nil } -func (s *storageZfs) ContainerBackupLoad(info backupInfo, data io.ReadSeeker, tarArgs []string) error { +func (s *storageZfs) ContainerBackupLoad(info backup.Info, data io.ReadSeeker, tarArgs []string) error { logger.Debugf("Loading ZFS storage volume for backup \"%s\" on storage pool \"%s\"", info.Name, s.pool.Name) if info.HasBinaryFormat { From a9fa1f6bab102f294916c30d419883c24868c424 Mon Sep 17 00:00:00 2001 From: Thomas Hipp <thomas.h...@canonical.com> Date: Mon, 7 Oct 2019 14:33:51 +0200 Subject: [PATCH 2/2] test: Add backup package to static analysis Signed-off-by: Thomas Hipp <thomas.h...@canonical.com> --- test/suites/static_analysis.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/test/suites/static_analysis.sh b/test/suites/static_analysis.sh index 44786668d3..79b41810e0 100644 --- a/test/suites/static_analysis.sh +++ b/test/suites/static_analysis.sh @@ -72,6 +72,7 @@ test_static_analysis() { golint -set_exit_status lxd-p2c + golint -set_exit_status lxd/backup golint -set_exit_status lxd/config golint -set_exit_status lxd/db golint -set_exit_status lxd/db/node
_______________________________________________ lxc-devel mailing list lxc-devel@lists.linuxcontainers.org http://lists.linuxcontainers.org/listinfo/lxc-devel