Introduce remote command execution support

Arguably, in a fair share of cases one would want to backup data not to a local device but rather a remote entity. One possibility for remotely accessing file systems is by mounting them locally using NFS, for example. However, once mounted over NFS, certain file system properties are abstracted from. The particularities of a btrfs file system, for instance, are not exposed through NFS (and similar approaches). This constraint renders any such mounting related approach effectively unusable for btrfs-backup. In order to still be able to backup data remotely, this change introduces support for running commands remotely instead of only locally. Since btrfs-backup is basically a sophisticated command execution engine around the btrfs tool suite, we can execute the commands not only locally but also remotely, using SSH, for instance (this fact of course implies that the remote side has a btrfs file system available along with the btrfs programs). With this change it is now possible to specify a remote command to use by means of the newly introduced --remote-cmd option. The command will be used to prefix all actually performed btrfs commands which means that the remote command must accept the command to execute remotely as its argument. Note that although the approach was designed with SSH in mind, really any command that allows for connecting to a remote host and running a command should work.

Author: d-e-s-o

btrfs-backup/README.md

@@ -93,3 +93,28 @@ achieved by means of the snapshots-only option, like so:
 $ btrfs-backup restore --snapshots-only
  --subvolume=subvolume/ backup/ snapshots/
 ```
+
+### Remote Execution
+In many cases it is a requirement to backup a subvolume to a remote host
+(or restore a subvolume from it). Mounting a remote btrfs file system
+locally by means of, for instance, sshfs will not provide the ability to
+use btrfs specific tools on it.
+To that end, commands can be run on the remote host directly (provided
+it offers an interface for command execution from the outside and that
+is has the required btrfs tool suite installed). A typical example for
+remote command execution is SSH. Using **btrfs-backup** on a remote host
+by means of an SSH connection can be achieved with the --remote-cmd
+option:
+
+```
+$ btrfs-backup backup --remote-cmd='/usr/bin/ssh server'
+ --subvolume=subvolume/ snapshots/ backup/
+```
+
+In this example, we assume that by invoking '/usr/bin/ssh server'
+locally we can establish a connection to the remote server. The command
+specified using the --remote-cmd option has to be given with the full
+path. Furthermore, this command must accept the command to execute
+remotely (that is, on the host named 'server' in our example above) as
+its arguments. Note that backup/ in this case does not refer to a local
+folder but rather one on the remote side.

btrfs-backup/src/deso/btrfs/main.py

@@ -54,7 +54,7 @@ def version():
  return "0.1"
 
 
-def run(method, subvolumes, src_repo, dst_repo, **kwargs):
+def run(method, subvolumes, src_repo, dst_repo, remote_cmd=None, **kwargs):
  """Start actual execution."""
  try:
  # This import pulls in all required modules and we check for
@@ -65,8 +65,14 @@ def run(method, subvolumes, src_repo, dst_repo, **kwargs):
  print("A command was not found:\n\"%s\"" % e, file=stderr)
  return 1
 
+ if remote_cmd:
+ # TODO: Right now we do not support remote commands that contain
+ # spaces in their path. E.g., "/bin/connect to server" would
+ # not be a valid command.
+ remote_cmd = remote_cmd.split()
+
  try:
- program = Program(subvolumes, src_repo, dst_repo)
+ program = Program(subvolumes, src_repo, dst_repo, remote_cmd)
  method(program)(**kwargs)
  return 0
  except ChildProcessError as e:
@@ -112,7 +118,13 @@ def addStandardArgs(parser):
 
 
 def addOptionalArgs(parser):
- """Add the optional --reverse argument to a parser."""
+ """Add the optional arguments to a parser."""
+ parser.add_argument(
+ "--remote-cmd", action="store", dest="remote_cmd", metavar="command",
+ help="The command to use for running btrfs on a remote site. Needs "
+ "to include the full path to the binary or script, e.g., "
+ "\"/usr/bin/ssh server\".",
+ )
  parser.add_argument(
  "--reverse", action="store_true", dest="reverse", default=False,
  help="Reverse (i.e., swap) the source and destination repositories.",
@@ -235,9 +247,11 @@ def main(argv):
 
  if ns.command == "backup":
  return run(lambda x: x.backup, subvolumes, src_repo, dst_repo,
+ remote_cmd=ns.remote_cmd,
  keep_for=ns.keep_for)
  elif ns.command == "restore":
  return run(lambda x: x.restore, subvolumes, src_repo, dst_repo,
+ remote_cmd=ns.remote_cmd,
  snapshots_only=ns.snapshots_only)
  else:
  assert False

btrfs-backup/src/deso/btrfs/program.py

@@ -28,11 +28,11 @@
 
 class Program:
  """A program object performs the actual work of synchronizing two repositories."""
- def __init__(self, subvolumes, src_repo, dst_repo):
+ def __init__(self, subvolumes, src_repo, dst_repo, remote_cmd=None):
  """Create a new Program object using the given subvolumes and repositories."""
  self._subvolumes = subvolumes
  self._src_repo = Repository(src_repo)
- self._dst_repo = Repository(dst_repo)
+ self._dst_repo = Repository(dst_repo, remote_cmd)
 
 
  def backup(self, keep_for=None):

btrfs-backup/src/deso/btrfs/repository.py

@@ -125,8 +125,8 @@ def _parseDiffLine(line):
  return path
 
 
-def _snapshots(directory):
- """Retrieve a list of snapshots in a directory.
+def _snapshots(repository):
+ """Retrieve a list of snapshots in a repository.
 
  Note:
  Because of a supposed bug in btrfs' handling of passed in
@@ -136,7 +136,8 @@ def _snapshots(directory):
  usage of this function is discouraged. Use the Repository's
  snapshots() method instead.
  """
- output, _ = execute(*listSnapshots(directory), read_out=True)
+ cmd = repository.command(listSnapshots, repository.path())
+ output, _ = execute(*cmd, read_out=True)
  # We might retrieve an empty output if no snapshots were present. In
  # this case, just return early here.
  if not output:
@@ -147,9 +148,10 @@ def _snapshots(directory):
  return [_parseListLine(line) for line in output]
 
 
-def _isRoot(directory):
+def _isRoot(directory, repository):
  """Check if a given directory represents the root of a btrfs file system."""
- output, _ = execute(*show(directory), read_out=True)
+ cmd = repository.command(show, directory)
+ output, _ = execute(*cmd, read_out=True)
  output = output.decode("utf-8")[:-1].split("\n")
 
  # The output of show() contains multiple lines in case the given
@@ -160,7 +162,7 @@ def _isRoot(directory):
  return len(output) == 1 and output[0].endswith(_SHOW_IS_ROOT)
 
 
-def _findRoot(directory):
+def _findRoot(directory, repository):
  """Find the root of the btrfs file system containing the given directory."""
  assert directory
  assert directory == abspath(directory)
@@ -169,7 +171,7 @@ def _findRoot(directory):
  # or later because of a dirname invocation. However, the show command
  # in _isRoot will fail for an empty directory (a case that will also
  # be hit if this function is run on a non-btrfs file system).
- while not _isRoot(directory):
+ while not _isRoot(directory, repository):
  new_directory = dirname(directory)
 
  # Executing a dirname on the root directory ('/') just returns the
@@ -302,8 +304,9 @@ def _findCommonSnapshots(src_snaps, dst_snaps):
 def _createSnapshot(subvolume, repository, snapshots):
  """Create a snapshot of the given subvolume in the given repository."""
  name = _snapshotName(subvolume, snapshots)
+ cmd = repository.command(mkSnapshot, subvolume, repository.path(name))
 
- execute(*mkSnapshot(subvolume, repository.path(name)))
+ execute(*cmd)
  return name
 
 
@@ -314,7 +317,7 @@ def _findOrCreate(subvolume, repository, snapshots):
  # If we found no snapshot or if files are changed between the current
  # state of the subvolume and the most recent snapshot we just found
  # then create a new snapshot.
- if not snapshot or _diff(snapshot, subvolume):
+ if not snapshot or _diff(snapshot, subvolume, repository):
  old = snapshot["path"] if snapshot else None
  new = _createSnapshot(subvolume, repository, snapshots)
  return new, old
@@ -355,12 +358,12 @@ def _deploy(snapshot, parent, src, dst, src_snaps, subvolume):
 
  # Be sure to have the snapshot persisted to disk before trying to
  # serialize it.
- execute(*syncFs(src.root))
+ execute(*src.command(syncFs, src.root))
  # Finally transfer the snapshot from the source repository to the
  # destination.
  pipeline([
- serialize(src.path(snapshot), parents),
- deserialize(dst.path())
+ src.command(serialize, src.path(snapshot), parents),
+ dst.command(deserialize, dst.path()),
  ])
 
 
@@ -440,7 +443,8 @@ def _restore(subvolume, src, dst, snapshots, snapshots_only):
  # Now that we got the snapshot back on the destination repository,
  # we can restore the actual subvolume from it (if desired).
  if not snapshots_only:
- execute(*mkSnapshot(dst.path(snapshot), subvolume, writable=True))
+ cmd = dst.command(mkSnapshot, dst.path(snapshot), subvolume, writable=True)
+ execute(*cmd)
 
 
 def restore(subvolumes, src, dst, snapshots_only=False):
@@ -455,7 +459,7 @@ def restore(subvolumes, src, dst, snapshots_only=False):
  _restore(subvolume, src, dst, snapshots, snapshots_only)
 
 
-def _diff(snapshot, subvolume):
+def _diff(snapshot, subvolume, repository):
  """Find the files that changed in a given subvolume with respect to a snapshot."""
  # Because of an apparent bug in btrfs(8) (or a misunderstanding on my
  # side), we cannot use the generation reported for a snapshot to
@@ -469,7 +473,8 @@ def _diff(snapshot, subvolume):
  # to clarify whether a new snapshot *always* also means a new
  # generation (I assume so, but it would be best to get
  # confirmation).
- output, _ = execute(*diff(subvolume, generation), read_out=True)
+ cmd = repository.command(diff, subvolume, generation)
+ output, _ = execute(*cmd, read_out=True)
  output = output.decode("utf-8")[:-1].split("\n")
  # The diff output usually is ended by a line such as:
  # "transid marker was" followed by a generation ID. We should ignore
@@ -500,7 +505,8 @@ def _purge(subvolume, repository, duration, snapshots):
  # old enough so that the snapshot should be deleted.
  time = datetime.strptime(string, _TIME_FORMAT)
  if time + duration < now:
- execute(*delete(repository.path(snapshot)))
+ cmd = repository.command(delete, repository.path(snapshot))
+ execute(*cmd)
 
 
 def _trail(path):
@@ -510,18 +516,19 @@ def _trail(path):
 
 class Repository:
  """This class represents a repository for snapshots."""
- def __init__(self, directory):
+ def __init__(self, directory, remote_cmd=None):
  """Initialize the object and bind it to the given directory."""
  # We always work with absolute paths here.
  directory = abspath(directory)
 
- self._root = _findRoot(directory)
+ self._remote_cmd = remote_cmd
+ self._root = _findRoot(directory, self)
  self._directory = _trail(directory)
 
 
  def snapshots(self):
  """Retrieve a list of snapshots in this repository."""
- snapshots = _snapshots(self._directory)
+ snapshots = _snapshots(self)
 
  # We need to work around the btrfs problem that not necessarily all
  # snapshots listed are located in our repository's directory.
@@ -584,14 +591,20 @@ def diff(self, snapshot, subvolume):
  if not found:
  raise FileNotFoundError("Snapshot not found: \"%s\"" % snapshot)
 
- return _diff(found, subvolume)
+ return _diff(found, subvolume, self)
 
 
  def path(self, *components):
  """Form an absolute path by combining the given path components."""
  return join(self._directory, *components)
 
 
+ def command(self, function, *args, **kwargs):
+ """Create a command."""
+ command = function(*args, **kwargs)
+ return (self._remote_cmd if self._remote_cmd else []) + command
+
+
  @property
  def root(self):
  """Retrieve the root directory of the btrfs file system the repository resides on."""

btrfs-backup/src/deso/btrfs/test/testMain.py

@@ -44,6 +44,7 @@
 )
 from deso.execute import (
  execute,
+ pipeline,
 )
 from os.path import (
  join,
@@ -174,6 +175,66 @@ def testKeepFor(self):
  self.assertEqual(len(glob(m.path("snapshots", "*"))), 1)
 
 
+ def testRemoteCommand(self):
+ """Verify that the remote command correctly prefixes some btrfs commands."""
+ remote_cmd = "/usr/bin/ssh server"
+
+ def isRemote(command):
+ """Check if a command is a remote command."""
+ return " ".join(command).startswith(remote_cmd)
+
+ def filterCommand(command):
+ """Filter all remote command parts from a command (if any)."""
+ # Since we do not have an SSH server running (and do not want to),
+ # filter out the remote command part before execution.
+ if isRemote(command):
+ command = command[2:]
+
+ return command
+
+ def filterCommands(commands):
+ """Filter all remote command parts from a command list (if any)."""
+ return [filterCommand(cmd) for cmd in commands]
+
+ def executeWrapper(*args, **kwargs):
+ """Wrapper around the execute function that stores the commands it executed."""
+ command = list(args)
+ all_commands.append(command)
+ command = filterCommand(command)
+
+ return execute(*command, **kwargs)
+
+ def pipelineWrapper(commands, *args, **kwargs):
+ """Wrapper around the pipeline function that stores the commands it executed."""
+ for command in commands:
+ all_commands.append(command)
+
+ commands = filterCommands(commands)
+ return pipeline(commands, *args, **kwargs)
+
+ all_commands = []
+ with patch("deso.btrfs.repository.execute", wraps=executeWrapper),\
+ patch("deso.btrfs.repository.pipeline", wraps=pipelineWrapper):
+ with alias(self._mount) as m:
+ make(m, "subvol", subvol=True)
+ make(m, "snapshots")
+ make(m, "backup")
+
+ remote = "--remote-cmd=%s" % remote_cmd
+ args = "backup --subvolume {subvol} {src} {dst}"
+ args = args.format(subvol=m.path("subvol"),
+ src=m.path("snapshots"),
+ dst=m.path("backup"))
+ result = btrfsMain([argv[0]] + args.split() + [remote])
+ self.assertEqual(result, 0)
+
+ # We do not check all commands here. However, we know for sure that
+ # btrfs receive should be run on the remote side. So check that it
+ # is properly prefixed.
+ receive, = list(filter(lambda x: "receive" in x, all_commands))
+ self.assertTrue(isRemote(receive), receive)
+
+
  def testRun(self):
  """Test a simple run of the program with two subvolumes."""
  def wipeSubvolumes(path, pattern="*"):

btrfs-backup/src/deso/btrfs/test/testRepository.py

@@ -38,7 +38,6 @@
  Repository,
  restore,
  _findCommonSnapshots,
- _findRoot,
  _snapshots,
  sync as syncRepos,
 )
@@ -92,21 +91,22 @@ def testRepositoryListNoSnapshotPresent(self):
  def testRepositorySubvolumeFindRootOnRoot(self):
  """Test retrieval of the absolute path of the btrfs root from the root."""
  with alias(self._mount) as m:
- self.assertEqual(_findRoot(m.path()), m.path())
+ repo = Repository(m.path())
+ self.assertEqual(repo.root, m.path())
 
 
  def testRepositorySubvolumeFindRootFromBelowRoot(self):
  """Test retrieval of the absolute path of the btrfs root from a sub-directory."""
  with alias(self._mount) as m:
- directory = make(m, "dir")
- self.assertEqual(_findRoot(directory), m.path())
+ repo = Repository(make(m, "dir"))
+ self.assertEqual(repo.root, m.path())
 
 
  def testRepositorySubvolumeFindRootFromSubvolume(self):
  """Test retrieval of the absolute path of the btrfs root from a true subvolume."""
  with alias(self._mount) as m:
- root = make(m, "root", subvol=True)
- self.assertEqual(_findRoot(root), m.path())
+ repo = Repository(make(m, "root", subvol=True))
+ self.assertEqual(repo.root, m.path())
 
 
  def testRepositoryFindCommonSnapshots(self):
@@ -217,13 +217,14 @@ def testRepositoryListNoSnapshotPresentInSubdir(self):
  with alias(self._mount) as m:
  # Create a new sub-directory where no snapshots are present.
  directory = make(m, "dir")
+ repository = Repository(directory)
  # TODO: The assertion should really be assertEqual! I consider
  # this behavior a bug because we pass in the -o option to
  # the list command which is promoted as: "print only
  # subvolumes below specified path".
  # There is no subvolume below the given path, so reporting a
  # snapshot here is wrong.
- self.assertNotEqual(_snapshots(directory), [])
+ self.assertNotEqual(_snapshots(repository), [])
 
 
  def testRepositoryListOnlySnapshotsInRepository(self):