Factor out replaceFileString functions

Another task to be performed in a different fashion depending on whether a list of filter commands is a "simple" pipeline or a spring is the replacement of '{file}' string with the actual file name when performing the backup or restore operation. This change factors out a new function that performs the proper replacement, depending on the type of the commands to execute. The behavior is largely different between the two types: in case of a pipeline we replicate the '{file}' string for all the files we want to expand it to and perform the replacement. Associated (long) options are replicated as well. For springs, however, we clone the entire command and substitute the string. The reason for this behavior is that the introduction of spring support was based on the premise that certain programs cannot handle multiple files. So, in such a case we have to invoke the program multiple times instead--with each instance operating on a different file.

Author: d-e-s-o

btrfs-backup/src/deso/btrfs/commands.py

@@ -19,6 +19,13 @@
 
 """Functionality related to handling commands in various forms."""
 
+from copy import (
+ deepcopy,
+)
+from deso.execute import (
+ formatCommands,
+)
+
 
 def isSpring(command):
  """Check if a command array actually describes a spring."""
@@ -55,12 +62,33 @@ def checkFileString(command):
  return _checkFileStringInCommand(command)
 
 
-def replaceFileString(command, files):
- """Replace the {file} string in a command with the actual file name.
+def _replaceFileStringInSpring(commands, files):
+ """Replace the {file} string in a spring with the actual file name(s)."""
+ def replicate(command, file_):
+ """Replicate and command containing a {file} string and replace it with the actual file."""
+ replica = deepcopy(command)
+ result = _replaceFileStringInCommand(replica, [file_])
+ assert result, replica
+ return replica
 
- Note: Any replacement is performed on the actual data, i.e., without
- making a copy beforehand.
- """
+ assert isSpring(commands), commands
+
+ # Note that we do not insist here on the {file} string being located
+ # in the first command of the spring. Not sure about use cases where
+ # it would not be there but we leave that open to users.
+ for i, command in enumerate(commands):
+ if "{file}" in " ".join(command):
+ # For a spring we replicate the entire command containing the
+ # '{file}' string (as opposed to the option associated with it, if
+ # any) and insert a duplicate into the list of commands.
+ commands[i:i+1] = [replicate(command, f) for f in files]
+ return True
+
+ return False
+
+
+def _replaceFileStringInCommand(command, files):
+ """Replace the {file} string in a command with the actual file name(s)."""
  # TODO: This function replicates the argument that contains the {file}
  # string to allow not only for lists of snapshot files in a
  # consecutive fashion (i.e., "/bin/cat {file}" is expanded to
@@ -86,3 +114,29 @@ def replaceFileString(command, files):
  return True
 
  return False
+
+
+def replaceFileString(command, files):
+ """Replace the {file} string in a command with the actual snapshot name.
+
+ Note: Any replacement is performed on the actual data, i.e., without
+ making a copy beforehand.
+ """
+ def raiseError(commands):
+ """Raise an error telling the user that no {file} string was found."""
+ error = "Replacement string {{file}} not found in: \"{cmd}\""
+ error = error.format(cmd=formatCommands(commands))
+ raise NameError(error)
+
+ assert isinstance(files, list), files
+
+ # The command might actually be a list of commands in case of a
+ # spring. In that case our replacement looks slightly different.
+ if isSpring(command):
+ if not _replaceFileStringInSpring(command, files):
+ raiseError(command)
+ else:
+ if not _replaceFileStringInCommand(command, files):
+ raiseError([command])
+
+ return command

btrfs-backup/src/deso/btrfs/repository.py

@@ -49,7 +49,6 @@
 )
 from deso.execute import (
  execute,
- formatCommands,
  pipeline,
 )
 from os import (
@@ -855,22 +854,13 @@ def parents(self, snapshot, subvolume, snapshots=None, dst_snapshots=None):
 
  def _filterPipeline(self, index, snapshots):
  """Create a pipeline out of the filter commands."""
- def replaceFiles(command, snapshots):
- """Replace the {file} string in a command with the actual snapshot name."""
- if not replaceFileString(command, snapshots):
- error = "Replacement string {{file}} not found in command: \"{cmd}\""
- error = error.format(cmd=formatCommands(command))
- raise NameError(error)
- else:
- return command
-
  # Convert all relative snapshot paths into absolute ones with the
  # appropriate extension.
  with alias(self._extension) as ext:
  snapshots = ["%s%s" % (self.path(s), ext) for s in snapshots]
 
  commands = deepcopy(self._filters)
- func = lambda: replaceFiles(commands[index], snapshots)
+ func = lambda: replaceFileString(commands[index], snapshots)
  # At least one command in the filters must contain a string {file}
  # which is now replaced by the actual snapshot name.
  commands[index] = self.command(func)

btrfs-backup/src/deso/btrfs/test/testCommands.py

@@ -23,6 +23,8 @@
  checkFileString,
  isSpring,
  replaceFileString,
+ _replaceFileStringInCommand,
+ _replaceFileStringInSpring,
 )
 from unittest import (
  main,
@@ -62,31 +64,104 @@ def testCheckFileString(self):
  self.assertFalse(checkFileString(command))
 
 
- def testReplaceFileString(self):
- """Verify that the replaceFileString function works as expected."""
- self.assertFalse(replaceFileString(["cat"], ["test"]), None)
- self.assertFalse(replaceFileString(["cat", "-o"], ["test"]), None)
+ def testReplaceFileStringInCommand(self):
+ """Verify that the _replaceFileStringInCommand function works as expected."""
+ self.assertFalse(_replaceFileStringInCommand(["cat"], ["test"]), None)
+ self.assertFalse(_replaceFileStringInCommand(["cat", "-o"], ["test"]), None)
 
  command = ["cat", "{file}"]
  expected = ["cat", "test"]
- self.assertTrue(replaceFileString(command, ["test"]))
+ self.assertTrue(_replaceFileStringInCommand(command, ["test"]))
  self.assertEqual(command, expected)
 
  command = ["cat", "-o", "{file}", "-a", "test2"]
  expected = ["cat", "-o", "test", "-a", "test2"]
- self.assertTrue(replaceFileString(command, ["test"]))
+ self.assertTrue(_replaceFileStringInCommand(command, ["test"]))
  self.assertEqual(command, expected)
 
  command = ["cat", "--a-long-option={file}", "-a", "test2"]
  expected = ["cat", "--a-long-option=test", "-a", "test2"]
- self.assertTrue(replaceFileString(command, ["test"]))
+ self.assertTrue(_replaceFileStringInCommand(command, ["test"]))
  self.assertEqual(command, expected)
 
  command = ["cat", "--input={file}", "-a", "test3"]
  expected = ["cat", "--input=test1", "--input=test2", "-a", "test3"]
- self.assertTrue(replaceFileString(command, ["test1", "test2"]))
+ self.assertTrue(_replaceFileStringInCommand(command, ["test1", "test2"]))
  self.assertEqual(command, expected)
 
 
+ def testReplaceFileStringInSpring(self):
+ """Verify that the _replaceFileStringInSpring function works as expected."""
+ commands = [["cat", "test"]]
+ self.assertFalse(_replaceFileStringInSpring(commands, ["test"]))
+
+ commands = [["cat", "-o", "test"]]
+ self.assertFalse(_replaceFileStringInSpring(commands, ["test"]))
+
+ commands = [["cat", "{file}"]]
+ expected = [["cat", "test"]]
+ self.assertTrue(_replaceFileStringInSpring(commands, ["test"]))
+ self.assertEqual(commands, expected)
+
+ commands = [["cat", "-o", "{file}", "-a", "test2"]]
+ expected = [
+ ["cat", "-o", "test1", "-a", "test2"],
+ ["cat", "-o", "test2", "-a", "test2"],
+ ["cat", "-o", "test3", "-a", "test2"],
+ ]
+ files = ["test1", "test2", "test3"]
+ self.assertTrue(_replaceFileStringInSpring(commands, files))
+ self.assertEqual(commands, expected)
+
+ commands = [["cat", "--a-long-option={file}"]]
+ expected = [
+ ["cat", "--a-long-option=1"],
+ ["cat", "--a-long-option=2"],
+ ["cat", "--a-long-option=42"],
+ ]
+ self.assertTrue(_replaceFileStringInSpring(commands, ["1", "2", "42"]))
+ self.assertEqual(commands, expected)
+
+ # We also allow command other than the first to contain the {file}
+ # string.
+ commands = [["cat", "test"], ["cat", "--a-long-option={file}"]]
+ expected = [
+ ["cat", "test"],
+ ["cat", "--a-long-option=1"],
+ ["cat", "--a-long-option=2"],
+ ["cat", "--a-long-option=42"],
+ ]
+ self.assertTrue(_replaceFileStringInSpring(commands, ["1", "2", "42"]))
+ self.assertEqual(commands, expected)
+
+
+ def testReplaceFileStringSuccess(self):
+ """Verify that the replaceFileString function works as expected."""
+ # Test the function with a "normal" command.
+ command = ["cat", "{file}"]
+ expected = ["cat", "test"]
+ self.assertEqual(replaceFileString(command, ["test"]), expected)
+
+ # Test the function with a spring.
+ command = [["cat", "{file}"], ["echo", "success"]]
+ expected = [["cat", "test"], ["echo", "success"]]
+ self.assertEqual(replaceFileString(command, ["test"]), expected)
+
+
+ def testReplaceFileStringFailure(self):
+ """Verify that the replaceFileString function throws correct errors."""
+ regex = r"Replacement string.*in: \"%s\"$"
+ command = [
+ ["cat", "test1"],
+ ["cat", "test2"],
+ ]
+ with self.assertRaisesRegex(NameError, regex % "cat test1 | cat test2"):
+ replaceFileString(command, ["test"])
+
+ command = ["echo", "data"]
+ with self.assertRaisesRegex(NameError, regex % "echo data"):
+ replaceFileString(command, ["test"])
+
+
 if __name__ == "__main__":
  main()