diff options
| author | Michael Vogt <michael.vogt@ubuntu.com> | 2015-03-10 14:13:10 +0100 |
|---|---|---|
| committer | git-ubuntu importer <ubuntu-devel-discuss@lists.ubuntu.com> | 2015-03-10 13:14:10 +0000 |
| commit | 14bb5b070b35516cd1b6f54f8db36fdec0109ce3 (patch) | |
| tree | 2d8f6891e9d1c36de4dceb2b435a2d6de287f684 | |
| parent | 27685142f69ecd273769f5691f766dcf89bf523c (diff) | |
| parent | b29322c022f2563f4a300d86eadaf9bebce5f0c1 (diff) | |
0.7.6 (patches applied)applied/0.7.6
Imported using git-ubuntu import.
| -rwxr-xr-x | bin/ubuntu-core-upgrade | 16 | ||||
| -rw-r--r-- | debian/changelog | 12 | ||||
| -rw-r--r-- | functional/__init__.py | 0 | ||||
| -rw-r--r-- | functional/test_upgrader.py | 628 | ||||
| -rw-r--r-- | ubuntucoreupgrader/tests/test_upgrader.py | 199 | ||||
| -rw-r--r-- | ubuntucoreupgrader/tests/utils.py | 266 | ||||
| -rw-r--r-- | ubuntucoreupgrader/upgrader.py | 1148 |
7 files changed, 952 insertions, 1317 deletions
diff --git a/bin/ubuntu-core-upgrade b/bin/ubuntu-core-upgrade index ebaed2c..16805ed 100755 --- a/bin/ubuntu-core-upgrade +++ b/bin/ubuntu-core-upgrade @@ -56,10 +56,6 @@ def setup_logger(options): if options.debug: log.setLevel(logging.DEBUG) - if options.dry_run or options.check_reboot: - # use default logger which displays output to stderr only. - return - # send data to syslog... handler = logging.handlers.SysLogHandler(address='/dev/log') @@ -105,26 +101,14 @@ def prepare_upgrade(options): def main(): options = parse_args(sys.argv[1:]) - if options.reboot_delay and options.reboot_delay < 0: - sys.exit('ERROR: must specify a positive number') - if not os.path.exists(options.root_dir): sys.exit('ERROR: root directory does not exist: {}' .format(options.root_dir)) - if options.check_reboot: - # check options must be inert - options.dry_run = True - if options.dry_run or options.root_dir != '/': prepare_upgrade(options) sys.exit(0) - if options.show_other_details: - upgrader = Upgrader(options, [], None) - upgrader.show_other_partition_details() - sys.exit(0) - if not options.cmdfile: sys.exit('ERROR: need command file') diff --git a/debian/changelog b/debian/changelog index e8b3a98..ce1d3e6 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,15 @@ +ubuntu-core-upgrader (0.7.6) vivid; urgency=low + + [ James Hunt] + * ubuntucoreupgrader/upgrader.py: + - get_file_contents(): Fix to avoid leaking the fd. + * functional/test_upgrader.py: Basic set of functional tests. + + [ Michael Vogt ] + * cleanup and removal of unused code + + -- Michael Vogt <michael.vogt@ubuntu.com> Tue, 10 Mar 2015 14:13:10 +0100 + ubuntu-core-upgrader (0.7.5) vivid; urgency=low [ Michael Vogt ] diff --git a/functional/__init__.py b/functional/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/functional/__init__.py diff --git a/functional/test_upgrader.py b/functional/test_upgrader.py new file mode 100644 index 0000000..8609376 --- /dev/null +++ b/functional/test_upgrader.py @@ -0,0 +1,628 @@ +#!/usr/bin/python3 +# -*- coding: utf-8 -*- +# -------------------------------------------------------------------- +# Copyright © 2014-2015 Canonical Ltd. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# -------------------------------------------------------------------- + +# -------------------------------------------------------------------- +# Functional tests for the snappy upgrader. +# -------------------------------------------------------------------- + +import sys +import os +import logging +import unittest +import shutil + +from ubuntucoreupgrader.upgrader import ( + Upgrader, + parse_args, +) + +base_dir = os.path.abspath(os.path.dirname(__file__)) +module_dir = os.path.normpath(os.path.realpath(base_dir + os.sep + '..')) +sys.path.append(base_dir) + +from ubuntucoreupgrader.tests.utils import ( + make_tmp_dir, + append_file, + TEST_DIR_MODE, + create_file, + UbuntuCoreUpgraderTestCase, + ) + +CMD_FILE = 'ubuntu_command' + + +def call_upgrader(command_file, root_dir, update): + ''' + Invoke the upgrader. + + :param command_file: commands file to drive the upgrader. + :param root_dir: Test directory to apply the upgrade to. + :param update: UpdateTree object. + ''' + + args = [] + args += ['--root-dir', root_dir] + args += ['--debug', '1'] + + # don't delete the archive and command files. + # The tests clean up after themselves so they will get removed then, + # but useful to have them around to diagnose test failures. + args.append('--leave-files') + + args.append(command_file) + commands = file_to_list(command_file) + + cache_dir = make_tmp_dir() + + def mock_get_cache_dir(): + cache_dir = update.tmp_dir + sys_dir = os.path.join(cache_dir, 'system') + os.makedirs(sys_dir, exist_ok=True) + return cache_dir + + upgrader = Upgrader(parse_args(args), commands, []) + upgrader.get_cache_dir = mock_get_cache_dir + upgrader.MOUNTPOINT_CMD = "true" + upgrader.run() + + shutil.rmtree(cache_dir) + + +def create_device_file(path, type='c', major=-1, minor=-1): + ''' + Create a device file. + + :param path: full path to device file. + :param type: 'c' or 'b' (character or block). + :param major: major number. + :param minor: minor number. + + XXX: This doesn't actually create a device node, + it simply creates a regular empty file whilst ensuring the filename + gives the impression that it is a device file. + + This hackery is done for the following reasons: + + - non-priv users cannot create device nodes (and the tests run as a + non-priv user). + - the removed file in the upgrade tar file does not actually specify the + _type_ of the files to remove. Hence, we can pretend that the file + to remove is a device file since the upgrader cannot know for sure + (it can check the existing on-disk file, but that isn't conclusive + since the admin may have manually modified a file, or the server + may have generated an invalid remove file - the upgrader cannot + know for sure. + ''' + assert (os.path.dirname(path).endswith('/dev')) + + append_file(path, 'fake-device file') + + +def create_directory(path): + ''' + Create a directory. + ''' + os.makedirs(path, mode=TEST_DIR_MODE, exist_ok=False) + + +def create_sym_link(source, dest): + ''' + Create a symbolic link. + + :param source: existing file to link to. + :param dest: name for the sym link. + ''' + dirname = os.path.dirname(dest) + os.makedirs(dirname, mode=TEST_DIR_MODE, exist_ok=True) + + os.symlink(source, dest) + + +def create_hard_link(source, dest): + ''' + Create a hard link. + + :param source: existing file to link to. + :param dest: name for the hard link. + ''' + os.link(source, dest) + + +def is_sym_link_broken(path): + ''' + :param path: symbolic link to check. + :return: True if the specified path is a broken symbolic link, + else False. + ''' + try: + os.lstat(path) + os.stat(path) + except: + return True + return False + + +def make_command_file(path, update_list): + ''' + Create a command file that the upgrader processes. + + :param path: full path to file to create, + :param update_list: list of update archives to include. + ''' + l = [] + + for file in update_list: + l.append('update {} {}.asc'.format(file, file)) + + # flatten + contents = "\n".join(l) + '\n' + + append_file(path, contents) + + +def file_to_list(path): + ''' + Convert the specified file into a list and return it. + ''' + lines = [] + + with open(path, 'r') as f: + lines = f.readlines() + + lines = [line.rstrip() for line in lines] + + return lines + + +class UpgraderFileRemovalTestCase(UbuntuCoreUpgraderTestCase): + ''' + Test how the upgrader handles the removals file. + ''' + + def test_remove_file(self): + ''' + Ensure the upgrader can remove a regular file. + ''' + + file = 'a-regular-file' + + self.update.add_to_removed_file([file]) + + archive = self.update.create_archive(self.TARFILE) + self.assertTrue(os.path.exists(archive)) + + cmd_file = os.path.join(self.update.tmp_dir, CMD_FILE) + make_command_file(cmd_file, [self.TARFILE]) + + file_path = os.path.join(self.victim_dir, file) + create_file(file_path, 'foo bar') + + self.assertTrue(os.path.exists(file_path)) + self.assertTrue(os.path.isfile(file_path)) + + call_upgrader(cmd_file, self.victim_dir, self.update) + + self.assertFalse(os.path.exists(file_path)) + + def test_remove_directory(self): + ''' + Ensure the upgrader can remove a directory. + ''' + dir = 'a-directory' + + self.update.add_to_removed_file([dir]) + + archive = self.update.create_archive(self.TARFILE) + self.assertTrue(os.path.exists(archive)) + + cmd_file = os.path.join(self.update.tmp_dir, CMD_FILE) + make_command_file(cmd_file, [self.TARFILE]) + + dir_path = os.path.join(self.victim_dir, dir) + create_directory(dir_path) + + self.assertTrue(os.path.exists(dir_path)) + self.assertTrue(os.path.isdir(dir_path)) + + call_upgrader(cmd_file, self.victim_dir, self.update) + + self.assertFalse(os.path.exists(dir_path)) + + def test_remove_sym_link_file(self): + ''' + Ensure the upgrader can remove a symbolic link to a file. + ''' + src = 'the-source-file' + link = 'the-symlink-file' + + self.update.add_to_removed_file([link]) + + archive = self.update.create_archive(self.TARFILE) + self.assertTrue(os.path.exists(archive)) + + cmd_file = os.path.join(self.update.tmp_dir, CMD_FILE) + make_command_file(cmd_file, [self.TARFILE]) + + src_file_path = os.path.join(self.victim_dir, src) + link_file_path = os.path.join(self.victim_dir, link) + + create_file(src_file_path, 'foo bar') + + self.assertTrue(os.path.exists(src_file_path)) + self.assertTrue(os.path.isfile(src_file_path)) + self.assertFalse(os.path.islink(src_file_path)) + + create_sym_link(src_file_path, link_file_path) + self.assertTrue(os.path.exists(link_file_path)) + self.assertTrue(os.path.islink(link_file_path)) + + call_upgrader(cmd_file, self.victim_dir, self.update) + + # original file should still be there + self.assertTrue(os.path.exists(src_file_path)) + self.assertTrue(os.path.isfile(src_file_path)) + self.assertFalse(os.path.islink(src_file_path)) + + # link should have gone + self.assertFalse(os.path.exists(link_file_path)) + + def test_remove_sym_link_directory(self): + ''' + Ensure the upgrader can remove a symbolic link to a directory. + ''' + dir = 'the-source-directory' + link = 'the-symlink-file' + + self.update.add_to_removed_file([link]) + + archive = self.update.create_archive(self.TARFILE) + self.assertTrue(os.path.exists(archive)) + + cmd_file = os.path.join(self.update.tmp_dir, CMD_FILE) + make_command_file(cmd_file, [self.TARFILE]) + + src_dir_path = os.path.join(self.victim_dir, dir) + link_file_path = os.path.join(self.victim_dir, link) + + create_directory(src_dir_path) + + self.assertTrue(os.path.exists(src_dir_path)) + self.assertTrue(os.path.isdir(src_dir_path)) + self.assertFalse(os.path.islink(src_dir_path)) + + create_sym_link(src_dir_path, link_file_path) + self.assertTrue(os.path.exists(link_file_path)) + self.assertTrue(os.path.islink(link_file_path)) + + call_upgrader(cmd_file, self.victim_dir, self.update) + + # original directory should still be there + self.assertTrue(os.path.exists(src_dir_path)) + self.assertTrue(os.path.isdir(src_dir_path)) + self.assertFalse(os.path.islink(src_dir_path)) + + # link should have gone + self.assertFalse(os.path.exists(link_file_path)) + + def test_remove_hardlink(self): + ''' + Ensure the upgrader can remove a hard link to a file. + ''' + src = 'the-source-file' + link = 'the-hardlink-file' + + self.update.add_to_removed_file([link]) + + archive = self.update.create_archive(self.TARFILE) + self.assertTrue(os.path.exists(archive)) + + cmd_file = os.path.join(self.update.tmp_dir, CMD_FILE) + make_command_file(cmd_file, [self.TARFILE]) + + src_file_path = os.path.join(self.victim_dir, src) + link_file_path = os.path.join(self.victim_dir, link) + + create_file(src_file_path, 'foo bar') + + src_inode = os.stat(src_file_path).st_ino + + self.assertTrue(os.path.exists(src_file_path)) + self.assertTrue(os.path.isfile(src_file_path)) + + create_hard_link(src_file_path, link_file_path) + self.assertTrue(os.path.exists(link_file_path)) + + link_inode = os.stat(link_file_path).st_ino + + self.assertTrue(src_inode == link_inode) + + call_upgrader(cmd_file, self.victim_dir, self.update) + + # original file should still be there + self.assertTrue(os.path.exists(src_file_path)) + self.assertTrue(os.path.isfile(src_file_path)) + + # Inode should not have changed. + self.assertTrue(os.stat(src_file_path).st_ino == src_inode) + + # link should have gone + self.assertFalse(os.path.exists(link_file_path)) + + def test_remove_device_file(self): + ''' + Ensure the upgrader can deal with a device file. + + XXX: Note that The upgrader currently "deals" with them by + XXX: ignoring them :-) + + ''' + file = '/dev/a-fake-device' + + self.update.add_to_removed_file([file]) + + archive = self.update.create_archive(self.TARFILE) + self.assertTrue(os.path.exists(archive)) + + cmd_file = os.path.join(self.update.tmp_dir, CMD_FILE) + make_command_file(cmd_file, [self.TARFILE]) + + file_path = '{}{}'.format(self.victim_dir, file) + + create_device_file(file_path) + + self.assertTrue(os.path.exists(file_path)) + + # sigh - we can't assert the that filetype is a char/block + # device because it won't be :) + self.assertTrue(os.path.isfile(file_path)) + + call_upgrader(cmd_file, self.victim_dir, self.update) + + self.assertFalse(os.path.exists(file_path)) + + +class UpgraderFileAddTestCase(UbuntuCoreUpgraderTestCase): + ''' + Test how the upgrader handles adding new files. + ''' + + def test_create_file(self): + ''' + Ensure the upgrader can create a regular file. + ''' + file = 'created-regular-file' + + file_path = os.path.join(self.update.system_dir, file) + + create_file(file_path, 'foo bar') + + archive = self.update.create_archive(self.TARFILE) + self.assertTrue(os.path.exists(archive)) + + cmd_file = os.path.join(self.update.tmp_dir, CMD_FILE) + make_command_file(cmd_file, [self.TARFILE]) + + file_path = os.path.join(self.victim_dir, file) + self.assertFalse(os.path.exists(file_path)) + + call_upgrader(cmd_file, self.victim_dir, self.update) + + self.assertTrue(os.path.exists(file_path)) + self.assertTrue(os.path.isfile(file_path)) + + def test_create_directory(self): + ''' + Ensure the upgrader can create a directory. + ''' + dir = 'created-directory' + + dir_path = os.path.join(self.update.system_dir, dir) + + create_directory(dir_path) + + archive = self.update.create_archive(self.TARFILE) + self.assertTrue(os.path.exists(archive)) + + cmd_file = os.path.join(self.update.tmp_dir, CMD_FILE) + make_command_file(cmd_file, [self.TARFILE]) + + dir_path = os.path.join(self.victim_dir, dir) + self.assertFalse(os.path.exists(dir_path)) + + call_upgrader(cmd_file, self.victim_dir, self.update) + + self.assertTrue(os.path.exists(dir_path)) + self.assertTrue(os.path.isdir(dir_path)) + + def test_create_absolute_sym_link_to_file(self): + ''' + Ensure the upgrader can create a symbolic link to a file (which + already exists and is not included in the update archive). + ''' + src = 'the-source-file' + link = 'the-symlink-file' + + # the file the link points to should *NOT* be below the 'system/' + # directory (since there isn't one post-unpack). + + # an absolute sym-link target path + src_file_path = '/{}'.format(src) + link_file_path = os.path.join(self.update.system_dir, link) + + victim_src_file_path = os.path.normpath('{}/{}' + .format(self.victim_dir, + src_file_path)) + victim_link_file_path = os.path.join(self.victim_dir, link) + + # Create a broken sym link ('/system/<link> -> /<src>') + create_sym_link(src_file_path, link_file_path) + + self.assertTrue(os.path.lexists(link_file_path)) + self.assertTrue(os.path.islink(link_file_path)) + self.assertTrue(is_sym_link_broken(link_file_path)) + + archive = self.update.create_archive(self.TARFILE) + self.assertTrue(os.path.exists(archive)) + + cmd_file = os.path.join(self.update.tmp_dir, CMD_FILE) + make_command_file(cmd_file, [self.TARFILE]) + + create_file(victim_src_file_path, 'foo') + + self.assertTrue(os.path.exists(victim_src_file_path)) + self.assertTrue(os.path.isfile(victim_src_file_path)) + + self.assertFalse(os.path.exists(victim_link_file_path)) + + call_upgrader(cmd_file, self.victim_dir, self.update) + + self.assertTrue(os.path.exists(victim_src_file_path)) + self.assertTrue(os.path.isfile(victim_src_file_path)) + + # upgrader should have created the link in the victim directory + self.assertTrue(os.path.lexists(victim_link_file_path)) + self.assertTrue(os.path.islink(victim_link_file_path)) + self.assertFalse(is_sym_link_broken(victim_link_file_path)) + + def test_create_relative_sym_link_to_file(self): + ''' + Ensure the upgrader can create a symbolic link to a file (which + already exists and is not included in the update archive). + ''' + src = 'a/b/c/the-source-file' + link = 'a/d/e/the-symlink-file' + + # a relative sym-link target path + # ##src_file_path = '../../b/c/{}'.format(src) + src_file_path = '../../b/c/the-source-file'.format(src) + + # the file the link points to should *NOT* be below the 'system/' + # directory (since there isn't one post-unpack). + + link_file_path = os.path.join(self.update.system_dir, link) + + victim_src_file_path = os.path.normpath('{}/{}' + .format(self.victim_dir, + src)) + victim_link_file_path = os.path.join(self.victim_dir, link) + + create_sym_link(src_file_path, link_file_path) + + self.assertTrue(os.path.lexists(link_file_path)) + self.assertTrue(os.path.islink(link_file_path)) + self.assertTrue(is_sym_link_broken(link_file_path)) + + archive = self.update.create_archive(self.TARFILE) + self.assertTrue(os.path.exists(archive)) + + cmd_file = os.path.join(self.update.tmp_dir, CMD_FILE) + make_command_file(cmd_file, [self.TARFILE]) + + create_file(victim_src_file_path, 'foo') + + self.assertTrue(os.path.exists(victim_src_file_path)) + self.assertTrue(os.path.isfile(victim_src_file_path)) + + self.assertFalse(os.path.exists(victim_link_file_path)) + + call_upgrader(cmd_file, self.victim_dir, self.update) + + self.assertTrue(os.path.exists(victim_src_file_path)) + self.assertTrue(os.path.isfile(victim_src_file_path)) + + # upgrader should have created the link in the victim directory + self.assertTrue(os.path.lexists(victim_link_file_path)) + self.assertTrue(os.path.islink(victim_link_file_path)) + self.assertFalse(is_sym_link_broken(victim_link_file_path)) + + def test_create_broken_sym_link_file(self): + ''' + Ensure the upgrader can create a broken symbolic link + (one that points to a non-existent file). + ''' + src = 'the-source-file' + link = 'the-symlink-file' + + # the file the link points to should *NOT* be below the 'system/' + # directory (since there isn't one post-unpack). + src_file_path = src + + link_file_path = os.path.join(self.update.system_dir, link) + + # Create a broken sym link ('/system/<link> -> /<src>') + create_sym_link(src_file_path, link_file_path) + + self.assertTrue(os.path.lexists(link_file_path)) + self.assertTrue(os.path.islink(link_file_path)) + self.assertTrue(is_sym_link_broken(link_file_path)) + + archive = self.update.create_archive(self.TARFILE) + self.assertTrue(os.path.exists(archive)) + + cmd_file = os.path.join(self.update.tmp_dir, CMD_FILE) + make_command_file(cmd_file, [self.TARFILE]) + + victim_src_file_path = os.path.join(self.victim_dir, src) + victim_link_file_path = os.path.join(self.victim_dir, link) + + self.assertFalse(os.path.exists(victim_src_file_path)) + self.assertFalse(os.path.exists(victim_link_file_path)) + + call_upgrader(cmd_file, self.victim_dir, self.update) + + # source still shouldn't exist + self.assertFalse(os.path.exists(victim_src_file_path)) + + # upgrader should have created the link in the victim directory + self.assertTrue(os.path.lexists(victim_link_file_path)) + self.assertTrue(os.path.islink(victim_link_file_path)) + self.assertTrue(is_sym_link_broken(victim_link_file_path)) + + +def main(): + kwargs = {} + format = \ + '%(asctime)s:' \ + '%(filename)s:' \ + '%(name)s:' \ + '%(funcName)s:' \ + '%(levelname)s:' \ + '%(message)s' + + kwargs['format'] = format + + # We want to see what's happening + kwargs['level'] = logging.DEBUG + + logging.basicConfig(**kwargs) + + unittest.main( + testRunner=unittest.TextTestRunner( + stream=sys.stdout, + verbosity=2, + + # don't keep running tests if one fails + # (... who _wouldn't_ want this???) + failfast=True + ), + + ) + +if __name__ == '__main__': + main() diff --git a/ubuntucoreupgrader/tests/test_upgrader.py b/ubuntucoreupgrader/tests/test_upgrader.py index 3c05dc8..9189a54 100644 --- a/ubuntucoreupgrader/tests/test_upgrader.py +++ b/ubuntucoreupgrader/tests/test_upgrader.py @@ -29,8 +29,7 @@ import tempfile import unittest import os import shutil - -from unittest.mock import patch +import sys from ubuntucoreupgrader.upgrader import ( tar_generator, @@ -38,7 +37,14 @@ from ubuntucoreupgrader.upgrader import ( parse_args, ) -script_name = os.path.basename(__file__) +base_dir = os.path.abspath(os.path.dirname(__file__)) +sys.path.append(base_dir) + +from ubuntucoreupgrader.tests.utils import ( + create_file, + make_tmp_dir, + UbuntuCoreUpgraderTestCase, + ) # file mode to use for creating test directories. TEST_DIR_MODE = 0o750 @@ -50,19 +56,6 @@ def make_default_options(): return parse_args([]) -def make_tmp_dir(tag=None): - ''' - Create a temporary directory. - ''' - - if tag: - prefix = '{}-{}-'.format(script_name, tag) - else: - prefix = script_name - - return tempfile.mkdtemp(prefix=prefix) - - class UpgradeTestCase(unittest.TestCase): def test_tar_generator_unpack_assets(self): @@ -172,28 +165,6 @@ class UpgradeTestCase(unittest.TestCase): shutil.rmtree(cache_dir) -def append_file(path, contents): - ''' - Append to a regular file (create it doesn't exist). - ''' - - dirname = os.path.dirname(path) - os.makedirs(dirname, mode=TEST_DIR_MODE, exist_ok=True) - - with open(path, 'a') as fh: - fh.writelines(contents) - - if not contents.endswith('\n'): - fh.write('\n') - - -def create_file(path, contents): - ''' - Create a regular file. - ''' - append_file(path, contents) - - def touch_file(path): ''' Create an empty file (creating any necessary intermediate @@ -223,152 +194,6 @@ def make_commands(update_list): return l -class UpdateTree(): - ''' - Representation of a directory tree that will be converted into an - update archive. - ''' - TEST_REMOVED_FILE = 'removed' - TEST_SYSTEM_DIR = 'system/' - - def __init__(self): - - # Directory tree used to construct the tar file from. - # Also used to hold the TEST_REMOVED_FILE file. - self.dir = make_tmp_dir(tag='UpdateTree-tar-source') - - self.removed_file = os.path.join(self.dir, self.TEST_REMOVED_FILE) - - # Directory to place create/modify files into. - self.system_dir = os.path.join(self.dir, self.TEST_SYSTEM_DIR) - - # Directory used to write the generated tarfile to. - # This directory should also be used to write the command file - # to. - self.tmp_dir = make_tmp_dir(tag='UpdateTree-cache') - - def destroy(self): - if os.path.exists(self.dir): - shutil.rmtree(self.dir) - - if os.path.exists(self.tmp_dir): - shutil.rmtree(self.tmp_dir) - - def tar_filter(self, member): - ''' - Function to filter the tarinfo members before creating the - archive. - ''' - # members are created with relative paths (no leading slash) - path = os.sep + member.name - - if member.name == '/.': - return None - - i = path.find(self.dir) - assert(i == 0) - - # remove the temporary directory elements - # (+1 for the os.sep we added above) - member.name = path[len(self.dir)+1:] - - return member - - def create_archive(self, name): - ''' - Create an archive with the specified name from the UpdateTree - object. Also creates a fake signature file alongside the archive - file since this is currently required by the upgrader (although - it is not validated). - - :param name: name of tarfile. - :param name: full path to xz archive to create. - :return full path to tar file with name @name. - ''' - - tar_path = os.path.join(self.tmp_dir, name) - tar = tarfile.open(tar_path, 'w:xz') - - # We can't just add recursively since that would attempt to add - # the parent directory. However, the real update tars don't - # include that, and attempting to ignore the parent directory - # results in an empty archive. So, walk the tree and add - # file-by-file. - for path, names, files in os.walk(self.dir): - for file in files: - full = os.path.join(path, file) - tar.add(full, recursive=False, filter=self.tar_filter) - if not files and not names: - # add (empty) directories - tar.add(path, recursive=False, filter=self.tar_filter) - - tar.close() - - signature = '{}.asc'.format(tar_path) - - with open(signature, 'w') as fh: - fh.write('fake signature file') - - return tar_path - - -class UbuntuCoreUpgraderTestCase(unittest.TestCase): - ''' - Base class for Upgrader tests. - ''' - - TARFILE = 'update.tar.xz' - - # result of last test run. Hack to deal with fact that even if a - # test fails, unittest still calls .tearDown() (whomever thought - # that was a good idea...?) - currentResult = None - - def setUp(self): - ''' - Test setup. - ''' - # Create an object to hold the tree that will be converted into - # an upgrade archive. - self.update = UpdateTree() - - # The directory which will have the update archive applied to - # it. - self.victim_dir = make_tmp_dir(tag='victim') - - def tearDown(self): - ''' - Test cleanup. - ''' - - if not self.currentResult.wasSuccessful(): - # Do not clean up - the only sane option if a test fails. - return - - self.update.destroy() - self.update = None - - shutil.rmtree(self.victim_dir) - self.victim_dir = None - - def run(self, result=None): - self.currentResult = result - unittest.TestCase.run(self, result) - - -def mock_get_root_partitions_by_label(): - matches = [] - - matches.append(('system-a', '/dev/sda3', '/')) - matches.append(('system-b', '/dev/sda4', '/writable/cache/system')) - - return matches - - -def mock_make_mount_private(target): - pass - - class UbuntuCoreUpgraderObectTestCase(UbuntuCoreUpgraderTestCase): def mock_get_cache_dir(self): @@ -380,10 +205,6 @@ class UbuntuCoreUpgraderObectTestCase(UbuntuCoreUpgraderTestCase): os.makedirs(self.sys_dir, exist_ok=True) return self.cache_dir - @patch('ubuntucoreupgrader.upgrader.get_root_partitions_by_label', - mock_get_root_partitions_by_label) - @patch('ubuntucoreupgrader.upgrader.make_mount_private', - mock_make_mount_private) def test_create_object(self): root_dir = make_tmp_dir() @@ -413,6 +234,7 @@ class UbuntuCoreUpgraderObectTestCase(UbuntuCoreUpgraderTestCase): upgrader = Upgrader(options, commands, []) upgrader.get_cache_dir = self.mock_get_cache_dir + upgrader.MOUNTPOINT_CMD = "true" upgrader.run() path = os.path.join(root_dir, file) @@ -420,5 +242,6 @@ class UbuntuCoreUpgraderObectTestCase(UbuntuCoreUpgraderTestCase): shutil.rmtree(root_dir) + if __name__ == "__main__": unittest.main() diff --git a/ubuntucoreupgrader/tests/utils.py b/ubuntucoreupgrader/tests/utils.py new file mode 100644 index 0000000..dc4f463 --- /dev/null +++ b/ubuntucoreupgrader/tests/utils.py @@ -0,0 +1,266 @@ +# -*- coding: utf-8 -*- +# -------------------------------------------------------------------- +# Copyright © 2014-2015 Canonical Ltd. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# -------------------------------------------------------------------- + +import os +import tempfile +import tarfile +import shutil +import unittest + +# file mode to use for creating test directories. +TEST_DIR_MODE = 0o750 + +script_name = os.path.basename(__file__) + + +def make_tmp_dir(tag=None): + ''' + Create a temporary directory. + ''' + + if tag: + prefix = '{}-{}-'.format(script_name, tag) + else: + prefix = script_name + + return tempfile.mkdtemp(prefix=prefix) + + +def append_file(path, contents): + ''' + Append to a regular file (create it doesn't exist). + ''' + + dirname = os.path.dirname(path) + os.makedirs(dirname, mode=TEST_DIR_MODE, exist_ok=True) + + with open(path, 'a') as fh: + fh.writelines(contents) + + if not contents.endswith('\n'): + fh.write('\n') + + +def create_file(path, contents): + ''' + Create a regular file. + ''' + append_file(path, contents) + + +def mock_get_root_partitions_by_label(): + ''' + Fake disk partition details for testing. + ''' + matches = [] + + matches.append(('system-a', '/dev/sda3', '/')) + matches.append(('system-b', '/dev/sda4', '/writable/cache/system')) + + return matches + + +class UpdateTree(): + ''' + Representation of a directory tree that will be converted into an + update archive. + ''' + TEST_REMOVED_FILE = 'removed' + TEST_SYSTEM_DIR = 'system/' + + def __init__(self): + + # Directory tree used to construct the tar file from. + # Also used to hold the TEST_REMOVED_FILE file. + self.dir = make_tmp_dir(tag='UpdateTree-tar-source') + + self.removed_file = os.path.join(self.dir, self.TEST_REMOVED_FILE) + + # Directory to place create/modify files into. + self.system_dir = os.path.join(self.dir, self.TEST_SYSTEM_DIR) + + # Directory used to write the generated tarfile to. + # This directory should also be used to write the command file + # to. + self.tmp_dir = make_tmp_dir(tag='UpdateTree-cache') + + def destroy(self): + if os.path.exists(self.dir): + shutil.rmtree(self.dir) + + if os.path.exists(self.tmp_dir): + shutil.rmtree(self.tmp_dir) + + def add_to_removed_file(self, removed_files): + ''' + Add the specified list of files to the removed file. + + The 'removed' file is simply a file with a well-known name that + contains a list of files (one per line) to be removed from a + system before the rest of the update archive is unpacked. + + :param removed_files: list of file names to add to the removed file. + + ''' + # all files listed in the removed list must be system files + final = list(map(lambda a: + '{}{}'.format(self.TEST_SYSTEM_DIR, a), removed_files)) + + contents = "".join(final) + append_file(self.removed_file, contents) + + def tar_filter(self, member): + ''' + Function to filter the tarinfo members before creating the + archive. + ''' + # members are created with relative paths (no leading slash) + path = os.sep + member.name + + if member.name == '/.': + return None + + i = path.find(self.dir) + assert(i == 0) + + # remove the temporary directory elements + # (+1 for the os.sep we added above) + member.name = path[len(self.dir)+1:] + + return member + + def create_archive(self, name): + ''' + Create an archive with the specified name from the UpdateTree + object. Also creates a fake signature file alongside the archive + file since this is currently required by the upgrader (although + it is not validated). + + :param name: name of tarfile. + :param name: full path to xz archive to create. + :return full path to tar file with name @name. + ''' + + self.tar_path = os.path.join(self.tmp_dir, name) + tar = tarfile.open(self.tar_path, 'w:xz') + + # We can't just add recursively since that would attempt to add + # the parent directory. However, the real update tars don't + # include that, and attempting to ignore the parent directory + # results in an empty archive. So, walk the tree and add + # file-by-file. + for path, names, files in os.walk(self.dir): + for file in files: + full = os.path.join(path, file) + tar.add(full, recursive=False, filter=self.tar_filter) + if not files and not names: + # add (empty) directories + tar.add(path, recursive=False, filter=self.tar_filter) + + tar.close() + + signature = '{}.asc'.format(self.tar_path) + + with open(signature, 'w') as fh: + fh.write('fake signature file') + + return self.tar_path + + +class UbuntuCoreUpgraderTestCase(unittest.TestCase): + ''' + Base class for Upgrader tests. + + Most of the tests follow a standard pattern: + + 1) Create an UpdateTree object: + + update = UpdateTree() + + This creates 2 temporary directories: + + - self.system dir: Used as to generate an update archive from. + + - self.tmp_dir: Used to write the generated archive file to. The + intention is that this directory should also be used to hold + the command file. + + 2) Removal tests call update.add_to_removed_file(file) to add a + particular file to the removals file in the update archive. + + 3) Create/Modify tests create files below update.system_dir. + + 4) Create an update archive (which includes the removals file + and all files below update.system_dir): + + archive = update.create_archive(self.TARFILE) + + 5) Create a command file (which tells the upgrader what to do + and which archive files to apply): + + make_command_file(...) + + 6) Create a victim directory. This is a temporary directory where + the upgrade will happen. + + 7) Start the upgrade: + + call_upgrader(...) + + 8) Perform checks on the victim directory to ensure that upgrade + did what was expected. + + ''' + + TARFILE = 'update.tar.xz' + + # result of last test run. Hack to deal with fact that even if a + # test fails, unittest still calls .tearDown() (whomever thought + # that was a good idea...?) + currentResult = None + + def setUp(self): + ''' + Test setup. + ''' + # Create an object to hold the tree that will be converted into + # an upgrade archive. + self.update = UpdateTree() + + # The directory which will have the update archive applied to + # it. + self.victim_dir = make_tmp_dir(tag='victim') + + def tearDown(self): + ''' + Test cleanup. + ''' + + if not self.currentResult.wasSuccessful(): + # Do not clean up - the only sane option if a test fails. + return + + self.update.destroy() + self.update = None + + shutil.rmtree(self.victim_dir) + self.victim_dir = None + + def run(self, result=None): + self.currentResult = result + unittest.TestCase.run(self, result) diff --git a/ubuntucoreupgrader/upgrader.py b/ubuntucoreupgrader/upgrader.py index ba90b62..502fbc4 100644 --- a/ubuntucoreupgrader/upgrader.py +++ b/ubuntucoreupgrader/upgrader.py @@ -40,34 +40,22 @@ NOTE : The signature files associated with each image file is *NOT* import sys import os import logging -import re import shutil import subprocess import tempfile import tarfile -import stat -import errno -import dbus import argparse -from enum import Enum -from time import sleep - script_name = os.path.basename(__file__) log = logging.getLogger() -# seconds to wait before a reboot (should one be required) -DEFAULT_REBOOT_DELAY = 2 - DEFAULT_ROOT = '/' # Name of the writable user data partition label as created by # ubuntu-device-flash(1). WRITABLE_DATA_LABEL = 'writable' -WRITABLE_MOUNTPOINT = '/writable' - # name of primary root filesystem partition label as created by # ubuntu-device-flash(1). SYSTEM_DATA_A_LABEL = 'system-a' @@ -113,24 +101,6 @@ def parse_args(args): parser = argparse.ArgumentParser(description='System image Upgrader') parser.add_argument( - '--check-reboot', - action='store_true', - help=''' - Attempt to determine if a reboot may be required. - - This option is similar to --dry-run: no system changes are made. - - Note that the result of this command cannot be definitive since a - reboot would be triggered if a service failed to start (but this - option does not attempt to restart any services). - ''') - - parser.add_argument( - '--clean-only', - action='store_true', - help='Clean up from a previous upgrader run, but do not upgrade)') - - parser.add_argument( '--debug', nargs='?', const=1, default=0, type=int, help='Dump debug info (specify numeric value to increase verbosity)') @@ -144,41 +114,16 @@ def parse_args(args): ''') parser.add_argument( - '--force-inplace-upgrade', - action='store_true', - help='Apply an upgrade to current rootfs even if running " \ - "on dual rootfs system') - - parser.add_argument( '--leave-files', action='store_true', help='Do not remove the downloaded system image files after upgrade') parser.add_argument( - '--no-reboot', - action='store_true', - help='Do not reboot even if one would normally be required') - - parser.add_argument( - '--reboot-delay', - type=int, default=DEFAULT_REBOOT_DELAY, - help=''' - Wait for specified number of seconds before rebooting - (default={}) - '''.format(DEFAULT_REBOOT_DELAY)) - - parser.add_argument( '--root-dir', default=DEFAULT_ROOT, help='Specify an alternative root directory (for testing ONLY)') parser.add_argument( - '--show-other-details', - action='store_true', - help='Dump the details of the system-image vesion on the " \ - "other root partition') - - parser.add_argument( '-t', '--tmpdir', help='Specify name for pre-existing temporary directory to use') @@ -210,100 +155,6 @@ def remove_prefix(path, prefix=TAR_FILE_SYSTEM_PREFIX): return path[len(prefix)-1:] -def get_command(pid): - ''' - Returns full path to binary associated with @pid, or None if the - lookup failed. - ''' - try: - exe = os.readlink('/proc/' + str(pid) + '/exe') - except: - # pid probably went away - return None - - return exe - - -def get_root_partitions_by_label(): - ''' - Returns a list of tuples of the recognised root filesystem partitions - available on this system. The tuples contain the following triplet: - - ( <partition-name>, <full-device-path>, <mountpoint> ) - - ''' - cmd = 'lsblk' - recognised = (SYSTEM_DATA_A_LABEL, SYSTEM_DATA_B_LABEL) - - matches = [] - - args = [cmd, '--ascii', '--output-all', '--pairs'] - log.debug('running: {}'.format(args)) - - proc = subprocess.Popen(args, - stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL, - universal_newlines=True) - - if proc.wait() != 0: - return matches - - stdout = proc.communicate()[0] - lines = stdout.split('\n') - - for line in lines: - line = line.rstrip() - if not line: - continue - - fields = {} - - # split the line into 'NAME="quoted value"' fields - results = re.findall(r'(?:[^\s"]|"(?:[^"])*")+', line) - for result in results: - name, value = result.split('=') - - # remove quotes - value = value.lstrip('"').rstrip('"') - - fields[name] = value - - if fields['NAME'] != fields['KNAME']: - # For SCSI devices, these fields match - continue - - if 'LABEL' in fields and fields['LABEL'] in recognised: - # reconstructing the device name like this is valid - # for SCSI devices - device = '/dev/{}'.format(fields['NAME']) - - if not os.path.exists(device): - continue - matches.append((fields['LABEL'], - device, - fields['MOUNTPOINT'])) - - return matches - - -def uses_ab_partitions(): - ''' - Returns: True if the system uses A/B partitions, else False. - ''' - return len(get_root_partitions_by_label()) == 2 - - -def get_writable_disk(): - ''' - Establish the disk partition for the writable user data partition. - ''' - cmd = "blkid -L '{}' 2>/dev/null".format(WRITABLE_DATA_LABEL) - log.debug('running: {}'.format(cmd)) - - output = subprocess.getoutput(cmd) - return output.rstrip() - - def fsck(device): ''' Run fsck(8) on specified device. @@ -414,31 +265,6 @@ def bind_mount(source, target): mount(source, target, 'bind') -def mount_all(): - ''' - Mount all filesystems if not already mounted. - ''' - - args = ['mount', '-a'] - log.debug('running: {}'.format(args)) - proc = subprocess.Popen(args, - stderr=subprocess.PIPE, - universal_newlines=True) - - if proc.wait() != 0: - stderr = proc.communicate()[1] - log.error('failed to mount all ({}): {}' - .format(args, stderr)) - - -def lazy_unmount(target): - ''' - async unmount the specified mount target. - ''' - - unmount(target, ['--lazy']) - - def unmount(target, options=None): ''' Unmount the specified mount target, using the specified list of @@ -473,355 +299,6 @@ def unmount(target, options=None): # sys.exit(1) -def make_mount_private(target): - ''' - Make the specified filesystem private. - ''' - - args = ['mount', '--make-rprivate', target] - log.debug('running: {}'.format(args)) - - proc = subprocess.Popen(args, - stderr=subprocess.PIPE, - universal_newlines=True) - - if proc.wait() != 0: - stderr = proc.communicate()[1] - log.error('failed to make {} private ({}): {}' - .format(target, args, stderr)) - - -def lazy_unmount_specified(mounts): - ''' - async nnmount all targets specified by @mounts. - ''' - - for mount in mounts: - lazy_unmount(mount) - - -def get_writable_mounts(): - ''' - Returns a list of (bind) mounts whose source location is the - writable partition. - - Note that the list of bind mounts is derived from the current - mounts. This is safer than simply checking fstab since it is - guaranteed correct. - - Note also that the list is reverse sorted so that assuming the list - is processed in order, child mounts will be handled before parent - mounts. - ''' - disk = get_writable_disk() - file = '/proc/mounts' - - mounts = [] - - try: - with open(file, 'r') as fh: - for mount in fh.readlines(): - mount = mount.strip() - fields = mount.split() - - # only consider user data mounts - if fields[0] != disk: - continue - - # ignore the primary mount - if fields[1] == WRITABLE_MOUNTPOINT: - continue - - # ignore root - if fields[1] == '/' or fields[1] == '/root': - continue - - mounts.append(fields[1]) - except: - sys.exit('Failed to read command file: {}'.format(file)) - - # Reverse sort to ensure each child mount is handled before its - # parent. - return sorted(mounts, reverse=True) - - -def get_affected_pids(unpack_inodes): - ''' - @unpack_inodes: list of inodes representing the files the system image - upgrade will modify. - - Returns a dict keyed by filename whose value is an array of pids - that are using the file currently. - - ''' - - # Command to list all open files as quickly as possible. - # - # XXX: Note that although we nominally only care about files open on - # the rootfs, we don't constrain lsof to only consider such files - # since the system-images contain files that are mounted in writable - # partitions (examples being /var/log/dpkg.log and - # /var/log/apt/history.log). - # - # By considering all files, we increase the likelihood of a reboot - # but in doing avoid unexpected system behaviour (where a process is - # seeking through an old copy of a config file for example). - # - # Discard stderr to avoid the annoying - # 'lsof: WARNING can't stat() ...' messages. - cmd = 'lsof -lnPR 2>/dev/null' - - log.debug('running: {}'.format(cmd)) - - output = subprocess.getoutput(cmd) - - # Hash showing which processes are using files that the - # upgrader needs to replace. - # - # key: filename. - # value: array of pids. - pid_file_map = {} - - # Read the lsof output. Note that we do _not_ ignore deleted files - # since in fact the files we are looking for have been deleted - # (unlinked, but not fully). - for line in output.split('\n'): - fields = line.split() - - # ignore header - if line.startswith('COMMAND'): - continue - - # ignore files with no inode - if fields[5] in ('netlink', 'unknown'): - continue - - pid = int(fields[1]) - - # Deleted files have one less field (size/offset). - if len(fields) == 9 and fields[4] == 'DEL': - - potential_inode = fields[7] - - if not potential_inode.isdigit(): - continue - - inode = int(potential_inode) - file = fields[8] - else: - potential_inode = fields[8] - - if not potential_inode.isdigit(): - continue - - inode = int(potential_inode) - file = fields[9] - - # ignore kernel threads - if file == '/': - continue - - # ignore anything that doesn't look like a file - if file[0] != '/': - continue - - # ignore files that don't relate to files the upgrade will - # change. - if inode not in unpack_inodes: - continue - - # create a hash of arrays / dict of lists - if file not in pid_file_map: - pid_file_map[file] = [] - - pid_file_map[file].append(pid) - - return pid_file_map - - -class Systemd(): - ''' - Interface to systemd init daemon. - - The upgrader talks to systemd via its private socket since this - protects the upgrader... - - - from issues with a broken dbus-daemon (admittedly unlikely). - - - from getting disconnected from systemd should the dbus-daemon need - to be restarted by the upgrader if the dbus-daemon is holding inodes - open (much more likely to happen). - - - against changes in the format of systemd's command-line tooling - output (as happened when 'systemctl show' output changed between - systemd 208 and 215). - - ''' - - SYSTEMD_BUS_NAME = 'org.freedesktop.systemd1' - SYSTEMD_MGR_INTERFACE = 'org.freedesktop.systemd1.Manager' - SYSTEMD_UNIT_INTERFACE = 'org.freedesktop.systemd1.Unit' - SYSTEMD_OBJECT_PATH = '/org/freedesktop/systemd1' - - SYSTEMD_PRIVATE_SOCKET = 'unix:path=/run/systemd/private' - - FREEDESKTOP_PROPERTIES = 'org.freedesktop.DBus.Properties' - - def __init__(self): - self.connection = \ - dbus.connection.Connection(self.SYSTEMD_PRIVATE_SOCKET) - self.proxy = self.connection.get_object(self.SYSTEMD_BUS_NAME, - self.SYSTEMD_OBJECT_PATH) - self.properties = dbus.Interface(self.proxy, - self.FREEDESKTOP_PROPERTIES) - - self.interface = dbus.Interface(self.proxy, - self.SYSTEMD_MGR_INTERFACE) - - # method for handling service start/stop/restart - self.mode = 'replace' - - def version(self): - ''' - Returns the currently running version of systemd. - ''' - return self.properties.Get(self.SYSTEMD_MGR_INTERFACE, 'Version') - - def find_unit(self, pid): - ''' - Find the systemd unit associated with @pid. - - Returns D-Bus object path for service associated with @pid, - or None. - ''' - try: - return self.interface.GetUnitByPID(pid) - except: - return None - - def get_service(self, name): - ''' - @name: D-Bus path for service. - - Return the unit associated with the specified @name. - ''' - unit = self.connection.get_object(self.SYSTEMD_BUS_NAME, - name) - interface = dbus.Interface(unit, self.SYSTEMD_UNIT_INTERFACE) - return interface - - def stop_service(self, name): - ''' - Stop the specified service. - - @name: D-Bus path for service. - - Returns: True on success, else False - ''' - - log.debug('stopping service {}'.format(name)) - - interface = self.get_service(name) - - try: - interface.Stop(self.mode) - return True - except: - return False - - def start_service(self, name): - ''' - Start the specified service. - - @name: D-Bus path for service. - - Returns: True on success, else False - ''' - - log.debug('starting service {}'.format(name)) - - interface = self.get_service(name) - - try: - interface.Start(self.mode) - return True - except: - return False - - def restart_service(self, name): - ''' - Restart the specified service. - - @name: D-Bus path for service - (for example, "/org/freedesktop/systemd1/unit/cron_2eservice"). - - Returns: True on success, else False - ''' - try: - interface = self.get_service(name) - interface.Restart(self.mode) - return True - except: - return False - - def stop_service_by_pid(self, pid): - ''' - Stop service specified by @pid. - ''' - unit = self.find_unit(pid) - self.stop_service(unit) - - # FIXME: should return new pid - def restart_service_by_pid(self, pid): - ''' - Restart service specified by @pid. - ''' - unit = self.find_unit(pid) - self.restart_service(unit) - - # FIXME: - # - # We should use D-Bus here, rather than calling the binary. - # - # However, even if we use D-Bus, currently the D-Bus logic will fail - # since after a restart, systemd will sever all D-Bus connections - # and even after closing the connection, deleting the - # appropriate objects, and recreating those objects, method calls - # (such as self.version()) hang for approximately 30 seconds and - # then fail with: - # - # dbus.exceptions.DBusException: org.freedesktop.DBus.Error.NoReply - # - def restart_manager(self): - ''' - Cause systemd to re-exec itself. - - Returns: True on success, else False. - ''' - - # FIXME: - # - # Don't attempt to restart for now to avoid disrupting the - # existing D-Bus connection. - log.error('FIXME: not restarting systemd') - return False - - log.debug('restarting systemd service manager') - - args = ['systemctl', 'daemon-reexec'] - log.debug('running: {}'.format(args)) - proc = subprocess.Popen(args, - stderr=subprocess.PIPE, - universal_newlines=True) - if proc.wait() != 0: - stderr = proc.communicate()[1] - log.error('failed to restart systemd ({}): {}' - .format(args, stderr)) - return False - - return True - - def tar_generator(tar, cache_dir, removed_files, root_dir): ''' Generator function to handle extracting members from the system @@ -942,29 +419,7 @@ class Upgrader(): DIR_MODE = 0o750 - def update_timestamp(self): - ''' - Update the timestamp file to record the time the last upgrade - completed successfully. - ''' - file = os.path.join(self.get_cache_dir(), self.TIMESTAMP_FILE) - open(file, 'w').close() - - def get_cache_dir(self): - ''' - Returns the full path to the cache directory, which is used as a - scratch pad, for downloading new images to and bind mounting the - rootfs. - ''' - return self.options.tmpdir \ - if self.options.tmpdir \ - else self.DEFAULT_CACHE_DIR - - def get_mount_target(self): - ''' - Get the full path to the mount target directory. - ''' - return os.path.join(self.get_cache_dir(), self.MOUNT_TARGET) + MOUNTPOINT_CMD = "mountpoint" def __init__(self, options, commands, remove_list): """ @@ -983,25 +438,6 @@ class Upgrader(): 'update': self._cmd_update, } - # Records why a reboot is required (or REBOOT_NONE - # if no reboot necessary) - self.reboot_reasons = Enum('REBOOT_REASON', - 'REBOOT_NONE ' + - 'REBOOT_BOOTME ' + - 'REBOOT_SERVICE ' + - 'REBOOT_OPEN_FILE ' + - 'REBOOT_NEW_IMAGE ') - - # List of recognised methods the upgrader supports - # to upgrade a system. - # - # If dual root filesystem partitions are found, the upgrader - # will upgrade to the "other" partition. Otherwise, an in-place - # upgrade will be applied. - self.upgrade_types = Enum('UPGRADE_TYPE', - 'UPGRADE_IN_PLACE ' + - 'UPGRADE_AB_PARTITIONS ') - self.options = options assert('root_dir' in self.options) @@ -1014,18 +450,8 @@ class Upgrader(): self.remove_list = remove_list self.full_image = False - # path => inode map set by save_links() - self.file_map = {} - self.lost_found = '/lost+found' - # Set on any of the following: - # - # - Failure to restart all services. - # - Failure to determine service associated with a pid. - # - System-image requested it ('bootme' flag). - self.reboot_reason = self.reboot_reasons.REBOOT_NONE - self.removed_file = TAR_FILE_REMOVED_FILE # Identify the directory the files the command file refers to @@ -1046,29 +472,29 @@ class Upgrader(): # Note: Only used by UPGRADE_IN_PLACE. self.other_is_empty = False - def set_reboot_reason(self, reason): + def update_timestamp(self): ''' - Set the reboot reason, if not already set. This ensures the - primary reason is retained. + Update the timestamp file to record the time the last upgrade + completed successfully. ''' - if self.reboot_reason != self.reboot_reasons.REBOOT_NONE: - # ignore - return - - self.reboot_reason == reason - - def determine_upgrade_type(self): + file = os.path.join(self.get_cache_dir(), self.TIMESTAMP_FILE) + open(file, 'w').close() - self.current_rootfs_device = self.get_rootfs() + def get_cache_dir(self): + ''' + Returns the full path to the cache directory, which is used as a + scratch pad, for downloading new images to and bind mounting the + rootfs. + ''' + return self.options.tmpdir \ + if self.options.tmpdir \ + else self.DEFAULT_CACHE_DIR - # Determine what sort of upgrade will be performed - if uses_ab_partitions(): - self.upgrade_type = self.upgrade_types.UPGRADE_AB_PARTITIONS - self.other_rootfs_device = self.get_other_rootfs() - self.rootfs_to_modify = self.other_rootfs_device - else: - self.upgrade_type = self.upgrade_types.UPGRADE_IN_PLACE - self.rootfs_to_modify = self.current_rootfs_device + def get_mount_target(self): + ''' + Get the full path to the mount target directory. + ''' + return os.path.join(self.get_cache_dir(), self.MOUNT_TARGET) def prepare(self): ''' @@ -1076,65 +502,23 @@ class Upgrader(): ''' target = self.get_mount_target() - - # Note that we need the stat of the mountpoint, - # *NOT* the device itself. - self.mountpoint_stat = os.stat(target) - - self.determine_upgrade_type() - - log.debug('system upgrade type is {}'.format(self.upgrade_type)) - if self.options.debug > 1: - log.debug('current rootfs device is {}' - .format(self.current_rootfs_device)) - log.debug('rootfs to update is {}' - .format(self.rootfs_to_modify)) - - if self.options.force_inplace_upgrade: - self.upgrade_type = self.upgrade_types.UPGRADE_IN_PLACE - self.rootfs_to_modify = self.current_rootfs_device - log.debug('forcing upgrade type to be {}' - .format(self.upgrade_type)) + if subprocess.call([self.MOUNTPOINT_CMD, "-q", target]) != 0: + raise Exception( + "The {} directory is not a mountpoint".format(target)) if self.options.dry_run: return - if self.options.clean_only: - self.cleanup_inodes() - return - if self.options.root_dir != '/': # Don't modify root when running in test mode return - if self.upgrade_type != self.upgrade_types.UPGRADE_IN_PLACE: - return - - # Necessary since systemd makes the rootfs shared, which allows - # mount operations visible across all mount namespaces. - log.debug('making {} private'.format(self.options.root_dir)) - - make_mount_private(self.options.root_dir) - - # Unmount all the bind mounts to avoid any possibility - # of writing to the writable partition. - mounts = get_writable_mounts() - - log.debug('unmounting writable partitions') - - lazy_unmount_specified(mounts) - - self.cleanup_inodes() - def run(self): ''' Execute the commands in the command file ''' self.prepare() - if self.options.clean_only: - return - for cmdline in self.commands: cmdline = cmdline.strip() @@ -1155,10 +539,6 @@ class Upgrader(): Final tidy-up. ''' - if self.options.dry_run: - self.show_outcome() - return - if self.options.leave_files: log.debug('not removing files') else: @@ -1170,42 +550,8 @@ class Upgrader(): if self.options.root_dir != '/': return - if self.upgrade_type == self.upgrade_types.UPGRADE_IN_PLACE: - log.debug('remounting all writable partitions') - mount_all() - - try: - prefix = self.get_saved_link_dir_prefix() - os.rmdir(prefix) - except: - # there must be remaining links, so leave them to be cleaned - # up via the initramfs. - pass - - self.show_outcome() - self.update_timestamp() - if self.reboot_reason == self.reboot_reasons.REBOOT_NONE: - return - - # A system with A/B partitions should not automatically reboot; - # let snappy advise the admin that a reboot should be performed. - if self.upgrade_type == \ - self.upgrade_types.UPGRADE_AB_PARTITIONS: - return - - if self.options.no_reboot: - log.warning('Not rebooting at user request') - return - - reboot_delay = self.options.reboot_delay - # give the admin a chance to see the message - log.debug('Waiting for {} seconds before rebooting' - .format(reboot_delay)) - sleep(reboot_delay) - os.system('/sbin/reboot') - def _cmd_format(self, args): try: target = args[0] @@ -1265,9 +611,6 @@ class Upgrader(): self.remount_rootfs(writable=True) - if self.upgrade_type != self.upgrade_types.UPGRADE_AB_PARTITIONS: - return - if self.other_is_empty: # Copy current rootfs data to the other rootfs's blank # partition. @@ -1300,11 +643,12 @@ class Upgrader(): target = self.get_mount_target() if writable: + root = subprocess.check_output( + ["findmnt", "-o", "SOURCE", "-n", target]).strip() + # ro->rw so need to fsck first. unmount(target) - root = self.get_other_rootfs() - # needs to be mounted writable, so check it first! fsck(root) @@ -1321,129 +665,6 @@ class Upgrader(): # rw->ro so no fsck required. remount(target, "ro") - def get_rootfs(self): - ''' - Returns the full path to the currently booted root partition. - ''' - roots = get_root_partitions_by_label() - return roots[0][1] if roots[0][2] else roots[1][1] - - def get_other_rootfs(self): - ''' - Returns the full device path to the "other" partition for - systems that have dual root filesystems (A/B partitions) - ''' - roots = get_root_partitions_by_label() - return roots[0][1] if not roots[0][2] else roots[1][1] - - def show_other_partition_details(self): - ''' - Mount the other partition and dump the contents of the - channel.ini file to stdout. - ''' - self.determine_upgrade_type() - - if not uses_ab_partitions(): - log.error('System does not have dual root partitions') - sys.exit(1) - - target = self.get_mount_target() - - file = os.path.normpath('{}/{}' - .format(target, - SYSTEM_IMAGE_CHANNEL_CONFIG)) - - log.debug('Reading file from other partition: {}'.format(file)) - - try: - with open(file, 'r') as f: - sys.stdout.write(f.read()) - except: - # no output by default, denoting an error - log.debug('Cannot find file on other partition: {}' - .format(file)) - - def show_outcome(self): - ''' - Show the user whether a reboot is required and if so, why. - ''' - log.info('System update completed for rootfs on {}' - .format(self.rootfs_to_modify)) - - if self.reboot_reason == self.reboot_reasons.REBOOT_NONE: - log.info('System update completed - no reboot required') - else: - log.warning('System update requires a reboot to finalise') - log.warning('(Reboot reason: {})' - .format(self.reboot_reason)) - - def get_saved_link_path(self, file): - ''' - Returns the full path to the hard-link copy of the inode - associated with @file. - ''' - prefix = self.get_saved_link_dir_prefix() - link_path = os.path.normpath('{}/{}'.format(prefix, file)) - - return link_path - - def get_saved_link_dir(self, file): - ''' - Returns the full path to the directory where the hard-link inode - copy for file @file is located. - ''' - prefix = self.get_saved_link_dir_prefix() - dirname = os.path.dirname(file) - saved_link_dirname = os.path.normpath('{}{}'.format(prefix, dirname)) - - return saved_link_dirname - - def get_saved_link_dir_prefix(self): - ''' - Returns the full path to the directory under which the - hard-link inode file copies are stored. - ''' - self.make_lost_and_found() - - return os.path.join(self.lost_found, - 'ubuntu-core-upgrader') - - def make_lost_and_found(self): - ''' - Create a lost+found directory on the root filesystem. - ''' - if os.path.exists(self.lost_found) and \ - os.path.isdir(self.lost_found) and \ - not os.path.islink(self.lost_found): - return - - cwd = os.getcwd() - os.chdir('/') - - if os.path.islink(self.lost_found) or os.path.isfile(self.lost_found): - os.remove(self.lost_found) - - cmd = 'mklost+found' - args = [cmd] - proc = subprocess.Popen(args, - stderr=subprocess.PIPE, - universal_newlines=True) - if proc.wait() != 0: - stderr = proc.communicate()[1] - log.error('failed to run {}: {}'.format(cmd, stderr)) - sys.exit(1) - - os.chdir(cwd) - - # create a subdirectory to work in - dir = os.path.join(self.lost_found, 'ubuntu-core-upgrader') - try: - os.makedirs(dir, mode=self.DIR_MODE, exist_ok=True) - except Exception as e: - log.error('cannot create directory {}: {}' - .format(dir, e)) - sys.exit(1) - def get_file_contents(self, tar, file): ''' @tar: tarfile object. @@ -1455,193 +676,17 @@ class Upgrader(): tar.extract(path=tmpdir, member=tar.getmember(file)) path = os.path.join(tmpdir, file) - lines = [line.rstrip() for line in open(path, 'r')] - - shutil.rmtree(tmpdir) - return lines + lines = [] - def cleanup_inodes(self): - ''' - Remove stale inodes from below '/lost+found'. - - If the upgrader has already been run and forced a reboot - due to unknown processes holding open inodes, hard links will - still exist below '/lost+found'. - - These need to be removed before the current upgrade process can - continue since it the next upgrade may affect the same files as - last time. - - Cleanup up via the upgrader isn't ideal since inodes are being - wasted in the time window between calls to the upgrader. - However, the only other alternatives are to perform the - cleanup at boot (initramfs) or shutdown and these options are - not ideal as problems could result in an unbootable system (they - would also require yet more toggling of the root FS to rw and we - try to avoid that wherever possible). - ''' - # Create the directory the file needs to live under - path = self.get_saved_link_dir_prefix() + with open(path, 'r') as f: + lines = f.readlines() - if os.path.exists(path): - log.debug('Cleaning up stale inodes from previous run') - shutil.rmtree(path) + lines = [line.rstrip() for line in lines] - def save_links(self, files): - ''' - @files: list of files that will be modified by the - upgrade. - - Create a new (hard) link to all files that will be modified by - the system-image upgrade. - - Also updates the file_map which maps the paths for the files to - change to their current inode. - - We cannot just remove files from the root partition blindly - since the chances are some of the files we wish to remove are - currently being used by running processes (either directly via - open(2) calls, or indirectly via the link loader pulling in - required shared libraries). - - Technically, we *can* remove any file, but we will never be able - to make the filesystem read-only again as it is now in an - inconsistent state (since the unlinked inodes are still in use). - This manifests itself with the dreaded 'mount: / busy' message - when attempting to flip the rootfs to be read-only once again. - - The solution is to hard-link all the files that we are about to - either change or delete into '/lost+found/', retaining their - original directory structure. We can then unlink the master copy, - but retain another copy of the same inode. - - The kernel is then happy to allow us to remount the rootfs - read-only once again. - ''' - - for file in files: - - if file.startswith('/dev/') and not os.path.exists(file): - continue - - if file.startswith('/run/'): - continue - - if not os.path.exists(file): - # ignore files that don't exist - they must be new files - # that are about to be created by the new system image. - continue - - st = os.stat(file) - mode = st.st_mode - - if stat.S_ISDIR(mode): - # ignore directories (and sym-links to directories) - continue - - if stat.S_ISCHR(mode) or stat.S_ISBLK(mode): - # ignore devices - continue - - # save original inode number - self.file_map[file] = st.st_ino - - if self.options.dry_run: - continue - - # File is crosses the filesytem boundary, - # so can only be ignored. - # - # It will either be a sym-link or a bind-mount - # unrelated to those specified by writable-paths(5). - if st.st_dev != self.mountpoint_stat.st_dev: - if self.options.debug > 1: - log.debug('Ignoring cross-FS file: {}' - .format(file)) - continue - - # Create the directory the file needs to live under - saved_link_dirname = self.get_saved_link_dir(file) - - # First, create the directory structure. - # - # Note that we cannot handle the deletion of directories that - # a process has open (readdir(2)) as you can't create a hard - # link to a directory. - # - # FIXME: consider: - # - sym links to files. - # - sym links to directories. - # - try: - if self.options.debug > 1: - log.debug('creating directory {}' - .format(saved_link_dirname)) - os.makedirs(saved_link_dirname, - mode=self.DIR_MODE, - exist_ok=True) - except Exception as e: - log.error('failed to create directory {}: {}' - .format(saved_link_dirname, e)) - sys.exit(1) - - link_path = self.get_saved_link_path(file) - - # Now, create a hard link to the original file in the - # directory. - log.debug('linking {} to {}'.format(file, link_path)) - - try: - os.link(file, link_path) - except Exception as e: - log.error('failed to create link for file {}: {}' - .format(file, e)) - sys.exit(1) - - def remove_links(self, exclude_list): - ''' - @exclude_list: list of files whose hard links below /lost+found should - NOT be removed. Note the each element is a full rootfs path. - - Remove all hard links to inodes below /lost+found except those - specified by the files in "exclude_list. - ''' - - if self.options.dry_run: - return + shutil.rmtree(tmpdir) - prefix = self.get_saved_link_dir_prefix() - - # Remove all the files that are not in @exclude_list - for root, dirs, files in os.walk(self.get_saved_link_dir_prefix(), - topdown=False): - for file in files: - link_path = os.path.join(root, file) - - path = link_path[len(prefix):] - - if path in exclude_list: - log.debug('not removing in-use file {}' - .format(file)) - continue - - log.debug('removing link {}'.format(link_path)) - os.unlink(link_path) - - # Now, remove all the directories we can. - # We KISS by just attempting to remove every directory. Removals - # will fail if the directory is not empty (as will happen when - # they contain a link to a file that is still in use), so we - # ignore such errors but any other is fatal. - for root, dirs, files in os.walk(self.get_saved_link_dir_prefix(), - topdown=False): - for dir in dirs: - try: - os.rmdir(os.path.join(root, dir)) - except OSError as e: - if e.errno != errno.ENOTEMPTY: - raise e + return lines def _cmd_update(self, args): ''' @@ -1694,35 +739,7 @@ class Upgrader(): else: to_remove = [] - if self.upgrade_type == self.upgrade_types.UPGRADE_IN_PLACE: - # Add all files to remove not already in the list of all - # files that will be affected by the upgrade. This overall - # list allows the upgrader to backup files in the case of - # in-place upgrades. - for f in to_remove: - - system_path = remove_prefix(f) - - # path should now be absolute - if not system_path.startswith('/'): - continue - - # ignore relative paths - if '../' in system_path: - continue - - if system_path not in all_files: - all_files.append(f) - - log.debug('saving inode details') - self.save_links(all_files) - - if self.upgrade_type == self.upgrade_types.UPGRADE_AB_PARTITIONS: - # An update to the "other" partition should always flag - # a reboot since the image on that partition is newer. - self.reboot_reason = self.reboot_reasons.REBOOT_NEW_IMAGE - - if not self.full_image and not self.options.check_reboot: + if not self.full_image: if found_removed_file: log.debug('processing {} file' @@ -1777,9 +794,8 @@ class Upgrader(): .format(final, e)) if self.options.dry_run: - if not self.options.check_reboot: - log.info('DRY-RUN: would apply the following files:') - tar.list(verbose=True) + log.info('DRY-RUN: would apply the following files:') + tar.list(verbose=True) else: log.debug('starting unpack') # see bug #1408579, this forces tarfile to use tarinfo.gid @@ -1795,42 +811,6 @@ class Upgrader(): self.options.root_dir)) tar.close() - if self.upgrade_type == self.upgrade_types.UPGRADE_AB_PARTITIONS: - # Nothing further to do - return - - # Look for pids that still have the original inodes for the old - # file versions open. - self.pid_file_map = get_affected_pids(list(self.file_map.values())) - - if len(self.pid_file_map): - log.debug('processes are using new image files from {}' - .format(file)) - self.set_reboot_reason(self.reboot_reasons.REBOOT_OPEN_FILE) - else: - log.debug('no processes are using new image files from {}' - .format(file)) - - # Remove what hard links we can (namely all those that do not - # relate to running processes that have the files open). - if self.upgrade_type == self.upgrade_types.UPGRADE_IN_PLACE: - self.remove_links(list(self.pid_file_map.keys())) - - if len(self.pid_file_map.keys()) == 0: - # there were no processes holding inodes open, so nothing - # more to be done. - return - - if self.options.check_reboot: - return - - if not self.restart_associated_services(): - self.set_reboot_reason(self.reboot_reasons.REBOOT_SERVICE) - else: - # if all services were restarted, all remaining hard-links can - # be removed, hence pass an empty exclude list. - self.remove_links([]) - def sync_partitions(self): ''' Copy all rootfs data from the current partition to the other @@ -1867,61 +847,3 @@ class Upgrader(): unmount(bindmount_rootfs_dir) os.rmdir(bindmount_rootfs_dir) self.other_is_empty = False - - def restart_associated_service(self, pid, file): - ''' - Lookup the service associated with @pid and restart. - - Returns True on success and False if either service could not be - established, or the restart failed. - ''' - - if pid == 1: - # As always, this is special :) - if self.options.dry_run: - log.info('DRY-RUN: would restart systemd (pid {})' - .format(pid)) - return True - else: - log.debug('restarting systemd (pid {})' .format(pid)) - return self.systemd.restart_manager() - else: - service = self.systemd.find_unit(pid) - if not service: - log.debug('cannot determine service for pid {}' - .format(pid)) - # trigger a reboot - return False - - if self.options.dry_run: - log.info('DRY-RUN: would restart service {} (pid {})' - .format(service, pid)) - return True - else: - cmd = get_command(pid) or '<<UNKNOWN>>' - log.debug('restarting service {} ({}, pid {}) ' - 'holding file {} open' - .format(service, cmd, pid, file)) - - return self.systemd.restart_service(service) - - def restart_associated_services(self): - ''' - Restart all services associated with the pids in self.pid_file_map. - - Returns False if any service failed to start, else True. - ''' - failed = False - - self.systemd = Systemd() - - log.debug('System is using systemd version {}' - .format(self.systemd.version())) - - for file in self.pid_file_map: - for pid in self.pid_file_map[file]: - ret = self.restart_associated_service(pid, file) - if not ret: - failed = True - - return not failed |
