Merge snaps into master [a=daniel-thewatkins] [r=fginther,philroche]

MP: https://code.launchpad.net/~daniel-thewatkins/cloud-images/+git/mfdiff/+merge/347725 

6 files changed, 687 insertions, 616 deletions

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..0a0ce11
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,3 @@
+.mypy_cache
+.pytest_cache
+.tox
diff --git a/mfdiff b/mfdiff
index 66744a5..158f7f1 100755..120000
--- a/mfdiff
+++ b/mfdiff
@@ -1,616 +1 @@
-#!/usr/bin/env python
-"""
-Given two manifest files for a particular release/arch, find the
-packages which have been added, removed, and changed. Print the
-changelog entries for the changed packages limited to changes between
-the two versions.
-"""
-
-# vi:ts=4 expandtab
-#
-# Copyright (C) 2010, 2017 Canonical Ltd.
-#
-# Authors: Scott Moser <scott.moser@canonical.com>
-# Robert C Jennings <robert.jennings@canonical.com>
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, version 3 of the License.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-import codecs
-import locale
-import logging
-import os
-import os.path
-import re
-import shutil
-import sys
-import tempfile
-from optparse import OptionParser
-
-import apt
-import requests
-from debian.changelog import Changelog
-from six import PY3, iteritems
-
-try:
- from six import viewkeys
-except ImportError:
- # The version of six in trusty doesn't have viewkeys, so pull in the six
- # code from a more recent version if we can't get it directly from six.
- import operator
- if PY3:
- viewkeys = operator.methodcaller("keys")
- else:
- viewkeys = operator.methodcaller("viewkeys")
-
-try:
- from apt import VersionCompare as version_compare
-except ImportError:
- from apt_pkg import version_compare as version_compare
-
-
-class MissingChangelogError(Exception):
- "The Changelog file could not be found on the server"
- pass
-
-
-class ChangelogMissingVersion(Exception):
- "The Changelog is missing starting and/or ending versions for the search"
- pass
-
-
-class UnknownSourceVersionError(Exception):
- "The binary package did not have a source version listed"
- pass
-
-
-def get_bin2src(packages, cache):
- """
- Find the source package names for a given binary package list
-
- :param list packages: List of binary packages
- :param :class:`apt.Cache` cache: Open/updated apt cache
- :return: List mapping binary package name to source package name
- :rtype: dict
- """
-
- ret = {}
- logging.debug('Finding source package names for all binary packages')
- for binpkg in packages:
- pkg_name = binpkg.split(':')[0]
- ret[binpkg] = cache[pkg_name].versions[0].source_name
- return ret
-
-
-def get_changelog(source, version, changelog_cache_d):
- """
- Download changelog for source / version and returns path to that
-
- :param str source: Source package name
- :param str version: Source package version
- :param str changelog_cache_d: path to store cached changelogs
- :raises MissingChangelogError: If changelog file could not be downloaded
- :return: changelog file for source package & version
- :rtype: str
- """
-
- cache_f = "%s/changelog.%s_%s" % (changelog_cache_d, source, version)
-
- if os.path.isfile(cache_f):
- logging.debug("Using cached changelog for %s:%s", source, version)
- return cache_f
-
- furls = []
- num_colon_m = re.compile("[0-9]:")
-
- cache_tmp_fd, cache_tmp_path = tempfile.mkstemp(
- ".changelog.%s_%s" % (source, version),
- prefix="." + tempfile.gettempprefix(), dir=".")
- cache_tmp = os.fdopen(cache_tmp_fd, "w")
- for pile in ("main", "universe", "multiverse", "restricted"):
- pre = source[0:1]
- # packages starting with 'lib' are special
- if source.startswith("lib"):
- pre = source[0:4]
-
- # packages with '1:' versions have different paths
- # update-manager at version 1:0.134.11.
- # is main/u/update-manager/update-manager_0.134.11/
- # rather than main/u/update-manager/update-manager_1:0.134.11
- url_version = version
- if num_colon_m.match(version):
- url_version = version[2:]
-
- # Changelog URL example http://changelogs.ubuntu.com/changelogs/\
- # pool/main/h/hal/hal_0.5.5.1-1ubuntu2/changelog
- changelog_url = "http://changelogs.ubuntu.com/changelogs/pool/" \
- "%s/%s/%s/%s_%s/changelog" % \
- (pile, pre, source, source, url_version)
-
- changelog = requests.get(changelog_url)
- if changelog.status_code == 200:
- cache_tmp.write(changelog.content)
- cache_tmp.close()
- shutil.copy2(cache_tmp_path, cache_f)
- os.unlink(cache_tmp_path)
- return cache_f
- else:
- logging.error("missing %s: %s", source, changelog_url)
- furls.append(changelog_url)
- if os.path.exists(cache_tmp_path):
- os.unlink(cache_tmp_path)
- raise MissingChangelogError("Failed to find changelog for %s at version "
- "%s.\n tried %s" % (source, version,
- ' '.join(furls)))
-
-
-def manifest_to_dict(filename):
- """
- Parse manifest file to create a package / version mapping
-
- :param str filename: Name of package manifest file
- :return: List of package versions by name
- :rtype: dict
- """
-
- ret = {}
- logging.debug('Reading package manifest from %s', filename)
- with open(filename, "r") as manifest:
- for line in manifest:
- (pkg, ver) = line.split()
- ret[pkg] = ver
- return ret
-
-
-def open_apt_cache(arch, release, cache_d=None):
- """
- Create, update, and open an apt cache.
-
- This creates an apt cache directory and write a sources.list file
- before updating and opening the cache. The caller is responsible
- for closing the cache.
-
- :param str arch: Package architecture
- :param str release: Ubuntu release name (e.g. Xenial)
- :param str cache_d: apt cache path
- :returns: tuple of Open/updated apt cache and cache path name
- :rtype: tuple(:class:`apt.Cache`, str)
- """
-
- if not cache_d:
- cache_d = "./cache.%s-%s" % (release, arch)
- logging.info("Using %s as the apt cache directory", cache_d)
-
- if arch in ['amd64', 'i386']:
- mirror = "http://archive.ubuntu.com/ubuntu/"
- else:
- mirror = "http://ports.ubuntu.com/ubuntu-ports/"
- logging.debug('Configuring apt cache using mirror %s', mirror)
-
- pockets = (release, "%s-updates" % release, "%s-security" % release,
- "%s-proposed" % release, )
- components = ("main", "universe")
- srclines = []
- for pocket in pockets:
- srcline = "deb %s %s %s" % (mirror, pocket, ' '.join(components))
- logging.debug('Adding source: %s', srcline)
- srclines.append(srcline)
- try:
- os.makedirs("%s/etc/apt" % cache_d)
- except OSError as oserror:
- if os.errno.EEXIST != oserror.errno:
- raise
- with open("%s/etc/apt/sources.list" % cache_d, "w") as asl:
- asl.write('\n'.join(srclines))
-
- apt.apt_pkg.config.set("Apt::Architecture", arch)
-
- logging.debug('Using host apt keys for signature verification')
- apt.apt_pkg.config.set("Dir::Etc::Trusted", "/etc/apt/trusted.gpg")
- apt.apt_pkg.config.set("Dir::Etc::TrustedParts",
- "/etc/apt/trusted.gpg.d/")
-
- cache = apt.Cache(rootdir=cache_d)
- logging.info('Updating apt cache')
- cache.update()
- logging.info('Update of apt cache complete')
- cache.open()
- return cache, cache_d
-
-
-def render_block(block):
- """
- Render a changelog block to something printable (dropping blank lines)
-
- :param :class:`debian.changelog.ChangeBlock` block: Changelog block
- :return: String containing the changelog block text
- :rtype: str
- """
- return '\n'.join([x for x in block.changes() if x])
-
-
-def print_blocks(block_list):
- """
- Print a Changelog block list
-
- :param list block_list: List of :class:`debian.changelog.ChangeBlock`
- """
-
- for block in block_list:
- print(render_block(block).encode('utf-8').decode('utf-8'))
-
-
-def kernel_fixups(manifest_from, manifest_to):
- """
- Fix up kernels so the pkg names match
-
- Kernel package names change from release to release so that they are
- co-installable, but we need to find matching package names in the
- two manifests to provide a list of changes between the versions of the
- package in each manifest.
- This function will return an altered version of manifest_from with kernel
- package names changed to match kernel package names in manifest_to. This
- will support later version comparisons.
-
- :param dict manifest_from: Dictionary mapping package to version for of
- the starting manifest
- :param dict manifest_to: Dictionary mapping package to version for the
- ending manifest
- :return: Starting manifest dictionary with altered kernel package names
- to match names in ending manifest
- :rtype dict:
- """
- kfixups = {}
- kmatch = re.compile("linux-image-[0-9]")
- for pkg in manifest_to:
- # if this is a linux-image-* binary package do some hacks to make it
- # look like manifest_from is the same (format like
- # linux-image-2.6.32-32-virtual)
- if kmatch.match(pkg):
- logging.debug('Found kernel %s in manifest #2', pkg)
- img_type = pkg.split("-")[-1]
- if pkg in manifest_from:
- logging.debug('Found same kernel in manifest #1')
- continue
- for fpkg in manifest_from:
- if kmatch.match(fpkg) and fpkg.endswith("-%s" % img_type):
- logging.debug('Found similar kernel %s in manifest #1',
- fpkg)
- kfixups[pkg] = fpkg
-
- for pkg_to, pkg_from in iteritems(kfixups):
- logging.debug('Substituting kernel %s for %s in manifest #1 to '
- 'enable version comparison', pkg_to, pkg_from)
- manifest_from[pkg_to] = manifest_from[pkg_from]
- del manifest_from[pkg_from]
- return manifest_from
-
-
-def find_added(manifest_from, manifest_to):
- "Find new packages in manifest_to"
- new = {}
- for pkg in sorted(viewkeys(manifest_to) - viewkeys(manifest_from)):
- logging.debug('New package: %s', pkg)
- new[pkg] = manifest_to[pkg]
- return new
-
-
-def find_removed(manifest_from, manifest_to):
- "Find packages removed from manifest_from"
- removed = {}
- for pkg in sorted(viewkeys(manifest_from) - viewkeys(manifest_to)):
- logging.debug('Removed package: %s', pkg)
- removed[pkg] = manifest_from[pkg]
- return removed
-
-
-def find_changed(manifest_from, manifest_to):
- "Find modified packages"
- changed = []
- for pkg in sorted(viewkeys(manifest_from) & viewkeys(manifest_to)):
- if manifest_from[pkg] != manifest_to[pkg]:
- logging.debug('Changed package: %s', pkg)
- changed.append(pkg)
- return changed
-
-
-def map_source_to_binary(cache, packages):
- "Create a dictionary of source to list of binary packages"
- src2bins = {}
- for bin_pkg in packages:
- bin_name = bin_pkg.split(':')[0]
- if bin_name in cache:
- src2bins.setdefault(
- cache[bin_name].versions[0].source_name, []).append(bin_pkg)
- return src2bins
-
-
-def get_pkg_versions(cache, binary):
- "Get all known versions from the apt cache"
- pkg_name = binary.split(':')[0]
- try:
- return cache[pkg_name].versions
- except KeyError:
- raise Exception(
- "%s not in cache or did not have version info in cache" %
- pkg_name)
-
-
-def source_version_for_binary(cache, binary, binary_ver):
- "Find the source version data for a specific binary version"
- versions = get_pkg_versions(cache, binary)
- try:
- return versions[binary_ver].source_version
- except KeyError:
- # Strip the architecture name from the binary
- source_name = cache[binary.split(':')[0]].versions[0].source_name
- msg = ("Unable to determine source version for %s. "
- "Binary package %s/%s not in known source version "
- "list (%s)" % (source_name, binary, binary_ver, versions))
- raise UnknownSourceVersionError(msg)
-
-
-def filter_changelog(changelog_path, version_low, version_high):
- """
- Extract changelog entries within a version range
-
- The range of changelog entries returned will include all entries
- after version_low up to, and including, version_high.
- If either the starting or ending version are not found in the
- list of changelog entries the result will be incomplete and
- a non-empty error message is returned to indicate the issue.
-
- :param str changelog_path: File name of the changelog to process
- :return: list of changelog blocks and an error_msg if incomplete
- :rtype tuple(list, str):
- """
-
- with open(changelog_path, "r") as fileptr:
- chlog = Changelog(fileptr.read())
- change_blocks = []
- start = False
- end = False
- error_msg = ''
-
- # The changelog blocks are in reverse order; we'll see high before low.
- for block in chlog:
- if block.version == version_high:
- start = True
- change_blocks = []
- if block.version == version_low:
- end = True
- break
- change_blocks.append(block)
- if not start:
- error_msg = "Missing starting version {} in {}. " \
- "Changlelog will be incomplete".format(
- version_high, changelog_path)
- logging.error(error_msg)
- if not end:
- if error_msg:
- # Start and end were not found, put a newline between their
- # error messages
- error_msg += '\n'
- error_msg += "Missing ending version {} in {}. " \
- "Changelog output truncated".format(
- version_low, changelog_path)
- logging.error(error_msg)
- return change_blocks, error_msg
-
-
-def print_changelogs(apt_cache, apt_cache_d, manifest_from, manifest_to,
- changed):
- """
- Print changelog entries for each changed package limited to
- changes in the package between the versions in the two manifests.
-
- :param :class:`apt.Cache` apt_cache: Open & up-to-date apt cache
- :param str apt_cache_d: apt cache path
- :param dict manifest_from: Packages and their versions in the
- first manifest file
- :param dict manifest_to: Packages and their versions in the
- second manifest file
- :param list changed: Packages which changed between the two manifests
- """
-
- srcs = {}
- errors = []
- src2bins = map_source_to_binary(apt_cache, changed)
-
- # Generate changelog data per unique source package
- for source_name in src2bins:
- srcs[source_name] = {"changelog_file": "", "changeblocks": []}
- src = srcs[source_name]
-
- # Use the first binary listed for a source package
- binary_name = src2bins[source_name][0]
-
- # Find the source version data for the binary in manifest #2
- try:
- src['version_to'] = source_version_for_binary(
- apt_cache, binary_name, manifest_to[binary_name])
- except UnknownSourceVersionError as excp:
- logging.error(str(excp))
- errors.append(excp)
- continue
-
- # Find the source version data for the binary in manifest #1
- binver_from = manifest_from[binary_name]
- try:
- src['version_from'] = source_version_for_binary(
- apt_cache, binary_name, binver_from)
- except UnknownSourceVersionError as excp:
- if manifest_to[binary_name] == src['version_to']:
- logging.info('Could not find source version data in apt '
- 'cache. Assuming source %s version %s from '
- 'binary %s', source_name, binver_from,
- binary_name)
- src['version_from'] = binver_from
- else:
- logging.error(str(excp))
- errors.append(excp)
- continue
-
- # Check for version regression between manifests
- try:
- if version_compare(src['version_from'], src['version_to']) > 0:
- msg = "Package version regression {} -> {}".format(
- src['version_from'], src['version_to'])
- raise UnknownSourceVersionError(msg)
- except UnknownSourceVersionError as excp:
- errors.append(excp)
- continue
-
- # Get the changelog for this source package
- try:
- # Use the apt cache directory to store the changelog cache
- srcs[source_name]["changelog_file"] = get_changelog(
- source_name, src['version_to'], changelog_cache_d=apt_cache_d)
- except MissingChangelogError as excp:
- errors.append(excp)
- continue
-
- # Filter the changelog to a list of blocks between the two versions
- try:
- srcs[source_name]["changeblocks"], incomplete = filter_changelog(
- srcs[source_name]["changelog_file"], src['version_from'],
- src['version_to'])
- if incomplete:
- raise ChangelogMissingVersion(incomplete)
- except ChangelogMissingVersion as exp:
- errors.append(exp)
- continue
-
- # Print changelog ranges for changed packages
- for source_name in sorted(src2bins):
- binlist = sorted(src2bins[source_name])
- binary = binlist[0]
- print("==== %s: %s => %s ====" %
- (source_name, manifest_from[binary], manifest_to[binary]))
- print("==== %s" % ' '.join(binlist))
- print_blocks(srcs[source_name]["changeblocks"])
-
- if errors:
- print("**** Errors ****")
- for error in errors:
- print(error)
-
-
-def parse_args():
- """
- Parse command line arguments
-
- :returns: options and remaining arguments from OptionParser.parse_args()
- :rtype list:
- """
-
- parser = OptionParser(usage="Usage: {} arch suite manifest1 manifest2\n"
- "Compare two manifest files, and show "
- "changelog differences."
- .format(os.path.basename(sys.argv[0])))
- parser.add_option("--cache-dir", dest="cache_d",
- help="cache dir for info", metavar="DIR", type="string",
- default=None)
- parser.add_option("-v", "--verbose", action="count", dest="loglevel",
- help="increase verbosity", default=0)
-
- (options, args) = parser.parse_args()
-
- if len(args) != 4:
- parser.error('you must provide arch, release, and 2 manifest files')
-
- return options, args
-
-
-def setup_logging(loglevel=0):
- """
- Configure logging
-
- By default, log WARNING and higher messages
- :param int: loglevel 0: Warning, 1: Info, 2: Debug
- """
-
- loglevel = [logging.WARNING,
- logging.INFO,
- logging.DEBUG][min(2, loglevel)]
-
- logging.basicConfig(
- level=loglevel,
- format="%(asctime)s %(name)s/%(levelname)s: %(message)s",
- stream=sys.stderr)
-
-
-def stdout_force_unicode():
- """
- Force output to UTF-8 at all times
-
- We want to output UTF-8 at all times to preserve Unicode characters
- in the changelog blocks. For Python 2 we will wrap sys.stdout with
- an instance of StreamWriter with our preferred coding. Python3 requires
- no changes.
-
- When writing output to the terminal we get the encoding of the
- terminal (utf-8 these days). When we redirect or pipe the output of
- the program it is generally not possible to know what the input
- encoding of the receiving program is, the encoding when redirecting
- to a file will be None (Python 2.7) or UTF-8 (Python 3)
-
- $ python2.7 -c "import sys; print sys.stdout.encoding" | cat
- None
-
- $ python3.4 -c "import sys; print(sys.stdout.encoding)" | cat
- UTF-8
-
- Source:
- https://wiki.python.org/moin/PrintFails#print.2C_write_and_Unicode_in_pre-3.0_Python
- """
-
- if sys.version_info[0] < 3:
- encoding = codecs.getwriter(locale.getpreferredencoding())
- sys.stdout = encoding(sys.stdout)
-
-
-def main():
- """
- Given two manifest files for a particular release/arch, find the
- packages which have been added, removed, and changed. Print the
- changelog entries for the changed packages limited to changes between
- the two versions.
- """
-
- stdout_force_unicode()
- options, (arch, release, manifest_from_filename,
- manifest_to_filename) = parse_args()
-
- setup_logging(options.loglevel)
-
- # index both manifests
- manifest_to = manifest_to_dict(manifest_to_filename)
- manifest_from = kernel_fixups(manifest_to_dict(manifest_from_filename),
- manifest_to)
-
- new = find_added(manifest_from, manifest_to)
- removed = find_removed(manifest_from, manifest_to)
- changed = find_changed(manifest_from, manifest_to)
-
- print("new: %s" % new)
- print("removed: %s" % removed)
- print("changed: %s" % changed)
-
- # if modified packages, download all changelogs from changelogs.
- if changed:
- cache, cache_d = open_apt_cache(arch, release, options.cache_d)
- print_changelogs(cache, cache_d, manifest_from, manifest_to, changed)
-
-
-if __name__ == "__main__":
- main()
+mfdiff.py \ No newline at end of file
diff --git a/mfdiff.py b/mfdiff.py
new file mode 100755
index 0000000..feb2a3e
--- /dev/null
+++ b/mfdiff.py
@@ -0,0 +1,619 @@
+#!/usr/bin/env python
+"""
+Given two manifest files for a particular release/arch, find the
+packages which have been added, removed, and changed. Print the
+changelog entries for the changed packages limited to changes between
+the two versions.
+"""
+
+# vi:ts=4 expandtab
+#
+# Copyright (C) 2010, 2017 Canonical Ltd.
+#
+# Authors: Scott Moser <scott.moser@canonical.com>
+# Robert C Jennings <robert.jennings@canonical.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, version 3 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+import codecs
+import locale
+import logging
+import os
+import os.path
+import re
+import shutil
+import sys
+import tempfile
+from optparse import OptionParser
+
+import apt
+import requests
+from debian.changelog import Changelog
+from six import PY3, iteritems
+
+try:
+ from six import viewkeys
+except ImportError:
+ # The version of six in trusty doesn't have viewkeys, so pull in the six
+ # code from a more recent version if we can't get it directly from six.
+ import operator
+ if PY3:
+ viewkeys = operator.methodcaller("keys")
+ else:
+ viewkeys = operator.methodcaller("viewkeys")
+
+try:
+ from apt import VersionCompare as version_compare
+except ImportError:
+ from apt_pkg import version_compare as version_compare
+
+
+class MissingChangelogError(Exception):
+ "The Changelog file could not be found on the server"
+ pass
+
+
+class ChangelogMissingVersion(Exception):
+ "The Changelog is missing starting and/or ending versions for the search"
+ pass
+
+
+class UnknownSourceVersionError(Exception):
+ "The binary package did not have a source version listed"
+ pass
+
+
+def get_bin2src(packages, cache):
+ """
+ Find the source package names for a given binary package list
+
+ :param list packages: List of binary packages
+ :param :class:`apt.Cache` cache: Open/updated apt cache
+ :return: List mapping binary package name to source package name
+ :rtype: dict
+ """
+
+ ret = {}
+ logging.debug('Finding source package names for all binary packages')
+ for binpkg in packages:
+ pkg_name = binpkg.split(':')[0]
+ ret[binpkg] = cache[pkg_name].versions[0].source_name
+ return ret
+
+
+def get_changelog(source, version, changelog_cache_d):
+ """
+ Download changelog for source / version and returns path to that
+
+ :param str source: Source package name
+ :param str version: Source package version
+ :param str changelog_cache_d: path to store cached changelogs
+ :raises MissingChangelogError: If changelog file could not be downloaded
+ :return: changelog file for source package & version
+ :rtype: str
+ """
+
+ cache_f = "%s/changelog.%s_%s" % (changelog_cache_d, source, version)
+
+ if os.path.isfile(cache_f):
+ logging.debug("Using cached changelog for %s:%s", source, version)
+ return cache_f
+
+ furls = []
+ num_colon_m = re.compile("[0-9]:")
+
+ cache_tmp_fd, cache_tmp_path = tempfile.mkstemp(
+ ".changelog.%s_%s" % (source, version),
+ prefix="." + tempfile.gettempprefix(), dir=".")
+ cache_tmp = os.fdopen(cache_tmp_fd, "w")
+ for pile in ("main", "universe", "multiverse", "restricted"):
+ pre = source[0:1]
+ # packages starting with 'lib' are special
+ if source.startswith("lib"):
+ pre = source[0:4]
+
+ # packages with '1:' versions have different paths
+ # update-manager at version 1:0.134.11.
+ # is main/u/update-manager/update-manager_0.134.11/
+ # rather than main/u/update-manager/update-manager_1:0.134.11
+ url_version = version
+ if num_colon_m.match(version):
+ url_version = version[2:]
+
+ # Changelog URL example http://changelogs.ubuntu.com/changelogs/\
+ # pool/main/h/hal/hal_0.5.5.1-1ubuntu2/changelog
+ changelog_url = "http://changelogs.ubuntu.com/changelogs/pool/" \
+ "%s/%s/%s/%s_%s/changelog" % \
+ (pile, pre, source, source, url_version)
+
+ changelog = requests.get(changelog_url)
+ if changelog.status_code == 200:
+ cache_tmp.write(changelog.content)
+ cache_tmp.close()
+ shutil.copy2(cache_tmp_path, cache_f)
+ os.unlink(cache_tmp_path)
+ return cache_f
+ else:
+ logging.error("missing %s: %s", source, changelog_url)
+ furls.append(changelog_url)
+ if os.path.exists(cache_tmp_path):
+ os.unlink(cache_tmp_path)
+ raise MissingChangelogError("Failed to find changelog for %s at version "
+ "%s.\n tried %s" % (source, version,
+ ' '.join(furls)))
+
+
+def manifest_to_dict(filename):
+ """
+ Parse manifest file to create a package / version mapping
+
+ :param str filename: Name of package manifest file
+ :return: List of package versions by name
+ :rtype: dict
+ """
+
+ ret = {}
+ logging.debug('Reading package manifest from %s', filename)
+ with open(filename, "r") as manifest:
+ for line in manifest:
+ if line.startswith('snap:'):
+ # XXX: Introduce support for snap packages
+ continue
+ (pkg, ver) = line.split()
+ ret[pkg] = ver
+ return ret
+
+
+def open_apt_cache(arch, release, cache_d=None):
+ """
+ Create, update, and open an apt cache.
+
+ This creates an apt cache directory and write a sources.list file
+ before updating and opening the cache. The caller is responsible
+ for closing the cache.
+
+ :param str arch: Package architecture
+ :param str release: Ubuntu release name (e.g. Xenial)
+ :param str cache_d: apt cache path
+ :returns: tuple of Open/updated apt cache and cache path name
+ :rtype: tuple(:class:`apt.Cache`, str)
+ """
+
+ if not cache_d:
+ cache_d = "./cache.%s-%s" % (release, arch)
+ logging.info("Using %s as the apt cache directory", cache_d)
+
+ if arch in ['amd64', 'i386']:
+ mirror = "http://archive.ubuntu.com/ubuntu/"
+ else:
+ mirror = "http://ports.ubuntu.com/ubuntu-ports/"
+ logging.debug('Configuring apt cache using mirror %s', mirror)
+
+ pockets = (release, "%s-updates" % release, "%s-security" % release,
+ "%s-proposed" % release, )
+ components = ("main", "universe")
+ srclines = []
+ for pocket in pockets:
+ srcline = "deb %s %s %s" % (mirror, pocket, ' '.join(components))
+ logging.debug('Adding source: %s', srcline)
+ srclines.append(srcline)
+ try:
+ os.makedirs("%s/etc/apt" % cache_d)
+ except OSError as oserror:
+ if os.errno.EEXIST != oserror.errno:
+ raise
+ with open("%s/etc/apt/sources.list" % cache_d, "w") as asl:
+ asl.write('\n'.join(srclines))
+
+ apt.apt_pkg.config.set("Apt::Architecture", arch)
+
+ logging.debug('Using host apt keys for signature verification')
+ apt.apt_pkg.config.set("Dir::Etc::Trusted", "/etc/apt/trusted.gpg")
+ apt.apt_pkg.config.set("Dir::Etc::TrustedParts",
+ "/etc/apt/trusted.gpg.d/")
+
+ cache = apt.Cache(rootdir=cache_d)
+ logging.info('Updating apt cache')
+ cache.update()
+ logging.info('Update of apt cache complete')
+ cache.open()
+ return cache, cache_d
+
+
+def render_block(block):
+ """
+ Render a changelog block to something printable (dropping blank lines)
+
+ :param :class:`debian.changelog.ChangeBlock` block: Changelog block
+ :return: String containing the changelog block text
+ :rtype: str
+ """
+ return '\n'.join([x for x in block.changes() if x])
+
+
+def print_blocks(block_list):
+ """
+ Print a Changelog block list
+
+ :param list block_list: List of :class:`debian.changelog.ChangeBlock`
+ """
+
+ for block in block_list:
+ print(render_block(block).encode('utf-8').decode('utf-8'))
+
+
+def kernel_fixups(manifest_from, manifest_to):
+ """
+ Fix up kernels so the pkg names match
+
+ Kernel package names change from release to release so that they are
+ co-installable, but we need to find matching package names in the
+ two manifests to provide a list of changes between the versions of the
+ package in each manifest.
+ This function will return an altered version of manifest_from with kernel
+ package names changed to match kernel package names in manifest_to. This
+ will support later version comparisons.
+
+ :param dict manifest_from: Dictionary mapping package to version for of
+ the starting manifest
+ :param dict manifest_to: Dictionary mapping package to version for the
+ ending manifest
+ :return: Starting manifest dictionary with altered kernel package names
+ to match names in ending manifest
+ :rtype dict:
+ """
+ kfixups = {}
+ kmatch = re.compile("linux-image-[0-9]")
+ for pkg in manifest_to:
+ # if this is a linux-image-* binary package do some hacks to make it
+ # look like manifest_from is the same (format like
+ # linux-image-2.6.32-32-virtual)
+ if kmatch.match(pkg):
+ logging.debug('Found kernel %s in manifest #2', pkg)
+ img_type = pkg.split("-")[-1]
+ if pkg in manifest_from:
+ logging.debug('Found same kernel in manifest #1')
+ continue
+ for fpkg in manifest_from:
+ if kmatch.match(fpkg) and fpkg.endswith("-%s" % img_type):
+ logging.debug('Found similar kernel %s in manifest #1',
+ fpkg)
+ kfixups[pkg] = fpkg
+
+ for pkg_to, pkg_from in iteritems(kfixups):
+ logging.debug('Substituting kernel %s for %s in manifest #1 to '
+ 'enable version comparison', pkg_to, pkg_from)
+ manifest_from[pkg_to] = manifest_from[pkg_from]
+ del manifest_from[pkg_from]
+ return manifest_from
+
+
+def find_added(manifest_from, manifest_to):
+ "Find new packages in manifest_to"
+ new = {}
+ for pkg in sorted(viewkeys(manifest_to) - viewkeys(manifest_from)):
+ logging.debug('New package: %s', pkg)
+ new[pkg] = manifest_to[pkg]
+ return new
+
+
+def find_removed(manifest_from, manifest_to):
+ "Find packages removed from manifest_from"
+ removed = {}
+ for pkg in sorted(viewkeys(manifest_from) - viewkeys(manifest_to)):
+ logging.debug('Removed package: %s', pkg)
+ removed[pkg] = manifest_from[pkg]
+ return removed
+
+
+def find_changed(manifest_from, manifest_to):
+ "Find modified packages"
+ changed = []
+ for pkg in sorted(viewkeys(manifest_from) & viewkeys(manifest_to)):
+ if manifest_from[pkg] != manifest_to[pkg]:
+ logging.debug('Changed package: %s', pkg)
+ changed.append(pkg)
+ return changed
+
+
+def map_source_to_binary(cache, packages):
+ "Create a dictionary of source to list of binary packages"
+ src2bins = {}
+ for bin_pkg in packages:
+ bin_name = bin_pkg.split(':')[0]
+ if bin_name in cache:
+ src2bins.setdefault(
+ cache[bin_name].versions[0].source_name, []).append(bin_pkg)
+ return src2bins
+
+
+def get_pkg_versions(cache, binary):
+ "Get all known versions from the apt cache"
+ pkg_name = binary.split(':')[0]
+ try:
+ return cache[pkg_name].versions
+ except KeyError:
+ raise Exception(
+ "%s not in cache or did not have version info in cache" %
+ pkg_name)
+
+
+def source_version_for_binary(cache, binary, binary_ver):
+ "Find the source version data for a specific binary version"
+ versions = get_pkg_versions(cache, binary)
+ try:
+ return versions[binary_ver].source_version
+ except KeyError:
+ # Strip the architecture name from the binary
+ source_name = cache[binary.split(':')[0]].versions[0].source_name
+ msg = ("Unable to determine source version for %s. "
+ "Binary package %s/%s not in known source version "
+ "list (%s)" % (source_name, binary, binary_ver, versions))
+ raise UnknownSourceVersionError(msg)
+
+
+def filter_changelog(changelog_path, version_low, version_high):
+ """
+ Extract changelog entries within a version range
+
+ The range of changelog entries returned will include all entries
+ after version_low up to, and including, version_high.
+ If either the starting or ending version are not found in the
+ list of changelog entries the result will be incomplete and
+ a non-empty error message is returned to indicate the issue.
+
+ :param str changelog_path: File name of the changelog to process
+ :return: list of changelog blocks and an error_msg if incomplete
+ :rtype tuple(list, str):
+ """
+
+ with open(changelog_path, "r") as fileptr:
+ chlog = Changelog(fileptr.read())
+ change_blocks = []
+ start = False
+ end = False
+ error_msg = ''
+
+ # The changelog blocks are in reverse order; we'll see high before low.
+ for block in chlog:
+ if block.version == version_high:
+ start = True
+ change_blocks = []
+ if block.version == version_low:
+ end = True
+ break
+ change_blocks.append(block)
+ if not start:
+ error_msg = "Missing starting version {} in {}. " \
+ "Changlelog will be incomplete".format(
+ version_high, changelog_path)
+ logging.error(error_msg)
+ if not end:
+ if error_msg:
+ # Start and end were not found, put a newline between their
+ # error messages
+ error_msg += '\n'
+ error_msg += "Missing ending version {} in {}. " \
+ "Changelog output truncated".format(
+ version_low, changelog_path)
+ logging.error(error_msg)
+ return change_blocks, error_msg
+
+
+def print_changelogs(apt_cache, apt_cache_d, manifest_from, manifest_to,
+ changed):
+ """
+ Print changelog entries for each changed package limited to
+ changes in the package between the versions in the two manifests.
+
+ :param :class:`apt.Cache` apt_cache: Open & up-to-date apt cache
+ :param str apt_cache_d: apt cache path
+ :param dict manifest_from: Packages and their versions in the
+ first manifest file
+ :param dict manifest_to: Packages and their versions in the
+ second manifest file
+ :param list changed: Packages which changed between the two manifests
+ """
+
+ srcs = {}
+ errors = []
+ src2bins = map_source_to_binary(apt_cache, changed)
+
+ # Generate changelog data per unique source package
+ for source_name in src2bins:
+ srcs[source_name] = {"changelog_file": "", "changeblocks": []}
+ src = srcs[source_name]
+
+ # Use the first binary listed for a source package
+ binary_name = src2bins[source_name][0]
+
+ # Find the source version data for the binary in manifest #2
+ try:
+ src['version_to'] = source_version_for_binary(
+ apt_cache, binary_name, manifest_to[binary_name])
+ except UnknownSourceVersionError as excp:
+ logging.error(str(excp))
+ errors.append(excp)
+ continue
+
+ # Find the source version data for the binary in manifest #1
+ binver_from = manifest_from[binary_name]
+ try:
+ src['version_from'] = source_version_for_binary(
+ apt_cache, binary_name, binver_from)
+ except UnknownSourceVersionError as excp:
+ if manifest_to[binary_name] == src['version_to']:
+ logging.info('Could not find source version data in apt '
+ 'cache. Assuming source %s version %s from '
+ 'binary %s', source_name, binver_from,
+ binary_name)
+ src['version_from'] = binver_from
+ else:
+ logging.error(str(excp))
+ errors.append(excp)
+ continue
+
+ # Check for version regression between manifests
+ try:
+ if version_compare(src['version_from'], src['version_to']) > 0:
+ msg = "Package version regression {} -> {}".format(
+ src['version_from'], src['version_to'])
+ raise UnknownSourceVersionError(msg)
+ except UnknownSourceVersionError as excp:
+ errors.append(excp)
+ continue
+
+ # Get the changelog for this source package
+ try:
+ # Use the apt cache directory to store the changelog cache
+ srcs[source_name]["changelog_file"] = get_changelog(
+ source_name, src['version_to'], changelog_cache_d=apt_cache_d)
+ except MissingChangelogError as excp:
+ errors.append(excp)
+ continue
+
+ # Filter the changelog to a list of blocks between the two versions
+ try:
+ srcs[source_name]["changeblocks"], incomplete = filter_changelog(
+ srcs[source_name]["changelog_file"], src['version_from'],
+ src['version_to'])
+ if incomplete:
+ raise ChangelogMissingVersion(incomplete)
+ except ChangelogMissingVersion as exp:
+ errors.append(exp)
+ continue
+
+ # Print changelog ranges for changed packages
+ for source_name in sorted(src2bins):
+ binlist = sorted(src2bins[source_name])
+ binary = binlist[0]
+ print("==== %s: %s => %s ====" %
+ (source_name, manifest_from[binary], manifest_to[binary]))
+ print("==== %s" % ' '.join(binlist))
+ print_blocks(srcs[source_name]["changeblocks"])
+
+ if errors:
+ print("**** Errors ****")
+ for error in errors:
+ print(error)
+
+
+def parse_args():
+ """
+ Parse command line arguments
+
+ :returns: options and remaining arguments from OptionParser.parse_args()
+ :rtype list:
+ """
+
+ parser = OptionParser(usage="Usage: {} arch suite manifest1 manifest2\n"
+ "Compare two manifest files, and show "
+ "changelog differences."
+ .format(os.path.basename(sys.argv[0])))
+ parser.add_option("--cache-dir", dest="cache_d",
+ help="cache dir for info", metavar="DIR", type="string",
+ default=None)
+ parser.add_option("-v", "--verbose", action="count", dest="loglevel",
+ help="increase verbosity", default=0)
+
+ (options, args) = parser.parse_args()
+
+ if len(args) != 4:
+ parser.error('you must provide arch, release, and 2 manifest files')
+
+ return options, args
+
+
+def setup_logging(loglevel=0):
+ """
+ Configure logging
+
+ By default, log WARNING and higher messages
+ :param int: loglevel 0: Warning, 1: Info, 2: Debug
+ """
+
+ loglevel = [logging.WARNING,
+ logging.INFO,
+ logging.DEBUG][min(2, loglevel)]
+
+ logging.basicConfig(
+ level=loglevel,
+ format="%(asctime)s %(name)s/%(levelname)s: %(message)s",
+ stream=sys.stderr)
+
+
+def stdout_force_unicode():
+ """
+ Force output to UTF-8 at all times
+
+ We want to output UTF-8 at all times to preserve Unicode characters
+ in the changelog blocks. For Python 2 we will wrap sys.stdout with
+ an instance of StreamWriter with our preferred coding. Python3 requires
+ no changes.
+
+ When writing output to the terminal we get the encoding of the
+ terminal (utf-8 these days). When we redirect or pipe the output of
+ the program it is generally not possible to know what the input
+ encoding of the receiving program is, the encoding when redirecting
+ to a file will be None (Python 2.7) or UTF-8 (Python 3)
+
+ $ python2.7 -c "import sys; print sys.stdout.encoding" | cat
+ None
+
+ $ python3.4 -c "import sys; print(sys.stdout.encoding)" | cat
+ UTF-8
+
+ Source:
+ https://wiki.python.org/moin/PrintFails#print.2C_write_and_Unicode_in_pre-3.0_Python
+ """
+
+ if sys.version_info[0] < 3:
+ encoding = codecs.getwriter(locale.getpreferredencoding())
+ sys.stdout = encoding(sys.stdout)
+
+
+def main():
+ """
+ Given two manifest files for a particular release/arch, find the
+ packages which have been added, removed, and changed. Print the
+ changelog entries for the changed packages limited to changes between
+ the two versions.
+ """
+
+ stdout_force_unicode()
+ options, (arch, release, manifest_from_filename,
+ manifest_to_filename) = parse_args()
+
+ setup_logging(options.loglevel)
+
+ # index both manifests
+ manifest_to = manifest_to_dict(manifest_to_filename)
+ manifest_from = kernel_fixups(manifest_to_dict(manifest_from_filename),
+ manifest_to)
+
+ new = find_added(manifest_from, manifest_to)
+ removed = find_removed(manifest_from, manifest_to)
+ changed = find_changed(manifest_from, manifest_to)
+
+ print("new: %s" % new)
+ print("removed: %s" % removed)
+ print("changed: %s" % changed)
+
+ # if modified packages, download all changelogs from changelogs.
+ if changed:
+ cache, cache_d = open_apt_cache(arch, release, options.cache_d)
+ print_changelogs(cache, cache_d, manifest_from, manifest_to, changed)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/test-requirements.txt b/test-requirements.txt
new file mode 100644
index 0000000..e079f8a
--- /dev/null
+++ b/test-requirements.txt
@@ -0,0 +1 @@
+pytest
diff --git a/tests.py b/tests.py
new file mode 100644
index 0000000..d5271b8
--- /dev/null
+++ b/tests.py
@@ -0,0 +1,51 @@
+import mfdiff
+
+
+def _deb_package_line(package_name, package_version):
+ """Format a deb package manifest line"""
+ return '\t'.join([package_name, package_version])
+
+
+def _snap_package_line(snap_name, snap_channel, snap_version):
+ """Format a snap package manifest line"""
+ return '\t'.join(
+ ['snap:{}'.format(snap_name), snap_channel, snap_version])
+
+
+
+class TestManifestToDict(object):
+
+ def test_empty_file_returns_empty_dict(self, tmpdir):
+ manifest_file = tmpdir.join('manifest')
+ manifest_file.write('')
+ assert mfdiff.manifest_to_dict(str(manifest_file)) == {}
+
+ def test_single_package(self, tmpdir):
+ package_name, package_version = 'package_name', '1.0-0ubuntu1'
+ manifest_file = tmpdir.join('manifest')
+ manifest_file.write(_deb_package_line(package_name, package_version))
+ assert mfdiff.manifest_to_dict(str(manifest_file)) == {
+ package_name: package_version,
+ }
+
+ def test_snaps_are_skipped(self, tmpdir):
+ package_name, package_version = 'package_name', '1.0-0ubuntu1'
+ manifest_file = tmpdir.join('manifest')
+ manifest_file.write('\n'.join([
+ _deb_package_line(package_name, package_version),
+ _snap_package_line('snap_name', 'snap_channel', 'snap_version')
+ ]))
+ assert mfdiff.manifest_to_dict(str(manifest_file)) == {
+ package_name: package_version,
+ }
+
+ def test_deb_packages_starting_with_snap_arent_skipped(self, tmpdir):
+ package_name, package_version = 'snapd', '1.0-0ubuntu1'
+ manifest_file = tmpdir.join('manifest')
+ manifest_file.write('\n'.join([
+ _deb_package_line(package_name, package_version),
+ _snap_package_line('snap_name', 'snap_channel', 'snap_version')
+ ]))
+ assert mfdiff.manifest_to_dict(str(manifest_file)) == {
+ package_name: package_version,
+ }
diff --git a/tox.ini b/tox.ini
new file mode 100644
index 0000000..dc15493
--- /dev/null
+++ b/tox.ini
@@ -0,0 +1,12 @@
+[tox]
+envlist = py27,py3
+skip_install=true
+skipsdist=true
+
+[testenv]
+deps=
+ -rtest-requirements.txt
+commands=
+ pytest tests.py
+# We need sitepackages because python-apt isn't in PyPI
+sitepackages=true