Merge features/refactor into master [a=tobijk] [r=daniel-thewatkins,philroche]

Refactor mfdiff script into a reusable component MP: https://code.launchpad.net/~cloud-images-release-managers/cloud-images/+git/mfdiff/+merge/359346 

28 files changed, 1286 insertions, 666 deletions

diff --git a/.gitignore b/.gitignore
index 0a0ce11..a0bb53d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,6 @@
 .mypy_cache
 .pytest_cache
 .tox
+*.pyc
+__pycache__
+mfdiff.egg-info/
diff --git a/debian/changelog b/debian/changelog
new file mode 100644
index 0000000..58fe8a5
--- /dev/null
+++ b/debian/changelog
@@ -0,0 +1,5 @@
+mfdiff (0.1.0-1) unstable; urgency=medium
+
+ * Initial release
+
+ -- Tobias Koch <tobias.koch@canonical.com> Wed, 14 Nov 2018 12:08:39 +0100
diff --git a/debian/compat b/debian/compat
new file mode 100644
index 0000000..f599e28
--- /dev/null
+++ b/debian/compat
@@ -0,0 +1 @@
+10
diff --git a/debian/control b/debian/control
new file mode 100644
index 0000000..0faed99
--- /dev/null
+++ b/debian/control
@@ -0,0 +1,51 @@
+Source: mfdiff
+Section: admin
+Priority: optional
+Maintainer: Tobias Koch <tobias.koch@canonical.com>
+Build-Depends: debhelper (>= 10),
+ dh-python,
+ python-all,
+ python-setuptools,
+ python3-all,
+ python3-setuptools,
+ tox
+Standards-Version: 4.1.2
+Homepage: https://code.launchpad.net/~cloud-images-release-managers/cloud-images/+git/mfdiff
+X-Python-Version: >= 2.6
+X-Python3-Version: >= 3.2
+Vcs-Git: https://anonscm.debian.org/git/python-modules/packages/mfdiff.git
+Vcs-Browser: https://anonscm.debian.org/cgit/python-modules/packages/mfdiff.git/
+Testsuite: autopkgtest-pkg-python
+
+Package: mfdiff
+Architecture: all
+Depends: python3-mfdiff
+Description: A program for diffing cloud image manifest files (Python 3)
+ During cloud image builds, the build system emits a manifest file for
+ each image variant. This manifest file contains a list of Debian packages
+ and a list of snaps included in the image. This tool allows taking
+ two manifests and comparing their contents.
+ .
+ This package depends on the Python 3 version of mfdiff.
+
+Package: python-mfdiff
+Architecture: all
+Depends: ${python:Depends}, ${misc:Depends}
+Description: A program for diffing cloud image manifest files (Python 2)
+ During cloud image builds, the build system emits a manifest file for
+ each image variant. This manifest file contains a list of Debian packages
+ and a list of snaps included in the image. This tool allows taking
+ two manifests and comparing their contents.
+ .
+ This package installs the library for Python 2.
+
+Package: python3-mfdiff
+Architecture: all
+Depends: ${python3:Depends}, ${misc:Depends}
+Description: A program for diffing cloud image manifest files (Python 3)
+ During cloud image builds, the build system emits a manifest file for
+ each image variant. This manifest file contains a list of Debian packages
+ and a list of snaps included in the image. This tool allows taking
+ two manifests and comparing their contents.
+ .
+ This package installs the library for Python 3.
diff --git a/debian/copyright b/debian/copyright
new file mode 100644
index 0000000..a040eab
--- /dev/null
+++ b/debian/copyright
@@ -0,0 +1,40 @@
+Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+Upstream-Name: mfdiff
+Source: https://code.launchpad.net/~cloud-images-release-managers/cloud-images/+git/mfdiff
+
+Files: *
+Copyright: Copyright (C) 2010, 2017, 2018 Canonical Ltd.
+ Scott Moser <scott.moser@canonical.com>
+ Robert C Jennings <robert.jennings@canonical.com>
+ Tobias Koch <tobias.koch@canonical.com>
+License: GPL-3
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, version 3 of the License.
+ .
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+ .
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Files: debian/*
+Copyright: 2018 Canonical Ltd.
+License: GPL-2+
+ This package is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+ .
+ This package is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+ .
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see <https://www.gnu.org/licenses/>
+ .
+ On Debian systems, the complete text of the GNU General
+ Public License version 2 can be found in "/usr/share/common-licenses/GPL-2".
diff --git a/debian/python-mfdiff.postinst b/debian/python-mfdiff.postinst
new file mode 100644
index 0000000..d28f4f0
--- /dev/null
+++ b/debian/python-mfdiff.postinst
@@ -0,0 +1,40 @@
+#!/bin/sh
+# postinst script for mfdiff
+#
+# see: dh_installdeb(1)
+
+set -e
+
+# summary of how this script can be called:
+# * <postinst> `configure' <most-recently-configured-version>
+# * <old-postinst> `abort-upgrade' <new version>
+# * <conflictor's-postinst> `abort-remove' `in-favour' <package>
+# <new-version>
+# * <postinst> `abort-remove'
+# * <deconfigured's-postinst> `abort-deconfigure' `in-favour'
+# <failed-install-package> <version> `removing'
+# <conflicting-package> <version>
+# for details, see https://www.debian.org/doc/debian-policy/ or
+# the debian-policy package
+
+
+case "$1" in
+ configure)
+ update-alternatives --install /usr/bin/mfdiff mfdiff /usr/bin/mfdiff-py2 100
+ ;;
+
+ abort-upgrade|abort-remove|abort-deconfigure)
+ ;;
+
+ *)
+ echo "postinst called with unknown argument \`$1'" >&2
+ exit 1
+ ;;
+esac
+
+# dh_installdeb will replace this with shell code automatically
+# generated by other debhelper scripts.
+
+#DEBHELPER#
+
+exit 0
diff --git a/debian/python-mfdiff.prerm b/debian/python-mfdiff.prerm
new file mode 100644
index 0000000..9937bed
--- /dev/null
+++ b/debian/python-mfdiff.prerm
@@ -0,0 +1,39 @@
+#!/bin/sh
+# prerm script for mfdiff
+#
+# see: dh_installdeb(1)
+
+set -e
+
+# summary of how this script can be called:
+# * <prerm> `remove'
+# * <old-prerm> `upgrade' <new-version>
+# * <new-prerm> `failed-upgrade' <old-version>
+# * <conflictor's-prerm> `remove' `in-favour' <package> <new-version>
+# * <deconfigured's-prerm> `deconfigure' `in-favour'
+# <package-being-installed> <version> `removing'
+# <conflicting-package> <version>
+# for details, see https://www.debian.org/doc/debian-policy/ or
+# the debian-policy package
+
+
+case "$1" in
+ remove|upgrade|deconfigure)
+ update-alternatives --remove mfdiff /usr/bin/mfdiff-py2
+ ;;
+
+ failed-upgrade)
+ ;;
+
+ *)
+ echo "prerm called with unknown argument \`$1'" >&2
+ exit 1
+ ;;
+esac
+
+# dh_installdeb will replace this with shell code automatically
+# generated by other debhelper scripts.
+
+#DEBHELPER#
+
+exit 0
diff --git a/debian/python3-mfdiff.postinst b/debian/python3-mfdiff.postinst
new file mode 100644
index 0000000..c69d090
--- /dev/null
+++ b/debian/python3-mfdiff.postinst
@@ -0,0 +1,40 @@
+#!/bin/sh
+# postinst script for mfdiff
+#
+# see: dh_installdeb(1)
+
+set -e
+
+# summary of how this script can be called:
+# * <postinst> `configure' <most-recently-configured-version>
+# * <old-postinst> `abort-upgrade' <new version>
+# * <conflictor's-postinst> `abort-remove' `in-favour' <package>
+# <new-version>
+# * <postinst> `abort-remove'
+# * <deconfigured's-postinst> `abort-deconfigure' `in-favour'
+# <failed-install-package> <version> `removing'
+# <conflicting-package> <version>
+# for details, see https://www.debian.org/doc/debian-policy/ or
+# the debian-policy package
+
+
+case "$1" in
+ configure)
+ update-alternatives --install /usr/bin/mfdiff mfdiff /usr/bin/mfdiff-py3 110
+ ;;
+
+ abort-upgrade|abort-remove|abort-deconfigure)
+ ;;
+
+ *)
+ echo "postinst called with unknown argument \`$1'" >&2
+ exit 1
+ ;;
+esac
+
+# dh_installdeb will replace this with shell code automatically
+# generated by other debhelper scripts.
+
+#DEBHELPER#
+
+exit 0
diff --git a/debian/python3-mfdiff.prerm b/debian/python3-mfdiff.prerm
new file mode 100644
index 0000000..1ba2308
--- /dev/null
+++ b/debian/python3-mfdiff.prerm
@@ -0,0 +1,39 @@
+#!/bin/sh
+# prerm script for mfdiff
+#
+# see: dh_installdeb(1)
+
+set -e
+
+# summary of how this script can be called:
+# * <prerm> `remove'
+# * <old-prerm> `upgrade' <new-version>
+# * <new-prerm> `failed-upgrade' <old-version>
+# * <conflictor's-prerm> `remove' `in-favour' <package> <new-version>
+# * <deconfigured's-prerm> `deconfigure' `in-favour'
+# <package-being-installed> <version> `removing'
+# <conflicting-package> <version>
+# for details, see https://www.debian.org/doc/debian-policy/ or
+# the debian-policy package
+
+
+case "$1" in
+ remove|upgrade|deconfigure)
+ update-alternatives --remove mfdiff /usr/bin/mfdiff-py3
+ ;;
+
+ failed-upgrade)
+ ;;
+
+ *)
+ echo "prerm called with unknown argument \`$1'" >&2
+ exit 1
+ ;;
+esac
+
+# dh_installdeb will replace this with shell code automatically
+# generated by other debhelper scripts.
+
+#DEBHELPER#
+
+exit 0
diff --git a/debian/rules b/debian/rules
new file mode 100755
index 0000000..2cc8151
--- /dev/null
+++ b/debian/rules
@@ -0,0 +1,20 @@
+#!/usr/bin/make -f
+
+export DH_VERBOSE = 1
+export PYBUILD_NAME=mfdiff
+
+%:
+	dh $@ --with python2,python3 --buildsystem=pybuild
+
+override_dh_auto_install:
+	dh_auto_install
+	mv debian/python-mfdiff/usr/bin/mfdiff debian/python-mfdiff/usr/bin/mfdiff-py2
+	mv debian/python3-mfdiff/usr/bin/mfdiff debian/python3-mfdiff/usr/bin/mfdiff-py3
+
+override_dh_auto_test:
+	tox
+
+override_dh_auto_clean:
+	rm -fr .pybuild .mypy_cache .pytest_cache .tox .eggs *.egg-info
+	find . \( -name __pycache__ -o -name "*.pyc" \) -prune \
+	-exec rm -fr '{}' ';'
diff --git a/debian/source/format b/debian/source/format
new file mode 100644
index 0000000..89ae9db
--- /dev/null
+++ b/debian/source/format
@@ -0,0 +1 @@
+3.0 (native)
diff --git a/debian/source/options b/debian/source/options
new file mode 100644
index 0000000..cb61fa5
--- /dev/null
+++ b/debian/source/options
@@ -0,0 +1 @@
+extend-diff-ignore = "^[^/]*[.]egg-info/"
diff --git a/mfdiff.py b/mfdiff.py
index feb2a3e..b6325c4 100755
--- a/mfdiff.py
+++ b/mfdiff.py
@@ -1,619 +1,13 @@
 #!/usr/bin/env python
-"""
-Given two manifest files for a particular release/arch, find the
-packages which have been added, removed, and changed. Print the
-changelog entries for the changed packages limited to changes between
-the two versions.
-"""
 
-# vi:ts=4 expandtab
-#
-# Copyright (C) 2010, 2017 Canonical Ltd.
-#
-# Authors: Scott Moser <scott.moser@canonical.com>
-# Robert C Jennings <robert.jennings@canonical.com>
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, version 3 of the License.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-import codecs
-import locale
-import logging
 import os
-import os.path
-import re
-import shutil
 import sys
-import tempfile
-from optparse import OptionParser
-
-import apt
-import requests
-from debian.changelog import Changelog
-from six import PY3, iteritems
-
-try:
- from six import viewkeys
-except ImportError:
- # The version of six in trusty doesn't have viewkeys, so pull in the six
- # code from a more recent version if we can't get it directly from six.
- import operator
- if PY3:
- viewkeys = operator.methodcaller("keys")
- else:
- viewkeys = operator.methodcaller("viewkeys")
-
-try:
- from apt import VersionCompare as version_compare
-except ImportError:
- from apt_pkg import version_compare as version_compare
-
-
-class MissingChangelogError(Exception):
- "The Changelog file could not be found on the server"
- pass
-
-
-class ChangelogMissingVersion(Exception):
- "The Changelog is missing starting and/or ending versions for the search"
- pass
-
-
-class UnknownSourceVersionError(Exception):
- "The binary package did not have a source version listed"
- pass
-
-
-def get_bin2src(packages, cache):
- """
- Find the source package names for a given binary package list
-
- :param list packages: List of binary packages
- :param :class:`apt.Cache` cache: Open/updated apt cache
- :return: List mapping binary package name to source package name
- :rtype: dict
- """
-
- ret = {}
- logging.debug('Finding source package names for all binary packages')
- for binpkg in packages:
- pkg_name = binpkg.split(':')[0]
- ret[binpkg] = cache[pkg_name].versions[0].source_name
- return ret
-
-
-def get_changelog(source, version, changelog_cache_d):
- """
- Download changelog for source / version and returns path to that
-
- :param str source: Source package name
- :param str version: Source package version
- :param str changelog_cache_d: path to store cached changelogs
- :raises MissingChangelogError: If changelog file could not be downloaded
- :return: changelog file for source package & version
- :rtype: str
- """
-
- cache_f = "%s/changelog.%s_%s" % (changelog_cache_d, source, version)
-
- if os.path.isfile(cache_f):
- logging.debug("Using cached changelog for %s:%s", source, version)
- return cache_f
-
- furls = []
- num_colon_m = re.compile("[0-9]:")
-
- cache_tmp_fd, cache_tmp_path = tempfile.mkstemp(
- ".changelog.%s_%s" % (source, version),
- prefix="." + tempfile.gettempprefix(), dir=".")
- cache_tmp = os.fdopen(cache_tmp_fd, "w")
- for pile in ("main", "universe", "multiverse", "restricted"):
- pre = source[0:1]
- # packages starting with 'lib' are special
- if source.startswith("lib"):
- pre = source[0:4]
-
- # packages with '1:' versions have different paths
- # update-manager at version 1:0.134.11.
- # is main/u/update-manager/update-manager_0.134.11/
- # rather than main/u/update-manager/update-manager_1:0.134.11
- url_version = version
- if num_colon_m.match(version):
- url_version = version[2:]
-
- # Changelog URL example http://changelogs.ubuntu.com/changelogs/\
- # pool/main/h/hal/hal_0.5.5.1-1ubuntu2/changelog
- changelog_url = "http://changelogs.ubuntu.com/changelogs/pool/" \
- "%s/%s/%s/%s_%s/changelog" % \
- (pile, pre, source, source, url_version)
-
- changelog = requests.get(changelog_url)
- if changelog.status_code == 200:
- cache_tmp.write(changelog.content)
- cache_tmp.close()
- shutil.copy2(cache_tmp_path, cache_f)
- os.unlink(cache_tmp_path)
- return cache_f
- else:
- logging.error("missing %s: %s", source, changelog_url)
- furls.append(changelog_url)
- if os.path.exists(cache_tmp_path):
- os.unlink(cache_tmp_path)
- raise MissingChangelogError("Failed to find changelog for %s at version "
- "%s.\n tried %s" % (source, version,
- ' '.join(furls)))
-
-
-def manifest_to_dict(filename):
- """
- Parse manifest file to create a package / version mapping
-
- :param str filename: Name of package manifest file
- :return: List of package versions by name
- :rtype: dict
- """
-
- ret = {}
- logging.debug('Reading package manifest from %s', filename)
- with open(filename, "r") as manifest:
- for line in manifest:
- if line.startswith('snap:'):
- # XXX: Introduce support for snap packages
- continue
- (pkg, ver) = line.split()
- ret[pkg] = ver
- return ret
-
-
-def open_apt_cache(arch, release, cache_d=None):
- """
- Create, update, and open an apt cache.
-
- This creates an apt cache directory and write a sources.list file
- before updating and opening the cache. The caller is responsible
- for closing the cache.
-
- :param str arch: Package architecture
- :param str release: Ubuntu release name (e.g. Xenial)
- :param str cache_d: apt cache path
- :returns: tuple of Open/updated apt cache and cache path name
- :rtype: tuple(:class:`apt.Cache`, str)
- """
-
- if not cache_d:
- cache_d = "./cache.%s-%s" % (release, arch)
- logging.info("Using %s as the apt cache directory", cache_d)
-
- if arch in ['amd64', 'i386']:
- mirror = "http://archive.ubuntu.com/ubuntu/"
- else:
- mirror = "http://ports.ubuntu.com/ubuntu-ports/"
- logging.debug('Configuring apt cache using mirror %s', mirror)
-
- pockets = (release, "%s-updates" % release, "%s-security" % release,
- "%s-proposed" % release, )
- components = ("main", "universe")
- srclines = []
- for pocket in pockets:
- srcline = "deb %s %s %s" % (mirror, pocket, ' '.join(components))
- logging.debug('Adding source: %s', srcline)
- srclines.append(srcline)
- try:
- os.makedirs("%s/etc/apt" % cache_d)
- except OSError as oserror:
- if os.errno.EEXIST != oserror.errno:
- raise
- with open("%s/etc/apt/sources.list" % cache_d, "w") as asl:
- asl.write('\n'.join(srclines))
-
- apt.apt_pkg.config.set("Apt::Architecture", arch)
-
- logging.debug('Using host apt keys for signature verification')
- apt.apt_pkg.config.set("Dir::Etc::Trusted", "/etc/apt/trusted.gpg")
- apt.apt_pkg.config.set("Dir::Etc::TrustedParts",
- "/etc/apt/trusted.gpg.d/")
-
- cache = apt.Cache(rootdir=cache_d)
- logging.info('Updating apt cache')
- cache.update()
- logging.info('Update of apt cache complete')
- cache.open()
- return cache, cache_d
-
-
-def render_block(block):
- """
- Render a changelog block to something printable (dropping blank lines)
-
- :param :class:`debian.changelog.ChangeBlock` block: Changelog block
- :return: String containing the changelog block text
- :rtype: str
- """
- return '\n'.join([x for x in block.changes() if x])
-
-
-def print_blocks(block_list):
- """
- Print a Changelog block list
-
- :param list block_list: List of :class:`debian.changelog.ChangeBlock`
- """
-
- for block in block_list:
- print(render_block(block).encode('utf-8').decode('utf-8'))
-
-
-def kernel_fixups(manifest_from, manifest_to):
- """
- Fix up kernels so the pkg names match
-
- Kernel package names change from release to release so that they are
- co-installable, but we need to find matching package names in the
- two manifests to provide a list of changes between the versions of the
- package in each manifest.
- This function will return an altered version of manifest_from with kernel
- package names changed to match kernel package names in manifest_to. This
- will support later version comparisons.
-
- :param dict manifest_from: Dictionary mapping package to version for of
- the starting manifest
- :param dict manifest_to: Dictionary mapping package to version for the
- ending manifest
- :return: Starting manifest dictionary with altered kernel package names
- to match names in ending manifest
- :rtype dict:
- """
- kfixups = {}
- kmatch = re.compile("linux-image-[0-9]")
- for pkg in manifest_to:
- # if this is a linux-image-* binary package do some hacks to make it
- # look like manifest_from is the same (format like
- # linux-image-2.6.32-32-virtual)
- if kmatch.match(pkg):
- logging.debug('Found kernel %s in manifest #2', pkg)
- img_type = pkg.split("-")[-1]
- if pkg in manifest_from:
- logging.debug('Found same kernel in manifest #1')
- continue
- for fpkg in manifest_from:
- if kmatch.match(fpkg) and fpkg.endswith("-%s" % img_type):
- logging.debug('Found similar kernel %s in manifest #1',
- fpkg)
- kfixups[pkg] = fpkg
-
- for pkg_to, pkg_from in iteritems(kfixups):
- logging.debug('Substituting kernel %s for %s in manifest #1 to '
- 'enable version comparison', pkg_to, pkg_from)
- manifest_from[pkg_to] = manifest_from[pkg_from]
- del manifest_from[pkg_from]
- return manifest_from
-
-
-def find_added(manifest_from, manifest_to):
- "Find new packages in manifest_to"
- new = {}
- for pkg in sorted(viewkeys(manifest_to) - viewkeys(manifest_from)):
- logging.debug('New package: %s', pkg)
- new[pkg] = manifest_to[pkg]
- return new
-
-
-def find_removed(manifest_from, manifest_to):
- "Find packages removed from manifest_from"
- removed = {}
- for pkg in sorted(viewkeys(manifest_from) - viewkeys(manifest_to)):
- logging.debug('Removed package: %s', pkg)
- removed[pkg] = manifest_from[pkg]
- return removed
-
-
-def find_changed(manifest_from, manifest_to):
- "Find modified packages"
- changed = []
- for pkg in sorted(viewkeys(manifest_from) & viewkeys(manifest_to)):
- if manifest_from[pkg] != manifest_to[pkg]:
- logging.debug('Changed package: %s', pkg)
- changed.append(pkg)
- return changed
-
-
-def map_source_to_binary(cache, packages):
- "Create a dictionary of source to list of binary packages"
- src2bins = {}
- for bin_pkg in packages:
- bin_name = bin_pkg.split(':')[0]
- if bin_name in cache:
- src2bins.setdefault(
- cache[bin_name].versions[0].source_name, []).append(bin_pkg)
- return src2bins
-
-
-def get_pkg_versions(cache, binary):
- "Get all known versions from the apt cache"
- pkg_name = binary.split(':')[0]
- try:
- return cache[pkg_name].versions
- except KeyError:
- raise Exception(
- "%s not in cache or did not have version info in cache" %
- pkg_name)
-
-
-def source_version_for_binary(cache, binary, binary_ver):
- "Find the source version data for a specific binary version"
- versions = get_pkg_versions(cache, binary)
- try:
- return versions[binary_ver].source_version
- except KeyError:
- # Strip the architecture name from the binary
- source_name = cache[binary.split(':')[0]].versions[0].source_name
- msg = ("Unable to determine source version for %s. "
- "Binary package %s/%s not in known source version "
- "list (%s)" % (source_name, binary, binary_ver, versions))
- raise UnknownSourceVersionError(msg)
-
-
-def filter_changelog(changelog_path, version_low, version_high):
- """
- Extract changelog entries within a version range
-
- The range of changelog entries returned will include all entries
- after version_low up to, and including, version_high.
- If either the starting or ending version are not found in the
- list of changelog entries the result will be incomplete and
- a non-empty error message is returned to indicate the issue.
-
- :param str changelog_path: File name of the changelog to process
- :return: list of changelog blocks and an error_msg if incomplete
- :rtype tuple(list, str):
- """
-
- with open(changelog_path, "r") as fileptr:
- chlog = Changelog(fileptr.read())
- change_blocks = []
- start = False
- end = False
- error_msg = ''
-
- # The changelog blocks are in reverse order; we'll see high before low.
- for block in chlog:
- if block.version == version_high:
- start = True
- change_blocks = []
- if block.version == version_low:
- end = True
- break
- change_blocks.append(block)
- if not start:
- error_msg = "Missing starting version {} in {}. " \
- "Changlelog will be incomplete".format(
- version_high, changelog_path)
- logging.error(error_msg)
- if not end:
- if error_msg:
- # Start and end were not found, put a newline between their
- # error messages
- error_msg += '\n'
- error_msg += "Missing ending version {} in {}. " \
- "Changelog output truncated".format(
- version_low, changelog_path)
- logging.error(error_msg)
- return change_blocks, error_msg
-
-
-def print_changelogs(apt_cache, apt_cache_d, manifest_from, manifest_to,
- changed):
- """
- Print changelog entries for each changed package limited to
- changes in the package between the versions in the two manifests.
-
- :param :class:`apt.Cache` apt_cache: Open & up-to-date apt cache
- :param str apt_cache_d: apt cache path
- :param dict manifest_from: Packages and their versions in the
- first manifest file
- :param dict manifest_to: Packages and their versions in the
- second manifest file
- :param list changed: Packages which changed between the two manifests
- """
-
- srcs = {}
- errors = []
- src2bins = map_source_to_binary(apt_cache, changed)
-
- # Generate changelog data per unique source package
- for source_name in src2bins:
- srcs[source_name] = {"changelog_file": "", "changeblocks": []}
- src = srcs[source_name]
-
- # Use the first binary listed for a source package
- binary_name = src2bins[source_name][0]
-
- # Find the source version data for the binary in manifest #2
- try:
- src['version_to'] = source_version_for_binary(
- apt_cache, binary_name, manifest_to[binary_name])
- except UnknownSourceVersionError as excp:
- logging.error(str(excp))
- errors.append(excp)
- continue
-
- # Find the source version data for the binary in manifest #1
- binver_from = manifest_from[binary_name]
- try:
- src['version_from'] = source_version_for_binary(
- apt_cache, binary_name, binver_from)
- except UnknownSourceVersionError as excp:
- if manifest_to[binary_name] == src['version_to']:
- logging.info('Could not find source version data in apt '
- 'cache. Assuming source %s version %s from '
- 'binary %s', source_name, binver_from,
- binary_name)
- src['version_from'] = binver_from
- else:
- logging.error(str(excp))
- errors.append(excp)
- continue
-
- # Check for version regression between manifests
- try:
- if version_compare(src['version_from'], src['version_to']) > 0:
- msg = "Package version regression {} -> {}".format(
- src['version_from'], src['version_to'])
- raise UnknownSourceVersionError(msg)
- except UnknownSourceVersionError as excp:
- errors.append(excp)
- continue
-
- # Get the changelog for this source package
- try:
- # Use the apt cache directory to store the changelog cache
- srcs[source_name]["changelog_file"] = get_changelog(
- source_name, src['version_to'], changelog_cache_d=apt_cache_d)
- except MissingChangelogError as excp:
- errors.append(excp)
- continue
-
- # Filter the changelog to a list of blocks between the two versions
- try:
- srcs[source_name]["changeblocks"], incomplete = filter_changelog(
- srcs[source_name]["changelog_file"], src['version_from'],
- src['version_to'])
- if incomplete:
- raise ChangelogMissingVersion(incomplete)
- except ChangelogMissingVersion as exp:
- errors.append(exp)
- continue
-
- # Print changelog ranges for changed packages
- for source_name in sorted(src2bins):
- binlist = sorted(src2bins[source_name])
- binary = binlist[0]
- print("==== %s: %s => %s ====" %
- (source_name, manifest_from[binary], manifest_to[binary]))
- print("==== %s" % ' '.join(binlist))
- print_blocks(srcs[source_name]["changeblocks"])
-
- if errors:
- print("**** Errors ****")
- for error in errors:
- print(error)
-
-
-def parse_args():
- """
- Parse command line arguments
-
- :returns: options and remaining arguments from OptionParser.parse_args()
- :rtype list:
- """
-
- parser = OptionParser(usage="Usage: {} arch suite manifest1 manifest2\n"
- "Compare two manifest files, and show "
- "changelog differences."
- .format(os.path.basename(sys.argv[0])))
- parser.add_option("--cache-dir", dest="cache_d",
- help="cache dir for info", metavar="DIR", type="string",
- default=None)
- parser.add_option("-v", "--verbose", action="count", dest="loglevel",
- help="increase verbosity", default=0)
-
- (options, args) = parser.parse_args()
-
- if len(args) != 4:
- parser.error('you must provide arch, release, and 2 manifest files')
-
- return options, args
-
-
-def setup_logging(loglevel=0):
- """
- Configure logging
-
- By default, log WARNING and higher messages
- :param int: loglevel 0: Warning, 1: Info, 2: Debug
- """
-
- loglevel = [logging.WARNING,
- logging.INFO,
- logging.DEBUG][min(2, loglevel)]
-
- logging.basicConfig(
- level=loglevel,
- format="%(asctime)s %(name)s/%(levelname)s: %(message)s",
- stream=sys.stderr)
-
-
-def stdout_force_unicode():
- """
- Force output to UTF-8 at all times
-
- We want to output UTF-8 at all times to preserve Unicode characters
- in the changelog blocks. For Python 2 we will wrap sys.stdout with
- an instance of StreamWriter with our preferred coding. Python3 requires
- no changes.
-
- When writing output to the terminal we get the encoding of the
- terminal (utf-8 these days). When we redirect or pipe the output of
- the program it is generally not possible to know what the input
- encoding of the receiving program is, the encoding when redirecting
- to a file will be None (Python 2.7) or UTF-8 (Python 3)
-
- $ python2.7 -c "import sys; print sys.stdout.encoding" | cat
- None
-
- $ python3.4 -c "import sys; print(sys.stdout.encoding)" | cat
- UTF-8
-
- Source:
- https://wiki.python.org/moin/PrintFails#print.2C_write_and_Unicode_in_pre-3.0_Python
- """
-
- if sys.version_info[0] < 3:
- encoding = codecs.getwriter(locale.getpreferredencoding())
- sys.stdout = encoding(sys.stdout)
-
-
-def main():
- """
- Given two manifest files for a particular release/arch, find the
- packages which have been added, removed, and changed. Print the
- changelog entries for the changed packages limited to changes between
- the two versions.
- """
-
- stdout_force_unicode()
- options, (arch, release, manifest_from_filename,
- manifest_to_filename) = parse_args()
-
- setup_logging(options.loglevel)
-
- # index both manifests
- manifest_to = manifest_to_dict(manifest_to_filename)
- manifest_from = kernel_fixups(manifest_to_dict(manifest_from_filename),
- manifest_to)
-
- new = find_added(manifest_from, manifest_to)
- removed = find_removed(manifest_from, manifest_to)
- changed = find_changed(manifest_from, manifest_to)
-
- print("new: %s" % new)
- print("removed: %s" % removed)
- print("changed: %s" % changed)
 
- # if modified packages, download all changelogs from changelogs.
- if changed:
- cache, cache_d = open_apt_cache(arch, release, options.cache_d)
- print_changelogs(cache, cache_d, manifest_from, manifest_to, changed)
+# make this script relocatable
+MFDIFF_SOURCE_DIR = os.path.normpath(os.path.dirname(
+ os.path.realpath(sys.argv[0])))
+sys.path.insert(1, MFDIFF_SOURCE_DIR)
 
+from ubuntu.cloudimage.mfdiff.cli import main
 
-if __name__ == "__main__":
- main()
+main()
diff --git a/setup.py b/setup.py
new file mode 100755
index 0000000..ffef7ed
--- /dev/null
+++ b/setup.py
@@ -0,0 +1,45 @@
+#!/usr/bin/env python
+# -*- encoding: utf-8 -*-
+# vi:ts=4 expandtab
+#
+# Copyright (C) 2010, 2017, 2018 Canonical Ltd.
+#
+# Authors: Scott Moser <scott.moser@canonical.com>
+# Robert C Jennings <robert.jennings@canonical.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, version 3 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+from textwrap import dedent
+from setuptools import setup
+
+setup(
+ name='mfdiff',
+ version=1.0,
+ maintainer='Cloud Image Release Managers',
+ license='GNU GPL v3',
+ description='Image manifest diffing tool',
+ long_description=dedent("""
+ During image builds, the build system emits a manifest file for each
+ image variant. This manifest file contains a list of Debian packages
+ and a list of snaps included in the image. This tool allows taking
+ two manifests and comparing their contents.
+ """).strip(),
+ url='https://code.launchpad.net/~cloudware/cloud-images/+git/mfdiff',
+ packages=['ubuntu.cloudimage.mfdiff'],
+ namespace_packages=['ubuntu', 'ubuntu.cloudimage'],
+ entry_points={
+ 'console_scripts': [
+ 'mfdiff = ubuntu.cloudimage.mfdiff.cli:main'
+ ]
+ },
+)
diff --git a/test/__init__.py b/test/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/test/__init__.py
diff --git a/test/test_manifest.py b/test/test_manifest.py
new file mode 100644
index 0000000..29ab614
--- /dev/null
+++ b/test/test_manifest.py
@@ -0,0 +1,42 @@
+from ubuntu.cloudimage.mfdiff import Manifest
+from .util import deb_package_line, snap_package_line
+
+
+class TestManifest(object):
+
+ def test_empty_file_results_in_empty_manifest(self, tmpdir):
+ manifest_file = tmpdir.join('manifest')
+ manifest_file.write('')
+ manifest = Manifest(str(manifest_file), 'bionic', 'amd64')
+ assert len(manifest) == 0
+ assert manifest.dict == {}
+
+ def test_single_package(self, tmpdir):
+ package_name, package_version = 'package_name', '1.0-0ubuntu1'
+ manifest_file = tmpdir.join('manifest')
+ manifest_file.write(deb_package_line(package_name, package_version))
+ manifest = Manifest(str(manifest_file), 'bionic', 'amd64')
+ assert len(manifest) == 1
+ assert manifest.dict == { package_name: package_version }
+
+ def test_snaps_are_skipped(self, tmpdir):
+ package_name, package_version = 'package_name', '1.0-0ubuntu1'
+ manifest_file = tmpdir.join('manifest')
+ manifest_file.write('\n'.join([
+ deb_package_line(package_name, package_version),
+ snap_package_line('snap_name', 'snap_channel', 'snap_version')
+ ]))
+ manifest = Manifest(str(manifest_file), 'bionic', 'amd64')
+ assert len(manifest) == 1
+ assert manifest.dict == { package_name: package_version }
+
+ def test_deb_packages_starting_with_snap_arent_skipped(self, tmpdir):
+ package_name, package_version = 'snapd', '1.0-0ubuntu1'
+ manifest_file = tmpdir.join('manifest')
+ manifest_file.write('\n'.join([
+ deb_package_line(package_name, package_version),
+ snap_package_line('snap_name', 'snap_channel', 'snap_version')
+ ]))
+ manifest = Manifest(str(manifest_file), 'bionic', 'amd64')
+ assert len(manifest) == 1
+ assert manifest.dict == { package_name: package_version }
diff --git a/test/test_manifestdiff.py b/test/test_manifestdiff.py
new file mode 100644
index 0000000..3f6cfec
--- /dev/null
+++ b/test/test_manifestdiff.py
@@ -0,0 +1,128 @@
+from ubuntu.cloudimage.mfdiff import Manifest, ManifestDiff
+from ubuntu.cloudimage.mfdiff.error import IncompatibleManifestError
+from .util import deb_package_line, snap_package_line, write_manifest_file
+
+class TestManifestDiff(object):
+
+ def test_incompatible_manifest_raise_error(self, tmpdir):
+ package_name, package_version = 'package', '1.0-0ubuntu1'
+ manifest_file1 = tmpdir.join('manifest1')
+ manifest_file2 = tmpdir.join('manifest2')
+ manifest_file1.write(deb_package_line(package_name, package_version))
+ manifest_file2.write(deb_package_line(package_name, package_version))
+ manifest1 = Manifest(str(manifest_file1), 'bionic', 'amd64')
+ manifest2 = Manifest(str(manifest_file2), 'bionic', 'arm64')
+
+ incompatible = False
+ try:
+ ManifestDiff(manifest1, manifest2)
+ except IncompatibleManifestError:
+ incompatible = True
+
+ assert incompatible == True
+
+ def test_find_added_packages(self, tmpdir):
+ package_list = [
+ ('package1', '1.0-1'),
+ ('package2', '0.9.1-0ubuntu1'),
+ ]
+ additional_packages = [
+ ('package3', '0.8a~tp1-1ubuntu1'),
+ ]
+
+ manifest_file1 = tmpdir.join('manifest1')
+ manifest_file2 = tmpdir.join('manifest2')
+ write_manifest_file(manifest_file1, package_list)
+ write_manifest_file(manifest_file2, package_list
+ + additional_packages)
+
+ manifest1 = Manifest(str(manifest_file1), 'bionic', 'amd64')
+ manifest2 = Manifest(str(manifest_file2), 'bionic', 'amd64')
+
+ diff = ManifestDiff(manifest1, manifest2)
+ added_packages = diff.get_added()
+ assert added_packages == dict(additional_packages)
+
+ def test_find_removed_packages(self, tmpdir):
+ package_list = [
+ ('package1', '1.0-1'),
+ ('package2', '0.9.1-0ubuntu1')
+ ]
+ additional_packages = [
+ ('package3', '0.8a~tp1-1ubuntu1')
+ ]
+
+ manifest_file1 = tmpdir.join('manifest1')
+ manifest_file2 = tmpdir.join('manifest2')
+ write_manifest_file(manifest_file1, package_list
+ + additional_packages)
+ write_manifest_file(manifest_file2, package_list)
+
+ manifest1 = Manifest(str(manifest_file1), 'bionic', 'amd64')
+ manifest2 = Manifest(str(manifest_file2), 'bionic', 'amd64')
+
+ diff = ManifestDiff(manifest1, manifest2)
+ removed_packages = diff.get_removed()
+ assert removed_packages == dict(additional_packages)
+
+ def test_find_changed_packages(self, tmpdir):
+ package_list1 = [
+ ('package1', '1.0-1'),
+ ('package2', '0.9.1-0ubuntu1'),
+ ('package3', '2.0'),
+ ]
+ package_list2 = [
+ ('package1', '1.0-1'),
+ ('package2', '0.10.1-0ubuntu1'),
+ ('package3', '2.0'),
+ ]
+
+ manifest_file1 = tmpdir.join('manifest1')
+ manifest_file2 = tmpdir.join('manifest2')
+ write_manifest_file(manifest_file1, package_list1)
+ write_manifest_file(manifest_file2, package_list2)
+
+ manifest1 = Manifest(str(manifest_file1), 'bionic', 'amd64')
+ manifest2 = Manifest(str(manifest_file2), 'bionic', 'amd64')
+ diff = ManifestDiff(manifest1, manifest2)
+ changed_packages = diff.get_changed()
+ assert changed_packages == ['package2']
+
+ def test_find_added_removed_changed(self, tmpdir):
+ package_list1 = [
+ ('package1', '1.0'),
+ ('package4', '1.0'),
+ ('package5', '1.0'),
+ ('package6', '1.0'),
+ ('package7', '1.0'),
+ ('package8', '1.0'),
+ ('package9', '1.0'),
+ ('package10', '1.0')
+ ]
+ package_list2 = [
+ ('package1', '1.0'),
+ ('package2', '1.0'),
+ ('package3', '1.0'),
+ ('package4', '2.0'),
+ ('package5', '2.0'),
+ ('package6', '1.0'),
+ ('package9', '1.0'),
+ ('package10', '1.0')
+ ]
+
+ manifest_file1 = tmpdir.join('manifest1')
+ manifest_file2 = tmpdir.join('manifest2')
+ write_manifest_file(manifest_file1, package_list1)
+ write_manifest_file(manifest_file2, package_list2)
+
+ manifest1 = Manifest(str(manifest_file1), 'bionic', 'amd64')
+ manifest2 = Manifest(str(manifest_file2), 'bionic', 'amd64')
+ diff = ManifestDiff(manifest1, manifest2)
+
+ added_packages = diff.get_added()
+ removed_packages = diff.get_removed()
+ changed_packages = diff.get_changed()
+
+ assert added_packages == {'package2': '1.0', 'package3': '1.0'}
+ assert removed_packages == {'package7': '1.0', 'package8': '1.0'}
+ assert changed_packages == ['package4', 'package5']
diff --git a/test/util.py b/test/util.py
new file mode 100644
index 0000000..9979a78
--- /dev/null
+++ b/test/util.py
@@ -0,0 +1,21 @@
+def deb_package_line(package_name, package_version):
+ """Format a deb package manifest line"""
+ return '\t'.join([package_name, package_version])
+
+
+def snap_package_line(snap_name, snap_channel, snap_version):
+ """Format a snap package manifest line"""
+ return '\t'.join(
+ ['snap:{}'.format(snap_name), snap_channel, snap_version])
+
+def write_manifest_file(path_obj, packages):
+ """Write a list of packages to a manifest file."""
+ manifest_lines = []
+
+ # Then add more to the second.
+ for package_name, package_version in packages:
+ manifest_lines.append(deb_package_line(
+ package_name, package_version))
+
+ path_obj.write("\n".join(manifest_lines))
+
diff --git a/tests.py b/tests.py
deleted file mode 100644
index d5271b8..0000000
--- a/tests.py
+++ /dev/null
@@ -1,51 +0,0 @@
-import mfdiff
-
-
-def _deb_package_line(package_name, package_version):
- """Format a deb package manifest line"""
- return '\t'.join([package_name, package_version])
-
-
-def _snap_package_line(snap_name, snap_channel, snap_version):
- """Format a snap package manifest line"""
- return '\t'.join(
- ['snap:{}'.format(snap_name), snap_channel, snap_version])
-
-
-
-class TestManifestToDict(object):
-
- def test_empty_file_returns_empty_dict(self, tmpdir):
- manifest_file = tmpdir.join('manifest')
- manifest_file.write('')
- assert mfdiff.manifest_to_dict(str(manifest_file)) == {}
-
- def test_single_package(self, tmpdir):
- package_name, package_version = 'package_name', '1.0-0ubuntu1'
- manifest_file = tmpdir.join('manifest')
- manifest_file.write(_deb_package_line(package_name, package_version))
- assert mfdiff.manifest_to_dict(str(manifest_file)) == {
- package_name: package_version,
- }
-
- def test_snaps_are_skipped(self, tmpdir):
- package_name, package_version = 'package_name', '1.0-0ubuntu1'
- manifest_file = tmpdir.join('manifest')
- manifest_file.write('\n'.join([
- _deb_package_line(package_name, package_version),
- _snap_package_line('snap_name', 'snap_channel', 'snap_version')
- ]))
- assert mfdiff.manifest_to_dict(str(manifest_file)) == {
- package_name: package_version,
- }
-
- def test_deb_packages_starting_with_snap_arent_skipped(self, tmpdir):
- package_name, package_version = 'snapd', '1.0-0ubuntu1'
- manifest_file = tmpdir.join('manifest')
- manifest_file.write('\n'.join([
- _deb_package_line(package_name, package_version),
- _snap_package_line('snap_name', 'snap_channel', 'snap_version')
- ]))
- assert mfdiff.manifest_to_dict(str(manifest_file)) == {
- package_name: package_version,
- }
diff --git a/tox.ini b/tox.ini
index dc15493..8caf5f9 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,12 +1,12 @@
 [tox]
 envlist = py27,py3
-skip_install=true
-skipsdist=true
+skip_install=false
+skipsdist=false
 
 [testenv]
 deps=
 -rtest-requirements.txt
 commands=
- pytest tests.py
+ python -m pytest test/
 # We need sitepackages because python-apt isn't in PyPI
 sitepackages=true
diff --git a/ubuntu/__init__.py b/ubuntu/__init__.py
new file mode 100644
index 0000000..69e3be5
--- /dev/null
+++ b/ubuntu/__init__.py
@@ -0,0 +1 @@
+__path__ = __import__('pkgutil').extend_path(__path__, __name__)
diff --git a/ubuntu/cloudimage/__init__.py b/ubuntu/cloudimage/__init__.py
new file mode 100644
index 0000000..69e3be5
--- /dev/null
+++ b/ubuntu/cloudimage/__init__.py
@@ -0,0 +1 @@
+__path__ = __import__('pkgutil').extend_path(__path__, __name__)
diff --git a/ubuntu/cloudimage/mfdiff/__init__.py b/ubuntu/cloudimage/mfdiff/__init__.py
new file mode 100644
index 0000000..02cab8e
--- /dev/null
+++ b/ubuntu/cloudimage/mfdiff/__init__.py
@@ -0,0 +1,3 @@
+from .manifest import Manifest
+from .manifestdiff import ManifestDiff
+from .manifestcache import ManifestCache
diff --git a/ubuntu/cloudimage/mfdiff/cli.py b/ubuntu/cloudimage/mfdiff/cli.py
new file mode 100644
index 0000000..ba8d102
--- /dev/null
+++ b/ubuntu/cloudimage/mfdiff/cli.py
@@ -0,0 +1,141 @@
+# vi:ts=4 expandtab
+#
+# Copyright (C) 2010, 2017, 2018 Canonical Ltd.
+#
+# Authors: Scott Moser <scott.moser@canonical.com>
+# Robert C Jennings <robert.jennings@canonical.com>
+# Tobias Koch <tobias.koch@canonical.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, version 3 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+"""
+Given two manifest files for a particular release/arch, find the
+packages which have been added, removed, and changed. Print the
+changelog entries for the changed packages limited to changes between
+the two versions.
+"""
+
+import codecs
+import locale
+import logging
+import os
+import sys
+from optparse import OptionParser
+
+from .manifest import Manifest
+from .manifestdiff import ManifestDiff
+
+def parse_args():
+ """
+ Parse command line arguments
+
+ :returns: options and remaining arguments from OptionParser.parse_args()
+ :rtype list:
+ """
+
+ parser = OptionParser(usage="Usage: {} arch suite manifest1 manifest2\n"
+ "Compare two manifest files, and show "
+ "changelog differences."
+ .format(os.path.basename(sys.argv[0])))
+ parser.add_option("--cache-dir", dest="cache_d",
+ help="cache dir for info", metavar="DIR", type="string",
+ default=None)
+ parser.add_option("-v", "--verbose", action="count", dest="loglevel",
+ help="increase verbosity", default=0)
+
+ (options, args) = parser.parse_args()
+
+ if len(args) != 4:
+ parser.error('you must provide arch, release, and 2 manifest files')
+
+ return options, args
+
+
+def setup_logging(loglevel=0):
+ """
+ Configure logging
+
+ By default, log WARNING and higher messages
+ :param int: loglevel 0: Warning, 1: Info, 2: Debug
+ """
+
+ loglevel = [logging.WARNING,
+ logging.INFO,
+ logging.DEBUG][min(2, loglevel)]
+
+ logging.basicConfig(
+ level=loglevel,
+ format="%(asctime)s %(name)s/%(levelname)s: %(message)s",
+ stream=sys.stderr)
+
+
+def stdout_force_unicode():
+ """
+ Force output to UTF-8 at all times
+
+ We want to output UTF-8 at all times to preserve Unicode characters
+ in the changelog blocks. For Python 2 we will wrap sys.stdout with
+ an instance of StreamWriter with our preferred coding. Python3 requires
+ no changes.
+
+ When writing output to the terminal we get the encoding of the
+ terminal (utf-8 these days). When we redirect or pipe the output of
+ the program it is generally not possible to know what the input
+ encoding of the receiving program is, the encoding when redirecting
+ to a file will be None (Python 2.7) or UTF-8 (Python 3)
+
+ $ python2.7 -c "import sys; print sys.stdout.encoding" | cat
+ None
+
+ $ python3.4 -c "import sys; print(sys.stdout.encoding)" | cat
+ UTF-8
+
+ Source:
+ https://wiki.python.org/moin/PrintFails#print.2C_write_and_Unicode_in_pre-3.0_Python
+ """
+
+ if sys.version_info[0] < 3:
+ encoding = codecs.getwriter(locale.getpreferredencoding())
+ sys.stdout = encoding(sys.stdout)
+
+
+def main():
+ """
+ Given two manifest files for a particular release/arch, find the
+ packages which have been added, removed, and changed. Print the
+ changelog entries for the changed packages limited to changes between
+ the two versions.
+ """
+
+ stdout_force_unicode()
+ options, (arch, release, manifest_from_filename,
+ manifest_to_filename) = parse_args()
+
+ setup_logging(options.loglevel)
+
+ # index both manifests and create diff object
+ manifest_to = Manifest(manifest_to_filename, release, arch)
+ manifest_from = Manifest(manifest_from_filename, release, arch)
+ manifest_diff = ManifestDiff(manifest_from, manifest_to)
+
+ added_pkgs = manifest_diff.get_added()
+ removed_pkgs = manifest_diff.get_removed()
+ changed_pkgs = manifest_diff.get_changed()
+
+ print("new: %s" % added_pkgs)
+ print("removed: %s" % removed_pkgs)
+ print("changed: %s" % changed_pkgs)
+
+ # if modified packages, download all changelogs from changelogs.
+ if changed_pkgs:
+ print(manifest_diff.render_changelog())
diff --git a/ubuntu/cloudimage/mfdiff/error.py b/ubuntu/cloudimage/mfdiff/error.py
new file mode 100644
index 0000000..5ab39e1
--- /dev/null
+++ b/ubuntu/cloudimage/mfdiff/error.py
@@ -0,0 +1,35 @@
+# vi:ts=4 expandtab
+#
+# Copyright (C) 2010, 2017, 2018 Canonical Ltd.
+#
+# Authors: Scott Moser <scott.moser@canonical.com>
+# Robert C Jennings <robert.jennings@canonical.com>
+# Tobias Koch <tobias.koch@canonical.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, version 3 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+class MissingChangelogError(Exception):
+ "The Changelog file could not be found on the server."
+ pass
+
+class ChangelogMissingVersion(Exception):
+ "The Changelog is missing starting and/or ending versions for the search."
+ pass
+
+class UnknownSourceVersionError(Exception):
+ "The binary package did not have a source version listed."
+ pass
+
+class IncompatibleManifestError(Exception):
+ "The manifests to diff are not compatible."
+ pass
diff --git a/ubuntu/cloudimage/mfdiff/manifest.py b/ubuntu/cloudimage/mfdiff/manifest.py
new file mode 100644
index 0000000..2ac24ba
--- /dev/null
+++ b/ubuntu/cloudimage/mfdiff/manifest.py
@@ -0,0 +1,62 @@
+# vi:ts=4 expandtab
+#
+# Copyright (C) 2010, 2017, 2018 Canonical Ltd.
+#
+# Authors: Scott Moser <scott.moser@canonical.com>
+# Robert C Jennings <robert.jennings@canonical.com>
+# Tobias Koch <tobias.koch@canonical.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, version 3 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+import logging
+
+
+class Manifest(object):
+ """Represents an indexed manifest file."""
+
+ def __init__(self, filename, release, arch):
+ self.release = release
+ self.arch = arch
+ self.dict = self._manifest_to_dict(filename)
+
+ def __iter__(self):
+ for item in self.dict:
+ yield item
+
+ def __getitem__(self, key):
+ return self.dict[key]
+
+ def __delitem__(self, key):
+ del self.dict[key]
+
+ def __len__(self):
+ return len(self.dict)
+
+ @classmethod
+ def _manifest_to_dict(cls, filename):
+ """
+ Parse manifest file to create a package / version mapping
+
+ :param str filename: Name of package manifest file
+ :return: List of package versions by name
+ :rtype: dict
+ """
+ ret = {}
+ logging.debug('Reading package manifest from %s', filename)
+ with open(filename, "r") as manifest:
+ for line in manifest:
+ if line.startswith("snap:"):
+ continue
+ (pkg, ver) = line.split()
+ ret[pkg] = ver
+ return ret
diff --git a/ubuntu/cloudimage/mfdiff/manifestcache.py b/ubuntu/cloudimage/mfdiff/manifestcache.py
new file mode 100644
index 0000000..e92e925
--- /dev/null
+++ b/ubuntu/cloudimage/mfdiff/manifestcache.py
@@ -0,0 +1,234 @@
+# vi:ts=4 expandtab
+#
+# Copyright (C) 2010, 2017, 2018 Canonical Ltd.
+#
+# Authors: Scott Moser <scott.moser@canonical.com>
+# Robert C Jennings <robert.jennings@canonical.com>
+# Tobias Koch <tobias.koch@canonical.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, version 3 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+import apt
+import logging
+import os
+import re
+import requests
+import shutil
+import tempfile
+
+from .error import MissingChangelogError, UnknownSourceVersionError
+
+
+class ManifestCache(object):
+ """An abstraction over apt.Cache and plain changelog files stored in a
+ local directory."""
+
+ def __init__(self, release, arch, cache_d=None):
+ """
+ :param str release: Ubuntu release name (e.g. Xenial)
+ :param str arch: Package architecture
+ :param str cache_d: Name of directory for storing cache entries.
+ """
+ if not cache_d:
+ cache_d = "./cache.%s-%s" % (release, arch)
+ logging.info("Using %s as the apt cache directory", cache_d)
+
+ self._cache_d = cache_d
+ self._apt_cache = None
+ self._release = release
+ self._arch = arch
+
+ def get_bin2src_mapping(self, packages):
+ """
+ Find the source package names for a given binary package list
+
+ :param list packages: List of binary packages
+ :return: List mapping binary package name to source package name
+ :rtype: dict
+ """
+ if not self._apt_cache:
+ self.open()
+
+ ret = {}
+ logging.debug('Finding source package names for all binary packages')
+ for binpkg in packages:
+ pkg_name = binpkg.split(':')[0]
+ ret[binpkg] = self._apt_cache[pkg_name].versions[0].source_name
+ return ret
+
+ def get_src2bin_mapping(self, packages):
+ """
+ Create a dictionary of source to list of binary packages.
+
+ :param list packages: List of sources packages
+ :return List mapping source packages name to list of binary packages.
+ :rtype: dict
+ """
+ if not self._apt_cache:
+ self.open()
+
+ src2bins = {}
+ for bin_pkg in packages:
+ bin_name = bin_pkg.split(':')[0]
+ if bin_name in self._apt_cache:
+ src2bins.setdefault(
+ self._apt_cache[bin_name].versions[0].source_name, [])\
+ .append(bin_pkg)
+ return src2bins
+
+ def get_changelog(self, source, version):
+ """
+ Download changelog for source / version and returns path to that
+
+ :param str source: Source package name
+ :param str version: Source package version
+ :raises MissingChangelogError: If changelog file could not be downloaded
+ :return: changelog file for source package & version
+ :rtype: str
+ """
+ cache_f = "%s/changelog.%s_%s" % (self._cache_d, source, version)
+
+ if os.path.isfile(cache_f):
+ logging.debug("Using cached changelog for %s:%s", source, version)
+ return cache_f
+
+ furls = []
+ num_colon_m = re.compile("[0-9]:")
+
+ cache_tmp_fd, cache_tmp_path = tempfile.mkstemp(
+ ".changelog.%s_%s" % (source, version),
+ prefix="." + tempfile.gettempprefix(), dir=".")
+ cache_tmp = os.fdopen(cache_tmp_fd, "wb")
+ for pile in ("main", "universe", "multiverse", "restricted"):
+ pre = source[0:1]
+ # packages starting with 'lib' are special
+ if source.startswith("lib"):
+ pre = source[0:4]
+
+ # packages with '1:' versions have different paths
+ # update-manager at version 1:0.134.11.
+ # is main/u/update-manager/update-manager_0.134.11/
+ # rather than main/u/update-manager/update-manager_1:0.134.11
+ url_version = version
+ if num_colon_m.match(version):
+ url_version = version[2:]
+
+ # Changelog URL example http://changelogs.ubuntu.com/changelogs/\
+ # pool/main/h/hal/hal_0.5.5.1-1ubuntu2/changelog
+ changelog_url = "http://changelogs.ubuntu.com/changelogs/pool/" \
+ "%s/%s/%s/%s_%s/changelog" % \
+ (pile, pre, source, source, url_version)
+
+ changelog = requests.get(changelog_url)
+ if changelog.status_code == 200:
+ cache_tmp.write(changelog.content)
+ cache_tmp.close()
+ shutil.copy2(cache_tmp_path, cache_f)
+ os.unlink(cache_tmp_path)
+ return cache_f
+ else:
+ logging.error("missing %s: %s", source, changelog_url)
+ furls.append(changelog_url)
+ if os.path.exists(cache_tmp_path):
+ os.unlink(cache_tmp_path)
+ raise MissingChangelogError("Failed to find changelog for %s at version "
+ "%s.\n tried %s" % (source, version,
+ ' '.join(furls)))
+
+ def get_pkg_versions(self, binary):
+ "Get all known versions from the apt cache"
+ if not self._apt_cache:
+ self.open()
+
+ pkg_name = binary.split(':')[0]
+ try:
+ return self._apt_cache[pkg_name].versions
+ except KeyError:
+ raise Exception(
+ "%s not in cache or did not have version info in cache" %
+ pkg_name)
+
+ def source_version_for_binary(self, binary, binary_ver):
+ "Find the source version data for a specific binary version"
+ if not self._apt_cache:
+ self.open()
+
+ versions = self.get_pkg_versions(binary)
+ try:
+ return versions[binary_ver].source_version
+ except KeyError:
+ # Strip the architecture name from the binary
+ source_name = self._apt_cache[binary.split(':')[0]].versions[0]\
+ .source_name
+ msg = ("Unable to determine source version for %s. "
+ "Binary package %s/%s not in known source version "
+ "list (%s)" % (source_name, binary, binary_ver, versions))
+ raise UnknownSourceVersionError(msg)
+
+ def open(self):
+ """
+ Create, update, and open an apt cache.
+
+ This creates an apt cache directory and write a sources.list file
+ before updating and opening the cache. The caller is responsible
+ for closing the cache.
+
+ :returns: tuple of Open/updated apt cache and cache path name
+ :rtype: tuple(:class:`apt.Cache`, str)
+ """
+
+ if self._arch in ['amd64', 'i386']:
+ mirror = "http://archive.ubuntu.com/ubuntu/"
+ else:
+ mirror = "http://ports.ubuntu.com/ubuntu-ports/"
+ logging.debug('Configuring apt cache using mirror %s', mirror)
+
+ pockets = (
+ self._release,
+ "%s-updates" % self._release,
+ "%s-security" % self._release,
+ "%s-proposed" % self._release,
+ )
+
+ components = ("main", "universe")
+ srclines = []
+ for pocket in pockets:
+ srcline = "deb %s %s %s" % (mirror, pocket, ' '.join(components))
+ logging.debug('Adding source: %s', srcline)
+ srclines.append(srcline)
+ try:
+ os.makedirs("%s/etc/apt" % self._cache_d)
+ except OSError as oserror:
+ if os.errno.EEXIST != oserror.errno:
+ raise
+ with open("%s/etc/apt/sources.list" % self._cache_d, "w") as asl:
+ asl.write('\n'.join(srclines))
+
+ apt.apt_pkg.config.set("Apt::Architecture", self._arch)
+
+ logging.debug('Using host apt keys for signature verification')
+ apt.apt_pkg.config.set("Dir::Etc::Trusted", "/etc/apt/trusted.gpg")
+ apt.apt_pkg.config.set("Dir::Etc::TrustedParts",
+ "/etc/apt/trusted.gpg.d/")
+
+ self._apt_cache = apt.Cache(rootdir=self._cache_d)
+ logging.info('Updating apt cache')
+ self._apt_cache.update()
+ logging.info('Update of apt cache complete')
+ self._apt_cache.open()
+
+ def close(self):
+ "Close the cache."
+ if self._apt_cache:
+ self._apt_cache.close()
+ self._apt_cache = None
diff --git a/ubuntu/cloudimage/mfdiff/manifestdiff.py b/ubuntu/cloudimage/mfdiff/manifestdiff.py
new file mode 100644
index 0000000..20e7804
--- /dev/null
+++ b/ubuntu/cloudimage/mfdiff/manifestdiff.py
@@ -0,0 +1,284 @@
+# vi:ts=4 expandtab
+#
+# Copyright (C) 2010, 2017, 2018 Canonical Ltd.
+#
+# Authors: Scott Moser <scott.moser@canonical.com>
+# Robert C Jennings <robert.jennings@canonical.com>
+# Tobias Koch <tobias.koch@canonical.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, version 3 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+import logging
+import re
+
+from debian.changelog import Changelog
+from six import PY3, iteritems
+
+try:
+ from apt import VersionCompare as version_compare
+except ImportError:
+ from apt_pkg import version_compare as version_compare
+
+try:
+ from six import viewkeys
+except ImportError:
+ # The version of six in trusty doesn't have viewkeys, so pull in the six
+ # code from a more recent version if we can't get it directly from six.
+ import operator
+ if PY3:
+ viewkeys = operator.methodcaller("keys")
+ else:
+ viewkeys = operator.methodcaller("viewkeys")
+
+from .error import (MissingChangelogError, UnknownSourceVersionError,
+ ChangelogMissingVersion, IncompatibleManifestError)
+from .manifestcache import ManifestCache
+
+
+class ManifestDiff(object):
+ """A convenience class representing a diff between to Manifest objects."""
+
+ def __init__(self, manifest_from, manifest_to, cache_d=None,
+ apply_fixups=True):
+ """
+ :param dict manifest_from: A Manifest object representing the
+ first manifest file
+ :param dict manifest_to: A Manifest object representing the
+ second manifest file
+ :param str cache_d: The path to the cache directory.
+ """
+ if (manifest_from.release != manifest_to.release or
+ manifest_from.arch != manifest_to.arch):
+ raise IncompatibleManifestError("The manifests do not belong to "
+ "the same release and architecture.")
+
+ self._manifest_from = manifest_from
+ self._manifest_to = manifest_to
+ self._added_pkgs = {}
+ self._removed_pkgs = {}
+ self._changed_pkgs = []
+
+ self._cache = ManifestCache(manifest_from.release,
+ manifest_from.arch)
+
+ if apply_fixups:
+ self.apply_kernel_fixups()
+
+ def get_added(self):
+ "Find new packages in manifest_to"
+ if not self._added_pkgs:
+ for pkg in sorted(viewkeys(self._manifest_to.dict) -
+ viewkeys(self._manifest_from.dict)):
+ logging.debug('New package: %s', pkg)
+ self._added_pkgs[pkg] = self._manifest_to.dict[pkg]
+ return self._added_pkgs
+
+ def get_removed(self):
+ "Find packages removed from manifest_from"
+ if not self._removed_pkgs:
+ for pkg in sorted(viewkeys(self._manifest_from.dict) -
+ viewkeys(self._manifest_to.dict)):
+ logging.debug('Removed package: %s', pkg)
+ self._removed_pkgs[pkg] = self._manifest_from[pkg]
+ return self._removed_pkgs
+
+ def get_changed(self):
+ "Find modified packages"
+ if not self._changed_pkgs:
+ changed = []
+ for pkg in sorted(viewkeys(self._manifest_from.dict) &
+ viewkeys(self._manifest_to.dict)):
+ if self._manifest_from[pkg] != self._manifest_to[pkg]:
+ logging.debug('Changed package: %s', pkg)
+ changed.append(pkg)
+
+ self._changed_pkgs = changed
+
+ return self._changed_pkgs
+
+ def render_changelog(self):
+ """
+ Return a string of changelog entries for each changed package limited
+ to changes in the package between the versions in the two manifests.
+ """
+
+ srcs = {}
+ errors = []
+ src2bins = self._cache.get_src2bin_mapping(self._changed_pkgs)
+
+ # Generate changelog data per unique source package
+ for source_name in src2bins:
+ srcs[source_name] = {"changelog_file": "", "changeblocks": []}
+ src = srcs[source_name]
+
+ # Use the first binary listed for a source package
+ binary_name = src2bins[source_name][0]
+
+ # Find the source version data for the binary in manifest #2
+ try:
+ src['version_to'] = self._cache.source_version_for_binary(
+ binary_name, self._manifest_to[binary_name])
+ except UnknownSourceVersionError as excp:
+ logging.error(str(excp))
+ errors.append(excp)
+ continue
+
+ # Find the source version data for the binary in manifest #1
+ binver_from = self._manifest_from[binary_name]
+ try:
+ src['version_from'] = self._cache.source_version_for_binary(
+ binary_name, binver_from)
+ except UnknownSourceVersionError as excp:
+ if self._manifest_to[binary_name] == src['version_to']:
+ logging.info('Could not find source version data in apt '
+ 'cache. Assuming source %s version %s from '
+ 'binary %s', source_name, binver_from,
+ binary_name)
+ src['version_from'] = binver_from
+ else:
+ logging.error(str(excp))
+ errors.append(excp)
+ continue
+
+ # Check for version regression between manifests
+ try:
+ if version_compare(src['version_from'], src['version_to']) > 0:
+ msg = "Package version regression {} -> {}".format(
+ src['version_from'], src['version_to'])
+ raise UnknownSourceVersionError(msg)
+ except UnknownSourceVersionError as excp:
+ errors.append(excp)
+ continue
+
+ # Get the changelog for this source package
+ try:
+ # Use the apt cache directory to store the changelog cache
+ srcs[source_name]["changelog_file"] = \
+ self._cache.get_changelog(source_name, src['version_to'])
+ except MissingChangelogError as excp:
+ errors.append(excp)
+ continue
+
+ # Filter the changelog to a list of blocks between the two versions
+ try:
+ srcs[source_name]["changeblocks"], incomplete = \
+ self._filter_changelog(
+ srcs[source_name]["changelog_file"], src['version_from'],
+ src['version_to']
+ )
+ if incomplete:
+ raise ChangelogMissingVersion(incomplete)
+ except ChangelogMissingVersion as exp:
+ errors.append(exp)
+ continue
+
+ result = []
+
+ # Print changelog ranges for changed packages
+ for source_name in sorted(src2bins):
+ binlist = sorted(src2bins[source_name])
+ binary = binlist[0]
+ result.append("==== %s: %s => %s ====" %
+ (source_name, self._manifest_from[binary],
+ self._manifest_to[binary]))
+ result.append("==== %s" % ' '.join(binlist))
+ for block in srcs[source_name]["changeblocks"]:
+ entry = '\n'.join([x for x in block.changes() if x])
+ # Are en-/decoding acrobatics for detecting encoding errors?
+ result.append(entry.encode('utf-8').decode('utf-8'))
+
+ return '\n'.join(result)
+
+ def apply_kernel_fixups(self):
+ """
+ Fix up kernels so the pkg names match
+
+ Kernel package names change from release to release so that they are
+ co-installable, but we need to find matching package names in the
+ two manifests to provide a list of changes between the versions of the
+ package in each manifest.
+ This function will return an altered version of manifest_from with kernel
+ package names changed to match kernel package names in manifest_to. This
+ will support later version comparisons.
+ """
+ kfixups = {}
+ kmatch = re.compile("linux-image-[0-9]")
+ for pkg in self._manifest_to:
+ # if this is a linux-image-* binary package do some hacks to make it
+ # look like manifest_from is the same (format like
+ # linux-image-2.6.32-32-virtual)
+ if kmatch.match(pkg):
+ logging.debug('Found kernel %s in manifest #2', pkg)
+ img_type = pkg.split("-")[-1]
+ if pkg in self._manifest_from:
+ logging.debug('Found same kernel in manifest #1')
+ continue
+ for fpkg in self._manifest_from:
+ if kmatch.match(fpkg) and fpkg.endswith("-%s" % img_type):
+ logging.debug('Found similar kernel %s in manifest #1',
+ fpkg)
+ kfixups[pkg] = fpkg
+
+ for pkg_to, pkg_from in iteritems(kfixups):
+ logging.debug('Substituting kernel %s for %s in manifest #1 to '
+ 'enable version comparison', pkg_to, pkg_from)
+ self._manifest_from[pkg_to] = self._manifest_from[pkg_from]
+ del self._manifest_from[pkg_from]
+
+ def _filter_changelog(self, changelog_path, version_low, version_high):
+ """
+ Extract changelog entries within a version range
+
+ The range of changelog entries returned will include all entries
+ after version_low up to, and including, version_high.
+ If either the starting or ending version are not found in the
+ list of changelog entries the result will be incomplete and
+ a non-empty error message is returned to indicate the issue.
+
+ :param str changelog_path: File name of the changelog to process
+ :return: list of changelog blocks and an error_msg if incomplete
+ :rtype tuple(list, str):
+ """
+
+ with open(changelog_path, "r") as fileptr:
+ chlog = Changelog(fileptr.read())
+ change_blocks = []
+ start = False
+ end = False
+ error_msg = ''
+
+ # The changelog blocks are in reverse order; we'll see high before low.
+ for block in chlog:
+ if block.version == version_high:
+ start = True
+ change_blocks = []
+ if block.version == version_low:
+ end = True
+ break
+ change_blocks.append(block)
+ if not start:
+ error_msg = "Missing starting version {} in {}. " \
+ "Changelog will be incomplete".format(
+ version_high, changelog_path)
+ logging.error(error_msg)
+ if not end:
+ if error_msg:
+ # Start and end were not found, put a newline between their
+ # error messages
+ error_msg += '\n'
+ error_msg += "Missing ending version {} in {}. " \
+ "Changelog output truncated".format(
+ version_low, changelog_path)
+ logging.error(error_msg)
+ return change_blocks, error_msg
+