Additional refactoring to reduce size of main

bzr-revno: 18.2.11

1 files changed, 176 insertions, 113 deletions

diff --git a/mfdiff b/mfdiff
index 3d78aed..5c818c2 100755
--- a/mfdiff
+++ b/mfdiff
@@ -148,14 +148,21 @@ def filecontents(filename):
 return fileptr.read()
 
 
-def prep_cacheroot(cache_d, release):
+def prep_cacheroot(arch, release, cache_d=None):
 """
 Create the apt cache directory and write a sources.list file
 
- :param str cache_d: apt cache path
+ :param str arch: Package architecture
 :param str release: Ubuntu release name (e.g. Xenial)
+ :param str cache_d: apt cache path
+ :returns: path name for apt cache, defaults to ./cache-{release}-{arch}/
+ :rtype: str
 """
 
+ if not cache_d:
+ cache_d = "./cache.%s-%s" % (release, arch)
+ logging.info("Using %s as the apt cache directory", cache_d)
+
 mirror = "http://archive.ubuntu.com/ubuntu/"
 logging.debug('Configuring apt cache using mirror %s', mirror)
 
@@ -175,6 +182,15 @@ def prep_cacheroot(cache_d, release):
 with open("%s/etc/apt/sources.list" % cache_d, "w") as asl:
 asl.write('\n'.join(srclines))
 
+ apt.apt_pkg.config.set("Apt::Architecture", arch)
+
+ logging.debug('Using host apt keys for signature verification')
+ apt.apt_pkg.config.set("Dir::Etc::Trusted", "/etc/apt/trusted.gpg")
+ apt.apt_pkg.config.set("Dir::Etc::TrustedParts",
+ "/etc/apt/trusted.gpg.d/")
+
+ return cache_d
+
 
 def get_cache(cache_d):
 """
@@ -205,43 +221,7 @@ def print_blocks(blist):
 print(render_block(block))
 
 
-def main():
- parser = OptionParser(usage="Usage: {} suite arch manifest1 manifest2\n"
- "Compare two manifest files, and show "
- "changelog differences."
- .format(os.path.basename(sys.argv[0])))
- parser.add_option("--cache-dir", dest="cache_d",
- help="cache dir for info", metavar="DIR", type="string",
- default=None)
- parser.add_option("-v", "--verbose", action="count", dest="loglevel",
- help="increase verbosity", default=0)
-
- (options, args) = parser.parse_args()
-
- if len(args) != 4:
- parser.error('you must provide arch, release, and 2 manifest files')
-
- # By default, log WARNING and higher messages
- loglevel = [logging.WARNING,
- logging.INFO,
- logging.DEBUG][min(2, options.loglevel)]
-
- logging.basicConfig(
- level=loglevel,
- format="%(asctime)s %(name)s/%(levelname)s: %(message)s",
- stream=sys.stderr)
-
- (arch, release, mf_from, mf_to) = args
-
- cache_d = options.cache_d
- if not cache_d:
- cache_d = "./cache.%s-%s" % (release, arch)
- logging.info("Using %s as the apt cache directory", cache_d)
-
- # index both manifests
- h_from = hashmffile(mf_from)
- h_to = hashmffile(mf_to)
-
+def kernel_fixups(h_from, h_to):
 # fix up kernels so the pkg names match
 kfixups = {}
 kmatch = re.compile("linux-image-[0-9]")
@@ -266,132 +246,215 @@ def main():
 'enable version comparison', pkg_to, pkg_from)
 h_from[pkg_to] = h_from[pkg_from]
 del h_from[pkg_from]
+ return h_from
+
 
+def find_added(h_from, h_to):
 # find new packages in mf2
 new = {}
 for pkg in sorted(viewkeys(h_to) - viewkeys(h_from)):
 logging.debug('New package: %s', pkg)
 new[pkg] = h_to[pkg]
+ return new
+
 
+def find_removed(h_from, h_to):
 # find packages removed from mf1
 removed = {}
 for pkg in sorted(viewkeys(h_from) - viewkeys(h_to)):
 logging.debug('Removed package: %s', pkg)
 removed[pkg] = h_from[pkg]
+ return removed
 
+
+def find_changed(h_from, h_to):
 # find modified packages
 changed = []
 for pkg in sorted(viewkeys(h_from) & viewkeys(h_to)):
 if h_from[pkg] != h_to[pkg]:
 logging.debug('Changed package: %s', pkg)
 changed.append(pkg)
+ return changed
 
- print("new: %s" % new)
- print("removed: %s" % removed)
- print("changed: %s" % changed)
 
- prep_cacheroot(cache_d, release)
- apt.apt_pkg.config.set("Apt::Architecture", arch)
+def map_source_to_binary(cache, packages):
+ # Create a dictionary of source to list of binary packages
+ src2bins = {}
+ for bin_pkg in packages:
+ bin_name = bin_pkg.split(':')[0]
+ src2bins.setdefault(
+ cache[bin_name].versions[0].source_name, []).append(bin_pkg)
+ return src2bins
 
- logging.debug('Using host apt keys for signature verification')
- apt.apt_pkg.config.set("Dir::Etc::Trusted", "/etc/apt/trusted.gpg")
- apt.apt_pkg.config.set("Dir::Etc::TrustedParts",
- "/etc/apt/trusted.gpg.d/")
 
- bin2src = {}
- srcs = {}
+def get_pkg_versions(cache, binary):
+ # Get all known versions from the apt cache
+ pkg_name = binary.split(':')[0]
+ try:
+ return cache[pkg_name].versions
+ except KeyError:
+ raise Exception(
+ "%s not in cache or did not have version info in cache" %
+ pkg_name)
+
+
+def source_version_for_binary(cache, binary, binary_ver):
+ # Find the source version data for a specific binary version
+ versions = get_pkg_versions(cache, binary)
+ try:
+ return versions[binary_ver].source_version
+ except KeyError:
+ source_name = cache[binary].versions[0].source_name
+ msg = ("Unable to determine source version for %s. "
+ "Binary package %s/%s not in known source version "
+ "list (%s)" % (source_name, binary, binary_ver, versions))
+ raise UnknownSourceVersionError(msg)
+
+
+def filter_changelog(changelog_path, version_start, version_end):
+ # Filter changelog contents for version range
+ chlog = Changelog(filecontents(changelog_path))
+ change_blocks = []
+ start = False
+ end = False
+ error_msg = ''
+ for block in chlog:
+ if block.version == version_end:
+ start = True
+ change_blocks = []
+ if block.version == version_start:
+ end = True
+ break
+ change_blocks.append(block)
+ if not start:
+ error_msg = "Missing starting version {} in {}. " \
+ "Changlelog will be incomplete".format(
+ version_start, changelog_path)
+ logging.error(error_msg)
+ if not end:
+ error_msg = "Missing ending version {} in {}. " \
+ "Changelog output truncated".format(
+ version_end, changelog_path)
+ logging.error(error_msg)
+ return change_blocks, error_msg
+
+
+def parse_args():
+ parser = OptionParser(usage="Usage: {} suite arch manifest1 manifest2\n"
+ "Compare two manifest files, and show "
+ "changelog differences."
+ .format(os.path.basename(sys.argv[0])))
+ parser.add_option("--cache-dir", dest="cache_d",
+ help="cache dir for info", metavar="DIR", type="string",
+ default=None)
+ parser.add_option("-v", "--verbose", action="count", dest="loglevel",
+ help="increase verbosity", default=0)
+
+ (options, args) = parser.parse_args()
+
+ if len(args) != 4:
+ parser.error('you must provide arch, release, and 2 manifest files')
+
+ return options, args
+
+
+def setup_logging(loglevel):
+ # By default, log WARNING and higher messages
+ loglevel = [logging.WARNING,
+ logging.INFO,
+ logging.DEBUG][min(2, loglevel)]
+
+ logging.basicConfig(
+ level=loglevel,
+ format="%(asctime)s %(name)s/%(levelname)s: %(message)s",
+ stream=sys.stderr)
+
+
+def main():
+ options, (arch, release, mf_from, mf_to) = parse_args()
+ setup_logging(options.loglevel)
+
+ # index both manifests
+ h_to = hashmffile(mf_to)
+ h_from = kernel_fixups(hashmffile(mf_from), h_to)
+
+ new = find_added(h_from, h_to)
+ removed = find_removed(h_from, h_to)
+ changed = find_changed(h_from, h_to)
+
+ cache_d = prep_cacheroot(arch, release, options.cache_d)
+
 # if modified packages, download all changelogs from changelogs.
- if len(changed):
- exceptions = []
+ srcs = {}
+ exceptions = []
+ if changed:
 cache = get_cache(cache_d)
 
- b2s_hash = get_bin2src(changed, cache)
-
- # Create a dictionary of source to list of binary packages
- for pkg in b2s_hash:
- bin2src.setdefault(b2s_hash[pkg], []).append(pkg)
+ src2bins = map_source_to_binary(cache, changed)
 
+ # XXX RCJ Factor this out of main
 # Generate changelog data per unique source package
- for src in bin2src:
- if src in srcs:
- continue
-
+ for src in src2bins:
 srcs[src] = {"changelog_file": "", "changeblocks": []}
 
- # Using the first binary listed for a source package, get
- # all known versions from the apt cache
- binary = bin2src[src][0]
- try:
- pkg_name = binary.split(':')[0]
- versions = cache[pkg_name].versions
- except KeyError:
- raise Exception(
- "%s not in cache or did not have version info in cache" %
- pkg_name)
+ # Use the first binary listed for a source package
+ binary = src2bins[src][0]
 
 # Find the source version data for the binary in manifest #2
 binver_to = h_to[binary]
 try:
- srcver_to = versions[binver_to].source_version
- except KeyError:
- msg = ("Unable to determine source version for %s. "
- "Binary package %s/%s not in known source version "
- "list (%s)" % (src, binary, binver_to, versions))
- logging.error(msg)
- raise UnknownSourceVersionError(msg)
+ srcver_to = source_version_for_binary(
+ cache, binary, binver_to)
+ except UnknownSourceVersionError as excp:
+ logging.error(excp.message)
+ exceptions.append(excp)
 
 # Find the source version data for the binary in manifest #1
 binver_from = h_from[binary]
 try:
- srcver_from = versions[binver_from].source_version
- except KeyError:
+ srcver_from = source_version_for_binary(
+ cache, binary, binver_from)
+ except UnknownSourceVersionError as excp:
 if binver_to == srcver_to:
 logging.info('Could not find source version data in apt '
 'cache. Assuming source %s version %s from '
 'binary %s', src, binver_from, binary)
 srcver_from = binver_from
 else:
- msg = ("Unable to determine source version for %s. "
- "Binary package %s/%s not in known source version "
- "list (%s)" % (src, binary, binver_from, versions))
- logging.error(msg)
+ logging.error(excp.message)
+ exceptions.append(excp)
+
+ try:
+ if version_compare(srcver_from, srcver_to) > 0:
+ msg = "Package version regression {} -> {}".format(
+ srcver_from, srcver_to)
 raise UnknownSourceVersionError(msg)
+ except UnknownSourceVersionError as excp:
+ exceptions.append(excp)
 
 try:
 changelog_path = getchangelog(src, srcver_to, cache_d)
- srcs[src]["changelog_file"] = changelog_path
-
- # Filter changelog contents for version range
- chlog = Changelog(filecontents(changelog_path))
- srcs[src]["changeblocks"] = []
- start = False
- end = False
- for block in chlog:
- if block.version == srcver_to:
- start = True
- if block.version == srcver_from:
- end = True
- break
- if start:
- srcs[src]["changeblocks"].append(block)
- if not (start and end):
- msg = ("No changelog output for %s. %s missing %s or %s." %
- (src, changelog_path, srcver_to, binver_from))
- if version_compare(binver_from, srcver_to) > 0:
- msg = "%s %s" % (msg, "from version > to version")
- logging.error(msg)
- raise ChangelogMissingVersion(msg)
-
- except MissingChangelogError as exp:
- exceptions.append(exp)
+ except MissingChangelogError as excp:
+ exceptions.append(excp)
+
+ srcs[src]["changelog_file"] = changelog_path
+
+ try:
+ srcs[src]["changeblocks"], incomplete = filter_changelog(
+ changelog_path, srcver_from, srcver_to)
+ if incomplete:
+ raise(ChangelogMissingVersion(incomplete))
 except ChangelogMissingVersion as exp:
 exceptions.append(exp)
- except UnknownSourceVersionError as exp:
- exceptions.append(exp)
 
+ print("new: %s" % new)
+ print("removed: %s" % removed)
+ print("changed: %s" % changed)
+
+ if changed:
 # Print changelog ranges for changed packages
- for src in sorted(bin2src):
- binlist = sorted(bin2src[src])
+ for src in sorted(src2bins):
+ binlist = sorted(src2bins[src])
 binary = binlist[0]
 print("==== %s: %s => %s ====" %
 (src, h_from[binary], h_to[binary]))