diff options
| -rwxr-xr-x | bin/boot_mode_test | 35 | ||||
| -rwxr-xr-x | bin/disk_cpu_load | 21 | ||||
| -rwxr-xr-x | bin/disk_stress_ng | 43 | ||||
| -rwxr-xr-x | bin/dmitest | 39 | ||||
| -rwxr-xr-x | bin/efi-pxeboot | 149 | ||||
| -rwxr-xr-x | bin/fwts_test | 82 | ||||
| -rwxr-xr-x | bin/maas-version-check | 33 | ||||
| -rwxr-xr-x | bin/memory_stress_ng | 7 | ||||
| -rwxr-xr-x | bin/network | 27 | ||||
| -rwxr-xr-x | bin/virtualization | 127 | ||||
| -rw-r--r-- | jobs/miscellanea.txt.in | 40 | ||||
| -rwxr-xr-x | manage.py | 2 |
12 files changed, 467 insertions, 138 deletions
diff --git a/bin/boot_mode_test b/bin/boot_mode_test index d4c2956..330bb2b 100755 --- a/bin/boot_mode_test +++ b/bin/boot_mode_test @@ -24,6 +24,35 @@ import os import sys import logging from argparse import ArgumentParser +from platform import linux_distribution + + +def version_check(check): + """Check the installed Ubuntu version to see if it is older than 16.04 + (Xenial) and run the requested check + + :returns: + 0 if if the installed version is older than 16.04 regardless of return + code from the requested check + + return code from requested check if the installed version is 16.04 or + newer. + """ + installed_version = int(linux_distribution()[1].split()[0]) + if installed_version < 16: + logging.info("This system appears to be older than 16.04 LTS so this " + "will not block a certification in progress.") + if check == 'efi': + efi_boot_check() + else: + secure_boot_check() + + return 0 + else: + if check == 'efi': + return efi_boot_check() + else: + return secure_boot_check() def efi_boot_check(): @@ -86,11 +115,7 @@ def main(): FORMAT = '%(levelname)s: %(message)s' logging.basicConfig(level=logging.INFO, format=FORMAT) - if args.check == 'efi': - return efi_boot_check() - else: - return secure_boot_check() - + return version_check(args.check) if __name__ == '__main__': sys.exit(main()) diff --git a/bin/disk_cpu_load b/bin/disk_cpu_load index 5e0d0eb..93b1569 100755 --- a/bin/disk_cpu_load +++ b/bin/disk_cpu_load @@ -24,13 +24,14 @@ # # Usage: # disk_cpu_load [ --max-load <load> ] [ --xfer <mebibytes> ] -# [ <device-filename> ] +# [ --verbose ] [ <device-filename> ] # # Parameters: # --max-load <load> -- The maximum acceptable CPU load, as a percentage. # Defaults to 30. # --xfer <mebibytes> -- The amount of data to read from the disk, in # mebibytes. Defaults to 4096 (4 GiB). +# --verbose -- If present, produce more verbose output # <device-filename> -- This is the WHOLE-DISK device filename (with or # without "/dev/"), e.g. "sda" or "/dev/sda". The # script finds a filesystem on that device, mounts @@ -44,6 +45,7 @@ set -e get_params() { disk_device="/dev/sda" short_device="sda" + verbose=0 max_load=30 xfer=4096 while [ $# -gt 0 ] ; do @@ -54,6 +56,8 @@ get_params() { --xfer) xfer="$2" shift ;; + --verbose) verbose=1 + ;; *) disk_device="/dev/$1" disk_device=`echo $disk_device | sed "s/\/dev\/\/dev/\/dev/g"` short_device=$(echo $disk_device | sed "s/\/dev//g") @@ -109,8 +113,15 @@ compute_cpu_load() { let diff_total=${end_total}-${start_total} let diff_used=$diff_total-$diff_idle + if [ "$verbose" == "1" ] ; then + echo "Start CPU time = $start_total" + echo "End CPU time = $end_total" + echo "CPU time used = $diff_used" + echo "Total elapsed time = $diff_total" + fi + if [ "$diff_total" != "0" ] ; then - let cpu_load=($diff_used*100)/$diff_total + cpu_load=$(echo "($diff_used*100)/$diff_total" | bc) else cpu_load=0 fi @@ -127,7 +138,13 @@ echo "Testing CPU load when reading $xfer MiB from $disk_device" echo "Maximum acceptable CPU load is $max_load" blockdev --flushbufs $disk_device start_load="$(grep "cpu " /proc/stat | tr -s " " | cut -d " " -f 2-)" +if [ "$verbose" == "1" ] ; then + echo "Beginning disk read...." +fi dd if="$disk_device" of=/dev/null bs=1048576 count="$xfer" &> /dev/null +if [ "$verbose" == "1" ] ; then + echo "Disk read complete!" +fi end_load="$(grep "cpu " /proc/stat | tr -s " " | cut -d " " -f 2-)" compute_cpu_load "$start_load" "$end_load" echo "Detected disk read CPU load is $cpu_load" diff --git a/bin/disk_stress_ng b/bin/disk_stress_ng index 8445488..802d116 100755 --- a/bin/disk_stress_ng +++ b/bin/disk_stress_ng @@ -63,11 +63,40 @@ get_params() { } # get_params() -# Find the largest partition that holds a supported filesystem on $disk_device. +# Find the largest logical volume in an LVM partition. # Output: # $largest_part -- Device filename of largest qualifying partition # $largest_size -- Size of largest qualifying partition # $largest_fs -- Filesystem (ext4, etc.) used on largest qualifying partition +# Note: Above variables are initialized in find_largest_partition(), which +# calls this function. +# Caveat: If LVM is used, there can be no guarantee that a specific disk +# device is actually being tested. Thus, an LVM configuration should span +# just one disk device. LVM may be used on one disk, but subsequent disks +# should use "raw" partitions. +find_largest_lv() { + local partonly=$(echo $partition | cut -f 3 -d "/") + for syslv in $(ls -d /sys/block/dm-*/slaves/$partonly) ; do + lv=$(echo "$syslv" | cut -f 4 -d "/") + size=$(cat /sys/block/$lv/size) + sector_size=$(cat /sys/block/$lv/queue/hw_sector_size) + let size=$size*$sector_size + local blkid_info=$(blkid -s TYPE /dev/$lv | grep -E ext2\|ext3\|ext4\|xfs\|jfs\|btrfs) + if [ "$size" -gt "$largest_size" ] && [ -n "$blkid_info" ] ; then + local blkid_info=$(blkid -s TYPE /dev/$lv) + largest_size=$size + largest_part="/dev/$lv" + largest_fs=$(blkid -s TYPE "/dev/$lv" | cut -d "=" -f 2) + fi + done +} # find_largest_lv() + + +# Find the largest partition that holds a supported filesystem on $disk_device. +# Output: +# $largest_part -- Device filename of largest qualifying partition or logical volume +# $largest_size -- Size of largest qualifying partition or logical volume +# $largest_fs -- Filesystem (ext4, etc.) used on largest qualifying partition or logicl volume # $unsupported_fs -- Empty or contains name of unsupported filesystem found on disk find_largest_partition() { largest_part="" @@ -76,11 +105,15 @@ find_largest_partition() { unsupported_fs="" for partition in $(echo "$partitions" | cut -d " " -f 1) ; do part_size=$(echo "$partitions" | grep "$partition " | cut -d " " -f 2) - local blkid_info=$(blkid -s TYPE /dev/$partition | grep -E ext2\|ext3\|ext4\|xfs\|jfs\|btrfs) + local blkid_info=$(blkid -s TYPE /dev/$partition | grep -E ext2\|ext3\|ext4\|xfs\|jfs\|btrfs\|LVM2_member) if [ "$part_size" -gt "$largest_size" ] && [ -n "$blkid_info" ] ; then - largest_size=$part_size - largest_part="/dev/$partition" - largest_fs=$(blkid -s TYPE "/dev/$partition" | cut -d "=" -f 2) + if [[ "$blkid_info" =~ .*LVM2_member.* ]] ; then + find_largest_lv + else + largest_size=$part_size + largest_part="/dev/$partition" + largest_fs=$(blkid -s TYPE "/dev/$partition" | cut -d "=" -f 2) + fi fi local blkid_info=$(blkid -s TYPE /dev/$partition | grep -E ntfs\|vfat\|hfs) if [ -n "$blkid_info" ] ; then diff --git a/bin/dmitest b/bin/dmitest index 27cfc8e..9be70be 100755 --- a/bin/dmitest +++ b/bin/dmitest @@ -28,6 +28,8 @@ :param --dmifile: Input filename; optional. If specified, file is used instead of dmidecode output. +:param --show_dmi: + Print the DMI data used. For debugging purposes if errors are encountered. :param --test_versions: Include chassis, system, and base boad version numbers among tests. :param --test_serials: @@ -228,26 +230,45 @@ def main(): choices=['server', 'desktop', 'cpu-check']) parser.add_argument('--dmifile', help="File to use in lieu of dmidecode.") + parser.add_argument('--show_dmi', action="store_true", + help="Print DMI Data used for debugging purposes.") parser.add_argument('--test_versions', action="store_true", help="Set to check version information") parser.add_argument('--test_serials', action="store_true", help="Set to check serial number information") args = parser.parse_args() + bad_data = False # Command to retrieve DMI information COMMAND = "dmidecode" + if args.dmifile: + COMMAND = ['cat', args.dmifile] + print("Reading " + args.dmifile + " as DMI data") try: - if args.dmifile: - print("Reading " + args.dmifile + " as DMI data") - stream = subprocess.check_output(['cat', args.dmifile], - universal_newlines=True).splitlines() - else: - stream = subprocess.check_output(COMMAND, - universal_newlines=True).splitlines() + dmi_out = subprocess.check_output(COMMAND).splitlines() except subprocess.CalledProcessError as err: print("Error running {}: {}".format(COMMAND, err)) return 1 + # Convert the bytes output separately, line by line, because it's possible + # that someone put non-encodable characters in DMI, which cases a + # UnicodeDecodeError that is non-helpful. LP: 1655155 + stream = [] + for line in dmi_out: + try: + stream.append(line.decode('utf-8')) + except UnicodeDecodeError as ude: + print("DATA ERROR: {}".format(ude)) + print("\tLINE NUMBER {}: {}".format(dmi_out.index(line) + 1, line)) + stream.append("ERROR: BAD DATA FOUND HERE") + bad_data = True + + if args.show_dmi: + print("===== DMI Data Used: =====") + for line in stream: + print(line) + print("===== DMI Output Complete =====") + retval = 0 if args.test_type == 'server' or args.test_type == 'desktop': retval += standard_tests(args, stream) @@ -272,6 +293,10 @@ def main(): else: print("\nPassed all tests") + if bad_data: + print("\nBad Characters discovered in DMI output. Rerun with " + "the --show_dmi option to see more") + return retval diff --git a/bin/efi-pxeboot b/bin/efi-pxeboot new file mode 100755 index 0000000..724fefb --- /dev/null +++ b/bin/efi-pxeboot @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 + +""" +Script to test that the system PXE-booted, if run in EFI mode + +Copyright (c) 2016 Canonical Ltd. + +Authors + Rod Smith <rod.smith@canonical.com> + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License version 3, +as published by the Free Software Foundation. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see <http://www.gnu.org/licenses/>. + +The purpose of this script is to identify whether an EFI-based +system booted from the network (test passes) or from a local disk +(test fails). + +Usage: + efi-pxeboot +""" + +import os +import shlex +import shutil +import sys + +from subprocess import Popen, PIPE + + +def discover_data(): + """Extract boot entry and boot order information. + + :returns: + boot_entries, boot_order, boot_current + """ + command = "efibootmgr -v" + bootinfo_bytes = (Popen(shlex.split(command), stdout=PIPE) + .communicate()[0]) + bootinfo = (bootinfo_bytes.decode(encoding="utf-8", errors="ignore") + .splitlines()) + boot_entries = {} + boot_order = [] + if len(bootinfo) > 1: + for s in bootinfo: + if "BootOrder" in s: + try: + boot_order = s.split(":")[1].replace(" ", "").split(",") + except IndexError: + pass + elif "BootCurrent" in s: + try: + boot_current = s.split(":")[1].strip() + except IndexError: + pass + else: + # On Boot#### lines, #### is characters 4-8.... + hex_value = s[4:8] + # ....and the description starts at character 10 + name = s[10:] + try: + # In normal efibootmgr output, only Boot#### entries + # have characters 4-8 that can be interpreted as + # hex values, so this will harmlessly error out on all + # but Boot#### entries.... + int(hex_value, 16) + boot_entries[hex_value] = name + except ValueError: + pass + return boot_entries, boot_order, boot_current + + +def is_pxe_booted(boot_entries, boot_order, boot_current): + retval = 0 + desc = boot_entries[boot_current] + print("The current boot item is {}".format(boot_current)) + print("The first BootOrder item is {}".format(boot_order[0])) + print("The description of Boot{} is '{}'".format(boot_current, desc)) + if boot_current != boot_order[0]: + # If the BootCurrent entry isn't the same as the first of the + # BootOrder entries, then something is causing the first boot entry + # to fail or be bypassed. This could be a Secure Boot failure, manual + # intervention, a bad boot entry, etc. This is not necessarily a + # problem, but warn of it anyhow.... + desc2 = boot_entries[boot_order[0]] + print("The description of Boot{} is '{}'".format(boot_order[0], desc2)) + print("WARNING: The system is booted using Boot{}, but the first". + format(boot_current)) + print("boot item is Boot{}!".format(boot_order[0])) + if "Network" in desc or "PXE" in desc or "NIC" in desc \ + or "Ethernet" in desc or "IP4" in desc or "IP6" in desc: + # These strings are present in network-boot descriptions. + print("The system seems to have PXE-booted; all OK.") + elif "ubuntu" in desc or "grub" in desc or "shim" in desc or "rEFInd" \ + or "refind_" in desc: + # This string indicates a boot directly from the normal Ubuntu GRUB + # or rEFInd installation on the hard disk. + print("The system seems to have booted directly from the hard disk!") + retval = 1 + elif "SATA" in desc or "Sata" in desc or "Hard Drive" in desc: + # These strings indicate booting with a "generic" disk entry (one + # that uses the fallback filename, EFI/BOOT/bootx64.efi or similar). + print("The system seems to have booted from a disk with the fallback " + "boot loader!") + retval = 1 + else: + # Probably a rare description. Call it an error so we can flag it and + # improve this script. + print("Unable to identify boot path.") + retval = 1 + return retval + + +def main(): + """Check to see if the system PXE-booted.""" + + if shutil.which("efibootmgr") is None: + print("The efibootmgr utility is not installed; exiting!") + return(4) + if not os.geteuid() == 0: + print("This program must be run as root (or via sudo); exiting!") + return(4) + + retval = 0 + boot_entries, boot_order, boot_current = discover_data() + if boot_entries == {}: + print("No EFI boot entries are available. This may indicate a " + "firmware problem.") + retval = 2 + if boot_order == []: + print("The EFI BootOrder variable is not available. This may " + "indicate a firmware") + print("problem.") + retval = 3 + if (retval == 0): + retval = is_pxe_booted(boot_entries, boot_order, boot_current) + return(retval) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/bin/fwts_test b/bin/fwts_test index 0e5ffc9..c8ff1f7 100755 --- a/bin/fwts_test +++ b/bin/fwts_test @@ -18,60 +18,35 @@ INTERACTIVE_TESTS = ['ac_adapter', 'power_button', 'brightness', 'lid'] +# Tests recommended by the Hardware Enablement Team (HWE) # These are performed on QA certification runs QA_TESTS = ['acpitests', - 'acpidump', - 'acpitables', - 'apicinstance', + 'apicedge', 'aspm', - 'bios32', + 'cpufreq', 'dmicheck', - 'ebda', - 'mpcheck', + 'esrt', + 'klog', + 'maxfreq', 'msr', + 'mtrr', 'nx', - 'version'] -# These are advanced tests that shouldn't affect certification status -NON_CERT_TESTS = ['bios_info', - 'cmosdump', - 'cpufreq', - 'crs', - 'crsdump', - 'csm', - 'ebdadump', - 'fan', - 'gpedump', - 'hda_audio', - 'maxfreq', - 'maxreadreq', - 'memmapdump', - 'microcode', - 'mpdump', - 'os2gap', - 'osilinux', - 'pciirq', - 'plddump', - 'pnp', - 'prsdump', - 'romdump', - 'securebootcert', - 'syntaxcheck', - 'uefidump', - 'uefirtmisc', - 'uefirttime', - 'uefirtvariable', - 'uefivarinfo', - 'wakealarm' - ] + 'oops', + 'uefibootpath', + 'uefirtmisc', + 'uefirttime', + 'uefirtvariable', + 'version', + 'virt'] # The following tests will record logs in a separate file for the HWE team -HWE_TESTS = ['mtrr', +HWE_TESTS = ['version', + 'mtrr', 'virt', 'apicedge', 'klog', - 'oops', - 'uefibootpath'] -CERT_TESTS = sorted(QA_TESTS + HWE_TESTS) -TESTS = sorted(QA_TESTS + NON_CERT_TESTS + HWE_TESTS) + 'oops'] +# By default, we launch all the tests +TESTS = sorted(list(set(QA_TESTS + HWE_TESTS))) def get_sleep_times(start_marker, end_marker, sleep_time, resume_time): @@ -210,9 +185,6 @@ def main(): group.add_argument('-t', '--test', action='append', help='Name of the test to run.') - group.add_argument('-a', '--all', - action='store_true', - help='Run ALL FWTS automated tests (assumes -w and -c)') group.add_argument('-s', '--sleep', nargs=REMAINDER, action='store', @@ -236,12 +208,6 @@ def main(): group.add_argument('--list', action='store_true', help='List all tests in fwts.') - group.add_argument('--list-cert', - action='store_true', - help='List all certification tests in fwts.') - group.add_argument('--list-advanced', - action='store_true', - help='List all advanced tests in fwts.') group.add_argument('--list-hwe', action='store_true', help='List all HWE concerned tests in fwts') @@ -282,12 +248,6 @@ def main(): elif args.list: print('\n'.join(TESTS)) return 0 - elif args.list_cert: - print('\n'.join(CERT_TESTS)) - return 0 - elif args.list_advanced: - print('\n'.join(NON_CERT_TESTS)) - return 0 elif args.list_hwe: print('\n'.join(HWE_TESTS)) return 0 @@ -296,8 +256,6 @@ def main(): return 0 elif args.test: tests.extend(args.test) - elif args.all: - tests.extend(TESTS) elif args.hwe: tests.extend(HWE_TESTS) elif args.qa: @@ -335,7 +293,7 @@ def main(): args.resume_time = 3 tests.extend(args.sleep) else: - tests.extend(CERT_TESTS) + tests.extend(TESTS) # run the tests we want if args.sleep: diff --git a/bin/maas-version-check b/bin/maas-version-check new file mode 100755 index 0000000..412717d --- /dev/null +++ b/bin/maas-version-check @@ -0,0 +1,33 @@ +#!/bin/bash + +# Copyright (C) 2012-2015 Canonical Ltd. + +# Authors +# Jeff Lane <jeff@ubuntu.com> + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, +# as published by the Free Software Foundation. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +MAAS_FILE="/etc/installed-by-maas" +MIN_VERSION="2.0" + +# Is the file there? +if [ -s $MAAS_FILE ]; then + maas_version=`cat $MAAS_FILE` + echo $maas_version +else + echo "ERROR: This system does not appear to have been installed by MAAS" + exit 1 +fi + +#is the version appropriate +exit `dpkg --compare-versions $maas_version "ge" $MIN_VERSION` diff --git a/bin/memory_stress_ng b/bin/memory_stress_ng index ab65e86..e619772 100755 --- a/bin/memory_stress_ng +++ b/bin/memory_stress_ng @@ -114,6 +114,8 @@ echo "Variable run time is $variable_time seconds per stressor" had_error=0 +numa_nodes=$(numactl --hardware | grep available | head -n 1 | cut -f 2 -d " ") + # NOTE: Specify stressors in two arrays rather than rely on stress-ng's # --class memory,vm option for two reasons: # 1. We want to run some stressors (those that exhaust all memory) @@ -130,6 +132,9 @@ crt_stressors=("bsearch" "context" "hsearch" "lsearch" "matrix" \ "memcpy" "null" "pipe" "qsort" "stack" "str" "stream" \ "tsearch" "vm-rw" "wcs" "zero" "mlock" "mmapfork" "mmapmany" \ "mremap" "shm-sysv" "vm-splice") +if [ "$numa_nodes" -gt 1 ]; then + crt_stressors+=("numa") +fi crt_runtime=$((${#crt_stressors[@]}*$base_time)) # Variable-run-time stressors -- run them longer on systems with more RAM.... @@ -152,7 +157,7 @@ echo "*******************************************************************" if [ $had_error = "0" ] ; then echo "** stress-ng memory test passed!" else - echo "** stress-ng memory test failed; most recent error was $return_code" + echo "** stress-ng memory test failed; most recent error was $result" fi echo "*******************************************************************" exit $result diff --git a/bin/network b/bin/network index 7ba8ef0..ae9cd38 100755 --- a/bin/network +++ b/bin/network @@ -280,29 +280,26 @@ class Interface(socket.socket): @property def max_speed(self): - # Parse ethtool data for max speed since /sys/class/net/DEV/speed only - # reports link speed. - - # Search for things that look like 100baseSX, - # 40000baseNX, 10000baseT + # Parse mii-tool data for max speed + # search for numbers in the line starting with 'capabilities' + # return largest number as max_speed try: - ethinfo = check_output(['ethtool', self.interface], - universal_newlines=True, - stderr=STDOUT).split(' ') + info = check_output(['mii-tool', '-v', self.interface], + universal_newlines=True, + stderr=STDOUT).split('\n') except FileNotFoundError: - logging.warning('ethtool not found! Unable to get max speed') + logging.warning('mii-tool not found! Unable to get max speed') ethinfo = None except CalledProcessError as e: - logging.error('ethtool returned an error!') + logging.error('mii-tool returned an error!') logging.error(e.output) ethinfo = None finally: - expression = '(\\d+)(base)([A-Z]+)|(\d+)(Mb/s)' - regex = re.compile(expression) + regex = re.compile(r'(\d+)(base)([A-Z]+)') speeds = [0] - if ethinfo: - for i in ethinfo: - hit = regex.search(i) + for line in filter(lambda l: 'capabilities' in l, info): + for s in line.split(' '): + hit = regex.search(s) if hit: speeds.append(int(re.sub("\D", "", hit.group(0)))) return max(speeds) diff --git a/bin/virtualization b/bin/virtualization index fe59656..0ee74e9 100755 --- a/bin/virtualization +++ b/bin/virtualization @@ -8,6 +8,7 @@ Copyright (C) 2013, 2014 Canonical Ltd. Authors Jeff Marcom <jeff.marcom@canonical.com> Daniel Manrique <roadmr@ubuntu.com> + Jeff Lane <jeff@ubuntu.com> This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License version 3, @@ -30,6 +31,7 @@ import os import re import logging import lsb_release +import requests import shlex import signal from subprocess import ( @@ -110,7 +112,7 @@ QEMU_ARCH_CONFIG = { 'qemu_extra_args': [ '-enable-kvm', '-machine', 'pseries,usb=off', - '-cpu', 'POWER8', + '-cpu', 'host', ], }, 's390x': { @@ -214,48 +216,108 @@ class KVMTest(object): # Gives us the stuff needed to build the URL to download the image return self.download_image(image_path) - def construct_cloud_filename(self): + def construct_cloud_url(self, image_url=None): """ Build a URL for official Ubuntu images hosted either at cloud-images.ubuntu.com or on a maas server hosting a mirror of cloud-images.ubuntu.com """ - if self.qemu_config['cloudimg_type'] == CLOUD_IMAGE_TYPE_TAR: - cloud_iso = "%s-server-cloudimg-%s.tar.gz" % ( - self.release, self.qemu_config['cloudimg_arch']) - elif self.qemu_config['cloudimg_type'] == CLOUD_IMAGE_TYPE_DISK: - cloud_iso = "%s-server-cloudimg-%s-disk1.img" % ( - self.release, self.qemu_config['cloudimg_arch']) - else: - logging.error("Unknown cloud image type") - sys.exit(1) - return cloud_iso + def _construct_filename(alt_pattern=None): + if self.qemu_config['cloudimg_type'] == CLOUD_IMAGE_TYPE_TAR: + cloud_iso = "%s-server-cloudimg-%s.tar.gz" % ( + self.release, self.qemu_config['cloudimg_arch']) + elif alt_pattern is "modern": + # LP 1635345 - yakkety and beyond have a new naming scheme + cloud_iso = "%s-server-cloudimg-%s.img" % ( + self.release, self.qemu_config['cloudimg_arch']) + elif self.qemu_config['cloudimg_type'] == CLOUD_IMAGE_TYPE_DISK: + cloud_iso = "%s-server-cloudimg-%s-disk1.img" % ( + self.release, self.qemu_config['cloudimg_arch']) + else: + logging.error("Unknown cloud image type") + sys.exit(1) + + return cloud_iso + + def _construct_url(initial_url, cloud_iso): + return "/".join((initial_url, cloud_iso)) + + def _test_cloud_url(url): + # test our URL to make sure it's reachable + ret = requests.head(url) + if ret.status_code is not 200: + return False + else: + return True - def download_image(self, image_url=None): - """ - Downloads Cloud image for same release as host machine - """ if image_url is None: # If we have not specified a URL to get our images from, default # to ubuntu.com - cloud_url = "http://cloud-images.ubuntu.com" - cloud_iso = self.construct_cloud_filename() - full_url = "/".join(( - cloud_url, self.release, "current", cloud_iso)) + cloud_iso = _construct_filename() + initial_url = "/".join(("http://cloud-images.ubuntu.com", + self.release, "current")) + full_url = _construct_url(initial_url, cloud_iso) + # Test our URL and rebuild with alternate name + if not _test_cloud_url(full_url): + logging.warn("Cloud Image URL not valid: %s" % full_url) + logging.warn(" * This means we could not reach the remote " + "file. Retrying with a different filename " + "schema.") + cloud_iso = _construct_filename("modern") + full_url = _construct_url(initial_url, cloud_iso) + # retest one more time then exit if it still fails + if not _test_cloud_url(full_url): + logging.error("Cloud URL is not valid: %s" % + full_url) + logging.error(" * It appears that there is a problem " + "finding the expected file. Check the URL " + "noted above.") + sys.exit(1) else: url = urlparse(image_url) - if url.path.endswith('/') or url.path == '': - # If we have a relative URL (MAAS server mirror) - cloud_url = image_url - cloud_iso = self.construct_cloud_filename() - full_url = "/".join(( - cloud_url, cloud_iso)) + if ( + url.path.endswith('/') or + url.path == '' or + not url.path.endswith(".img") + ): + # If we have a relative URL (local copies of official images) + # http://192.168.0.1/ or http://192.168.0.1/images/ + cloud_iso = _construct_filename() + full_url = _construct_url(image_url.rstrip("/"), cloud_iso) + if not _test_cloud_url(full_url): + logging.warn("Cloud Image URL not valid: %s" % full_url) + logging.warn(" * This means we could not reach the remote " + "file. Retrying with a different filename " + "schema.") + cloud_iso = _construct_filename("modern") + full_url = _construct_url(image_url.rstrip("/"), cloud_iso) + if not _test_cloud_url(full_url): + logging.error("Cloud URL is not valid: %s" % + full_url) + logging.error(" * It appears that there is a problem " + "finding the expected file. Check the " + "URL noted above.") + sys.exit(1) else: # Assume anything else is an absolute URL to a remote server - cloud_iso = url.path.split('/')[-1] - cloud_url = "{}://{}".format(url.scheme, url.netloc) - full_url = image_url - logging.debug("Downloading {}, from {}".format(cloud_iso, cloud_url)) + if not _test_cloud_url(image_url): + logging.error("Cloud Image URL invalid: %s" % image_url) + logging.error(" * Check the URL and ensure it is correct") + sys.exit(1) + else: + full_url = image_url + + return full_url + + def download_image(self, image_url=None): + """ + Downloads Cloud image for same release as host machine + """ + if image_url is None: + full_url = self.construct_cloud_url() + else: + full_url = self.construct_cloud_url(image_url) + logging.debug("Acquiring cloud image from: {}".format(full_url)) # Attempt download try: @@ -394,7 +456,7 @@ final_message: CERTIFICATION BOOT COMPLETE # Download cloud image self.image = self.download_image() else: - logging.debug('Cloud image location specified: %s.' % + logging.debug('Cloud image location specified: %s' % self.image) self.image = self.url_to_path(self.image) @@ -507,6 +569,9 @@ def main(): except AttributeError: pass # avoids exception when trying to run without specifying 'kvm' + # silence normal output from requests module + logging.getLogger("requests").setLevel(logging.WARNING) + # to check if not len(sys.argv) > 1 if len(vars(args)) == 0: parser.print_help() diff --git a/jobs/miscellanea.txt.in b/jobs/miscellanea.txt.in index b5544c2..295ea95 100644 --- a/jobs/miscellanea.txt.in +++ b/jobs/miscellanea.txt.in @@ -72,6 +72,20 @@ environ: PLAINBOX_SESSION_SHARE command: fwts_test -l $PLAINBOX_SESSION_SHARE/fwts_results.log +plugin: shell +category_id: 2013.com.canonical.plainbox::miscellanea +id: miscellanea/fwupdate +estimated_duration: 1.0 +depends: miscellanea/efi_boot_mode +requires: + cpuinfo.platform in ("i386", "x86_64", "aarch64", "armhf") + package.name == 'fwupdate' +_description: + Determine if EFI firmware supports update from OS. +_summary: + Check for firmware update support. +command: fwupdate -s + plugin: attachment category_id: 2013.com.canonical.plainbox::miscellanea id: miscellanea/fwts_results.log @@ -132,6 +146,20 @@ command: boot_mode_test secureboot plugin: shell category_id: 2013.com.canonical.plainbox::miscellanea +estimated_duration: 0.5 +user: root +id: miscellanea/efi_pxeboot +requires: + cpuinfo.platform in ("i386", "x86_64", "aarch64") +depends: miscellanea/efi_boot_mode +_summary: Test that system booted from the network +_description: + Test to verify that the system booted from the network. + Works only on EFI-based systems. +command: efi-pxeboot + +plugin: shell +category_id: 2013.com.canonical.plainbox::miscellanea id: miscellanea/bmc_info requires: package.name == 'ipmitool' @@ -163,7 +191,7 @@ id: miscellanea/dmitest_server requires: package.name == 'dmidecode' estimated_duration: 0.5 user: root -command: dmitest --test_versions --test_serials server +command: dmitest --test_versions server _description: Sanity check of DMI system identification data (for servers) _summary: @@ -275,15 +303,9 @@ plugin: shell category_id: 2013.com.canonical.plainbox::miscellanea estimated_duration: 0.1 id: miscellanea/get_maas_version -command: - if [ -s /etc/installed-by-maas ]; then - cat /etc/installed-by-maas - else - echo "MAAS VERSION NOT FOUND" >&2 - false - fi +command: maas-version-check _description: If system was installed via MAAS from a cert server, the MAAS version used should be contained in /etc/installed-by-maas -_summary: Attach MAAS version used to deploy the SUT +_summary: Verify MAAS version used to deploy the SUT plugin: shell category_id: 2013.com.canonical.plainbox::miscellanea @@ -30,7 +30,7 @@ class InstallPyModules(InstallCommand): setup( name='plainbox-provider-checkbox', namespace='2013.com.canonical.certification', - version="0.33.0", + version="0.34.0rc1", description=N_("Checkbox provider"), gettext_domain='plainbox-provider-checkbox', strict=False, deprecated=False, |