diff options
| -rwxr-xr-x | bin/disk_smart | 181 | ||||
| -rwxr-xr-x | bin/fwts_test | 2 | ||||
| -rwxr-xr-x | bin/memory_compare | 55 | ||||
| -rwxr-xr-x | bin/network_device_info | 8 | ||||
| -rwxr-xr-x | bin/network_info | 11 | ||||
| -rwxr-xr-x | bin/removable_storage_test | 142 | ||||
| -rw-r--r-- | jobs/mediacard.txt.in | 2 | ||||
| -rwxr-xr-x | manage.py | 2 | ||||
| -rw-r--r-- | po/pl.po | 2 |
9 files changed, 262 insertions, 143 deletions
diff --git a/bin/disk_smart b/bin/disk_smart index c7f58dfa..1aea483c 100755 --- a/bin/disk_smart +++ b/bin/disk_smart @@ -7,6 +7,7 @@ Copyright (C) 2010 Canonical Ltd. Authors Jeff Lane <jeffrey.lane@canonical.com> Brendan Donegan <brendan.donegan@canonical.com> + Rod Smith <rod.smith@canonical.com> This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License version 2, @@ -34,6 +35,11 @@ USB/eSATA/eSAS attached storage devices. Changelog: +v1.3: Fix detection of SMART availability & activate SMART if available + but deactivated. Also use smartctl return value rather than string- + matching to determine if a test has failed; this should be more + robust, as output strings vary between disks. +v1.2: Handle multiple output formats for "smartctl -l" v1.1: Put delay before first attempt to acces log, rather than after v1.0: added debugger class and code to allow for verbose debug output if needed @@ -58,8 +64,10 @@ import os import sys import time import logging +import shlex -from subprocess import Popen, PIPE +from subprocess import Popen, PIPE, check_call, check_output +from subprocess import CalledProcessError from argparse import ArgumentParser @@ -72,27 +80,57 @@ class ListHandler(logging.StreamHandler): msg = msg.decode() logger = logging.getLogger(record.name) new_record = logger.makeRecord(record.name, record.levelno, - record.pathname, record.lineno, msg, record.args, - record.exc_info, record.funcName) + record.pathname, record.lineno, + msg, record.args, + record.exc_info, + record.funcName) logging.StreamHandler.emit(self, new_record) else: logging.StreamHandler.emit(self, record) +def enable_smart(disk): + """ Enable SMART on the specified disk + :param disk: + disk device filename (e.g., /dev/sda) + :returns: + True if enabling smart was successful, False otherwise + """ + logging.debug('SMART disabled; attempting to enable it.') + command = 'smartctl -s on %s' % disk + try: + check_call(shlex.split(command)) + return True + except CalledProcessError: + return False + + +# Returns True if SMART is enabled. If not enabled, attempt to +# enable it and return result of that attempt. def is_smart_enabled(disk): # Check with smartctl to see if SMART is available and enabled on the disk command = 'smartctl -i %s' % disk diskinfo_bytes = (Popen(command, stdout=PIPE, shell=True) - .communicate()[0]) + .communicate()[0]) diskinfo = diskinfo_bytes.decode().splitlines() logging.debug('SMART Info for disk %s', disk) logging.debug(diskinfo) - return (len(diskinfo) > 2 - and 'Enabled' in diskinfo[-2] - and 'Available' in diskinfo[-3]) + # Return True if the output (diskinfo) includes BOTH + # "SMART support is.*Available" AND "SMART support is.*Enabled". + # If SMART is available but not enabled, try to enable it. + if len(diskinfo) > 2: + if any("SMART support is" in s and "Available" in s + for s in diskinfo): + if any("SMART support is" in s and "Enabled" in s + for s in diskinfo): + return True + else: + return enable_smart(disk) + else: + return False def run_smart_test(disk, type='short'): @@ -110,20 +148,23 @@ def run_smart_test(disk, type='short'): def get_smart_entries(disk, type='selftest'): entries = [] - command = 'smartctl -l %s %s' % (type, disk) - stdout = Popen(command, stdout=PIPE, shell=True).stdout + try: + stdout = check_output(['smartctl', '-l', type, disk], + universal_newlines=True) + returncode = 0 + except CalledProcessError as err: + stdout = err.output + returncode = err.returncode # Skip intro lines - while True: - line = stdout.readline().decode() - if not line: - raise Exception('Failed to parse SMART log entries') - - if line.startswith('SMART'): + stdout_lines = iter(stdout.splitlines()) + for line in stdout_lines: + if (line.startswith('SMART') or + line.startswith('No self-tests have been logged')): break # Get lengths from header - line = stdout.readline().decode() + line = next(stdout_lines) if not line.startswith('Num'): entries.append('No entries found in log yet') return entries @@ -135,8 +176,7 @@ def get_smart_entries(disk, type='selftest'): # Get remaining lines entries = [] - for line_bytes in stdout.readlines(): - line = line_bytes.decode() + for line in stdout_lines: if line.startswith('#'): entry = {} for i, column in enumerate(columns): @@ -144,10 +184,64 @@ def get_smart_entries(disk, type='selftest'): # Convert some columns to integers entry['number'] = int(entry['number'][1:]) - entry['lifetime'] = int(entry['lifetime']) entries.append(entry) - return entries + return entries, returncode + + +# Returns True if an "in-progress" message is found in the smartctl +# output, False if such a message is not found. In the former case, +# the in-progress message entries are logged. +def in_progress(current_entries): + statuses = [entry for entry in current_entries + if isinstance(entry, dict) + and 'status' in entry + and (entry['status'] == 'Self-test routine in progress' + or "Self test in progress" in entry['status'])] + if statuses: + for entry in statuses: + logging.debug('%s %s %s %s' % (entry['number'], + entry['description'], + entry['status'], + entry['remaining'])) + return True + else: + return False + + +# Wait for SMART test to complete; return status and return code. +# Note that different disks return different types of values. +# Some return no status reports while a test is ongoing; others +# show a status line at the START of the list of tests, and +# others show a status line at the END of the list of tests +# (and then move it to the top once the tests are done). +def poll_for_status(args, disk, previous_entries): + # Priming read... this is here in case our test is finished or fails + # immediate after it beginsAccording to. + logging.debug('Polling selftest.log for status') + keep_going = True + + while keep_going: + # Poll every sleep seconds until test is complete$ + time.sleep(args.sleep) + + current_entries, returncode = get_smart_entries(disk) + if current_entries != previous_entries: + if not in_progress(current_entries): + keep_going = False + + if args.timeout is not None: + if args.timeout <= 0: + logging.debug('Polling timed out') + return 1 + else: + args.timeout -= args.sleep + + if isinstance(current_entries[0], str): + return current_entries[0], returncode + else: + return current_entries[0]['status'], returncode + def main(): description = 'Tests that SMART capabilities on disks that support SMART function.' @@ -184,7 +278,7 @@ def main(): logger.setLevel(logging.INFO) # Make sure we're root, because smartctl doesn't work otherwise. - if not os.geteuid()==0: + if not os.geteuid() == 0: parser.error("You must be root to run this program") # If SMART is available and enabled, we proceed. Otherwise, we exit as the @@ -195,8 +289,8 @@ def main(): return 0 # Initiate a self test and start polling until the test is done - previous_entries = get_smart_entries(disk) - logging.info("Starting SMART self-test on %s" % disk) + previous_entries, returncode = get_smart_entries(disk) + logging.info("Starting SMART self-test on %s", disk) if run_smart_test(disk) != 0: logging.error("Error reported during smartctl test") return 1 @@ -205,42 +299,17 @@ def main(): # Abort the previous instance # so that polling can identify the difference run_smart_test(disk) - previous_entries = get_smart_entries(disk) - - # Priming read... this is here in case our test is finished or fails - # immediate after it begins. - logging.debug('Polling selftest.log for status') - - while True: - # Poll every sleep seconds until test is complete$ - time.sleep(args.sleep) - - current_entries = get_smart_entries(disk) - if isinstance(current_entries[0], str): - logging.debug(current_entries[0]) - else: - logging.debug('%s %s %s %s' % (current_entries[0]['number'], - current_entries[0]['description'], - current_entries[0]['status'], - current_entries[0]['remaining'])) - if current_entries != previous_entries \ - and current_entries[0]["status"] != 'Self-test routine in progress': - break - - if args.timeout is not None: - if args.timeout <= 0: - logging.debug('Polling timed out') - return 1 - else: - args.timeout -= args.sleep + previous_entries, returncode = get_smart_entries(disk) - status = current_entries[0]['status'] + status, returncode = poll_for_status(args, disk, previous_entries) - if status != 'Completed without error': - log = get_smart_entries(disk) + if returncode != 0: + log, returncode = get_smart_entries(disk) logging.error("FAIL: SMART Self-Test appears to have failed for some reason. " - "Run 'sudo smartctl -l selftest %s' to see the SMART log" % disk) - logging.debug("Last self-test run status: %s" % status) + "Run 'sudo smartctl -l selftest %s' to see the SMART log", + disk) + logging.debug("Last smartctl return code: %d", returncode) + logging.debug("Last smartctl run status: %s", status) return 1 else: logging.info("PASS: SMART Self-Test completed without error") diff --git a/bin/fwts_test b/bin/fwts_test index 029c1ec9..127858a3 100755 --- a/bin/fwts_test +++ b/bin/fwts_test @@ -317,7 +317,7 @@ def main(): args.resume_time = 3 tests.extend(args.sleep) else: - tests.extend(TESTS) + tests.extend(CERT_TESTS) # run the tests we want if args.sleep: diff --git a/bin/memory_compare b/bin/memory_compare index 2c26f673..12a14ecd 100755 --- a/bin/memory_compare +++ b/bin/memory_compare @@ -27,6 +27,7 @@ import sys from math import log, copysign from subprocess import check_output, PIPE +from checkbox_support.helpers.human_readable_bytes import HumanReadableBytes from checkbox_support.parsers.lshwjson import LshwJsonParser from checkbox_support.parsers.meminfo import MeminfoParser @@ -82,51 +83,23 @@ def get_threshold(installed_memory): return 10 -def bytes_to_human(my_bytes): - """ Convert my_bytes to a scaled representation with a - suffix - """ - if my_bytes == 0: - return "0 bytes" - - suffixes = ["bytes", "KiB", "MiB", "GiB", "TiB", - "PiB", "EiB", "ZiB", "YiB"] - - try: - sign = copysign(1, my_bytes) - except OverflowError as excp: - return "(Number too large: {})".format(excp) - my_bytes = abs(my_bytes) - # my_bytes' base-1024 logarithm. - exponent = log(my_bytes, 1024) - try: - suffix = suffixes[int(exponent)] - except IndexError: - return "(Number too large)" - scalar = my_bytes / (1024**int(exponent)) - - return "{:.2f} {}".format(sign * scalar, suffix) - - def main(): if os.geteuid() != 0: print("This script must be run as root.", file=sys.stderr) return 1 - installed_memory = get_installed_memory_size() - visible_memory = get_visible_memory_size() + installed_memory = HumanReadableBytes(get_installed_memory_size()) + visible_memory = HumanReadableBytes(get_visible_memory_size()) threshold = get_threshold(installed_memory) - difference = installed_memory - visible_memory + difference = HumanReadableBytes(installed_memory - visible_memory) try: percentage = difference / installed_memory * 100 except ZeroDivisionError: print("Results:") - print("\t/proc/meminfo reports:\t{}".format( - bytes_to_human(visible_memory)), + print("\t/proc/meminfo reports:\t{}".format(visible_memory), file=sys.stderr) - print("\tlshw reports:\t{}".format( - bytes_to_human(installed_memory)), + print("\tlshw reports:\t{}".format(installed_memory), file=sys.stderr) print("\nFAIL: Either lshw or /proc/meminfo returned a memory size " "of 0 kB", file=sys.stderr) @@ -134,22 +107,18 @@ def main(): if percentage <= threshold: print("Results:") - print("\t/proc/meminfo reports:\t{}".format( - bytes_to_human(visible_memory))) - print("\tlshw reports:\t{}".format(bytes_to_human(installed_memory))) + print("\t/proc/meminfo reports:\t{}".format(visible_memory)) + print("\tlshw reports:\t{}".format(installed_memory)) print("\nPASS: Meminfo reports %s less than lshw, a " "difference of %.2f%%. This is less than the " - "%d%% variance allowed." % (bytes_to_human(difference), - percentage, threshold)) + "%d%% variance allowed." % (difference, percentage, threshold)) return 0 else: print("Results:", file=sys.stderr) - print("\t/proc/meminfo reports:\t{}".format( - bytes_to_human(visible_memory)), - file=sys.stderr) - print("\tlshw reports:\t{}".format(bytes_to_human(installed_memory)), + print("\t/proc/meminfo reports:\t{}".format(visible_memory), file=sys.stderr) - print("\nFAIL: Meminfo reports %d bytes less than lshw, " + print("\tlshw reports:\t{}".format(installed_memory), file=sys.stderr) + print("\nFAIL: Meminfo reports %d less than lshw, " "a difference of %.2f%%. Only a variance of %d%% in " "reported memory is allowed." % (difference, percentage, threshold), file=sys.stderr) diff --git a/bin/network_device_info b/bin/network_device_info index b8c02675..f4b79110 100755 --- a/bin/network_device_info +++ b/bin/network_device_info @@ -200,8 +200,10 @@ def match_counts(nm_devices, udev_devices, devtype): if udev.category == udevtype] if len(nm_type_devices) != len(udev_type_devices): print("ERROR: devices missing - udev showed %d %s devices, but " - "NetworkManager saw %d devices in %s" % (len(udev_type_devices), - udevtype, len(nm_type_devices), devtype), file=sys.stderr) + "NetworkManager saw %d devices in %s" % + (len(udev_type_devices), udevtype, + len(nm_type_devices), devtype), + file=sys.stderr) return False else: return True @@ -253,7 +255,7 @@ def main(args): if not match_counts(nm_devices, udev_devices, "WiFi"): return 1 - elif not match_counts(nm_devices, udev_devices, ("Ethernet","Modem")): + elif not match_counts(nm_devices, udev_devices, ("Ethernet", "Modem")): return 1 else: return 0 diff --git a/bin/network_info b/bin/network_info index b9b46dde..9c20b89b 100755 --- a/bin/network_info +++ b/bin/network_info @@ -38,9 +38,10 @@ def get_ip_address(interface): struct.pack('256s', interface[:15].encode()) )[20:24]) + def get_ipv6_address(interface): cmd = ['/sbin/ip', '-6', 'addr', 'show', 'dev', interface] - proc = subprocess.check_output(cmd,universal_newlines=True) + proc = subprocess.check_output(cmd, universal_newlines=True) ipaddr = proc.split()[8].strip() return ipaddr @@ -63,14 +64,14 @@ def main(args): print("Interface: %s" % interface) print("Connected: %s" % connected) try: - print("IPv4: %s" % get_ip_address(interface)) + print("IPv4: %s" % get_ip_address(interface)) except IOError: print("IPv4: n/a") - try: - print("IPv6: %s" % get_ipv6_address(interface)) + try: + print("IPv6: %s" % get_ipv6_address(interface)) except IOError: print("IPv6: n/a") - except: + except: print("IPv6: n/a") print("MAC: %s\n" % get_mac_address(interface)) diff --git a/bin/removable_storage_test b/bin/removable_storage_test index 9360becc..8e7732b8 100755 --- a/bin/removable_storage_test +++ b/bin/removable_storage_test @@ -22,6 +22,7 @@ from checkbox_support.dbus.udisks2 import is_udisks2_supported from checkbox_support.dbus.udisks2 import lookup_udev_device from checkbox_support.dbus.udisks2 import map_udisks1_connection_bus from checkbox_support.heuristics.udisks2 import is_memory_card +from checkbox_support.helpers.human_readable_bytes import HumanReadableBytes from checkbox_support.parsers.udevadm import CARD_READER_RE from checkbox_support.parsers.udevadm import GENERIC_RE from checkbox_support.parsers.udevadm import FLASH_RE @@ -252,24 +253,6 @@ class DiskTest(): else: self.rem_disks_memory_cards_nm[dev_file] = None self.rem_disks_nm[dev_file] = None - # LP: #1313581 - # Compare the pci slot name of the devices using xhci and - # the pci slot name of the disks, - # which is usb3 disks in this case so far, - # to make sure the usb3 disk does be on the bus using xhci - # TODO: it will be better to extend to be all kinds of drivers. - try: - udev_devices_xhci = get_udev_xhci_devices(udev_client) - for udev_device_xhci in udev_devices_xhci: - pci_slot_name = udev_device_xhci.get_property('PCI_SLOT_NAME') - for udev_device in udev_devices: - devpath = udev_device.get_property('DEVPATH') - if (self._compare_pci_slot_from_devpath(devpath, - pci_slot_name)): - self.rem_disks_xhci[ - udev_device.get_property('DEVNAME')] = 'xhci' - except: - logging.error("Failed to get driver information.") def _probe_disks_udisks1(self, bus): """ @@ -320,6 +303,30 @@ class DiskTest(): self.rem_disks_nm[dev_file] = None self.rem_disks_memory_cards_nm[dev_file] = None + def get_disks_xhci(self): + """ + Compare + 1. the pci slot name of the devices using xhci + 2. the pci slot name of the disks, + which is usb3 disks in this case so far, + to make sure the usb3 disk does be on the controller using xhci + """ + # LP: #1378724 + udev_client = GUdev.Client() + # Get a collection of all udev devices corresponding to block devices + udev_devices = get_udev_block_devices(udev_client) + # Get a collection of all udev devices corresponding to xhci devices + udev_devices_xhci = get_udev_xhci_devices(udev_client) + for udev_device_xhci in udev_devices_xhci: + pci_slot_name = udev_device_xhci.get_property('PCI_SLOT_NAME') + for udev_device in udev_devices: + devpath = udev_device.get_property('DEVPATH') + if (self._compare_pci_slot_from_devpath(devpath, + pci_slot_name)): + self.rem_disks_xhci[ + udev_device.get_property('DEVNAME')] = 'xhci' + return self.rem_disks_xhci + def mount(self): passed_mount = {} @@ -420,10 +427,17 @@ def main(): help='The number of random data files to generate') parser.add_argument('-s', '--size', action='store', - type=int, - default=1048576, - help=("The size of the test data file to use " - "in Bytes. Default is %(default)s")) + type=HumanReadableBytes, + default='1MiB', + help=("The size of the test data file to use. " + "You may use SI or IEC suffixes like: 'K', 'M'," + "'G', 'T', 'Ki', 'Mi', 'Gi', 'Ti', etc. Default" + " is %(default)s")) + parser.add_argument('--auto-reduce-size', + action='store_true', + default=False, + help=("Automatically reduce size to fit in the target" + "filesystem. Reducing until fits in 1MiB")) parser.add_argument('-n', '--skip-not-mount', action='store_true', default=False, @@ -516,11 +530,36 @@ def main(): if not args.min_speed or int(test.rem_disks_speed[disk]) >= int(args.min_speed)} + if len(disks_eligible) == 0: + logging.error( + "No %s disks with speed higher than %s bits/s", + args.device, args.min_speed) + return 1 write_sizes = [] test_files = {} + disks_freespace = {} + for disk, path in disks_eligible.items(): + stat = os.statvfs(path) + disks_freespace[disk] = stat.f_bfree * stat.f_bsize + smallest_freespace = min(disks_freespace.values()) + desired_size = args.size + if desired_size > smallest_freespace: + if args.auto_reduce_size: + min_space = HumanReadableBytes("1MiB") + if smallest_freespace < min_space: + raise IOError("Not enough space. {} is required" + .format(min_space)) + new_size = HumanReadableBytes(int(0.8 * smallest_freespace)) + logging.warning("Automatically reducing test data size" + ". {} requested. Reducing to {}." + .format(desired_size, new_size)) + desired_size = new_size + else: + raise IOError("Not enough space. {} is required" + .format(desired_size)) # Generate our data file(s) for count in range(args.count): - test_files[count] = RandomData(args.size) + test_files[count] = RandomData(desired_size) write_sizes.append(os.path.getsize( test_files[count].tfile.name)) total_write_size = sum(write_sizes) @@ -610,20 +649,59 @@ def main(): "Completed %s test iterations, but there were" " errors", args.count) return 1 - elif len(disks_eligible) == 0: - logging.error( - "No %s disks with speed higher than %s bits/s", - args.device, args.min_speed) - return 1 - else: # LP: 1313581 + # Try to figure out whether the disk + # is SuperSpeed USB and using xhci_hcd driver. if (args.driver == 'xhci_hcd'): - if(5000000000 == test.rem_disks_speed[disk] and - 'xhci' == test.rem_disks_xhci[disk]): + # The speed reported by udisks is sometimes + # less than 5G bits/s, for example, + # it may be 705032705 bits/s + # So using + # 500000000 + # = 500 M bits/s + # > 480 M bits/s ( USB 2.0 spec.) + # to make sure that it is higher USB version than 2.0 + # + # int() for int(test.rem_disks_speed[disk]) + # is necessary + # because the speed value of + # the dictionary rem_disks_speed is + # 1. str or int from _probe_disks_udisks2 + # 2. int from _probe_disks_udisks1. + # This is really a mess. : ( + print("\t\t--------------------------------") + if(500000000 < int(test.rem_disks_speed[disk])): print("\t\tDevice Detected: SuperSpeed USB") - print("\t\tDriver Detected: xhci_hcd") + # Unlike rem_disks_speed, + # which must has the connect speed + # for each disk devices, + # disk devices may not use xhci as + # controller drivers. + # This will raise KeyError for no + # associated disk device was found. + xhci_disks = test.get_disks_xhci() + # pep8 style suggest to limit the try clause + # to the absolute minimum amount of code necessary + try: + disk_xhci_flag = xhci_disks[disk] + except KeyError: + print("\t\tDisk does not use xhci_hci.") + return 1 + else: + if('xhci' == disk_xhci_flag): + print("\t\tDriver Detected: xhci_hcd") + else: + print("\t\tDisk does not use xhci_hci.") + logging.debug("disk_xhci_flag is not xhci") + return 1 else: + # Give it a hint for the detection failure. + # LP: #1362902 + print(("\t\tNo SuperSpeed USB using xhci_hcd " + "was detected correctly.")) + print(("\t\tHint: please use dmesg to check " + "the system status again.")) return 1 # Pass is not assured if (not args.pass_speed or diff --git a/jobs/mediacard.txt.in b/jobs/mediacard.txt.in index 262f1ffe..71fb8ab7 100644 --- a/jobs/mediacard.txt.in +++ b/jobs/mediacard.txt.in @@ -20,7 +20,7 @@ id: mediacard/mmc-storage estimated_duration: 30.0 depends: mediacard/mmc-insert user: root -command: removable_storage_test -s 67120000 --memorycard sdio usb scsi +command: removable_storage_test -s 67120000 --memorycard sdio usb scsi --auto-reduce-size _description: This test is automated and executes after the mediacard/mmc-insert test is run. It tests reading and writing to the MMC card. @@ -4,7 +4,7 @@ from plainbox.provider_manager import N_ setup( name='2013.com.canonical.certification:checkbox', - version="0.13", + version="0.14", description=N_("Checkbox provider"), gettext_domain='2013.com.canonical.certification.checkbox', strict=False, deprecated=False, @@ -14,7 +14,7 @@ msgstr "" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" -"X-Launchpad-Export-Date: 2014-09-27 05:37+0000\n" +"X-Launchpad-Export-Date: 2014-10-15 05:42+0000\n" "X-Generator: Launchpad (build 17196)\n" "Language: Polish\n" |