diff options
51 files changed, 898 insertions, 542 deletions
diff --git a/bin/brightness_test b/bin/brightness_test index 818796e..af1de1b 100755 --- a/bin/brightness_test +++ b/bin/brightness_test @@ -5,9 +5,11 @@ # # This file is part of Checkbox. # -# Copyright 2012 Canonical Ltd. +# Copyright 2012-2018 Canonical Ltd. # -# Authors: Alberto Milone <alberto.milone@canonical.com> +# Authors: +# Alberto Milone <alberto.milone@canonical.com> +# Sylvain Pineau <sylvain.pineau@canonical.com> # # Checkbox is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, @@ -33,7 +35,7 @@ from glob import glob class Brightness(object): def __init__(self, path='/sys/class/backlight'): self.sysfs_path = path - self._interfaces = self._get_interfaces_from_path() + self.interfaces = self._get_interfaces_from_path() def read_value(self, path): '''Read the value from a file''' @@ -94,36 +96,6 @@ class Brightness(object): return interfaces - def get_best_interface(self): - '''Get the best acpi interface''' - # Follow the heuristic in https://www.kernel.org/doc/Documentation/ - #ABI/stable/sysfs-class-backlight - if len(self._interfaces) == 0: - return None - else: - interfaces = {} - for interface in self._interfaces: - try: - with open(interface + '/type') as type_file: - iface_type = type_file.read().strip() - except IOError: - continue - interfaces[iface_type] = interface - - if interfaces.get('firmware'): - return interfaces.get('firmware') - elif interfaces.get('platform'): - return interfaces.get('platform') - elif interfaces.get('raw'): - return interfaces.get('raw') - else: - fallback_type = sorted(interfaces.keys())[0] - print("WARNING: no interface of type firmware/platform/raw " - "found. Using {} of type {}".format( - interfaces[fallback_type], - fallback_type)) - return interfaces[fallback_type] - def was_brightness_applied(self, interface): '''See if the selected brightness was applied @@ -132,9 +104,9 @@ class Brightness(object): ''' if (abs(self.get_actual_brightness(interface) - self.get_last_set_brightness(interface)) > 1): - return False + return 1 else: - return True + return 0 def main(): @@ -146,33 +118,35 @@ def main(): file=sys.stderr) exit(1) - interface = brightness.get_best_interface() # If no backlight interface can be found - if not interface: + if len(brightness.interfaces) == 0: exit(1) - # Get the current brightness which we can restore later - current_brightness = brightness.get_actual_brightness(interface) + exit_status = 0 + for interface in brightness.interfaces: + + # Get the current brightness which we can restore later + current_brightness = brightness.get_actual_brightness(interface) - # Get the maximum value for brightness - max_brightness = brightness.get_max_brightness(interface) + # Get the maximum value for brightness + max_brightness = brightness.get_max_brightness(interface) - # Set the brightness to half the max value - brightness.write_value(max_brightness / 2, - os.path.join(interface, - 'brightness')) + # Set the brightness to half the max value + brightness.write_value(max_brightness / 2, + os.path.join(interface, + 'brightness')) - # Check that "actual_brightness" reports the same value we - # set "brightness" to - exit_status = not brightness.was_brightness_applied(interface) + # Check that "actual_brightness" reports the same value we + # set "brightness" to + exit_status += brightness.was_brightness_applied(interface) - # Wait a little bit before going back to the original value - time.sleep(2) + # Wait a little bit before going back to the original value + time.sleep(2) - # Set the brightness back to its original value - brightness.write_value(current_brightness, - os.path.join(interface, - 'brightness')) + # Set the brightness back to its original value + brightness.write_value(current_brightness, + os.path.join(interface, + 'brightness')) exit(exit_status) diff --git a/bin/disk_stress_ng b/bin/disk_stress_ng index 0b5de83..df5694e 100755 --- a/bin/disk_stress_ng +++ b/bin/disk_stress_ng @@ -121,7 +121,7 @@ find_largest_partition() { exit 1 fi local blkid_info=$(blkid -s TYPE $part_location | grep -E ext2\|ext3\|ext4\|xfs\|jfs\|btrfs\|LVM2_member) - if [ "$part_size" > "$largest_size" ] && [ -n "$blkid_info" ] ; then + if [ "$part_size" -gt "$largest_size" ] && [ -n "$blkid_info" ] ; then if [[ "$blkid_info" =~ .*LVM2_member.* ]] ; then find_largest_lv else diff --git a/bin/lock_screen_watcher b/bin/lock_screen_watcher index b8dd886..4e9847e 100755 --- a/bin/lock_screen_watcher +++ b/bin/lock_screen_watcher @@ -1,116 +1,49 @@ #!/usr/bin/env python3 -import argparse -import sys -import subprocess -import lsb_release - +# This file is part of Checkbox. +# +# Copyright 2018 Canonical Ltd. +# Written by: Sylvain Pineau <sylvain.pineau@canonical.com> +# +# Checkbox is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, +# as published by the Free Software Foundation. +# +# Checkbox is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Checkbox. If not, see <http://www.gnu.org/licenses/>. + +import dbus +from dbus.mainloop.glib import DBusGMainLoop from gi.repository import GObject - -from checkbox_support.dbus import connect_to_system_bus - -import threading -import time - -GObject.threads_init() - -class SceenSaverStatusHelper(threading.Thread): - - def __init__(self, loop): - super(SceenSaverStatusHelper, self).__init__() - self._loop = loop - self.quit = False - - def query(self): - if (lsb_release.get_distro_information()["ID"] == "Ubuntu"): - if (lsb_release.get_distro_information()["CODENAME"] == "trusty"): - # trusty uses login screen as screen saver - process_ps = subprocess.Popen(["ps", "aux"], stdout=subprocess.PIPE) - process_grep = subprocess.Popen(["grep", - "/usr/lib/unity/unity-panel-service --lockscreen-mode"], - stdin=process_ps.stdout, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) - process_ps.stdout.close() - stdout = process_grep.communicate()[0] - if (len(stdout.decode().split("\n")) == 3): - print("the screensaver is active") - self._loop.quit() - p = subprocess.Popen(["gnome-screensaver-command", "-q"], stdout=subprocess.PIPE) - stdout, stderr = p.communicate() - # parse the stdout string from the command "gnome-screensaver-command -q" - # the result should be "active" or "inactive" - if "active" == stdout.decode().split(" ")[-1].rstrip() : - print("the screensaver is active") - self._loop.quit() - - def run(self): - while not self.quit: - GObject.idle_add(self.query) - time.sleep(1) - - -class HotkeyFunctionListener: - - def __init__(self, system_bus, loop): - self._bus = system_bus - self._loop = loop - # Assume the test passes, this is changed when timeout expires - self._error = False - - def _on_timeout_expired(self): - """ - Internal function called when the timer expires. - - Basically it's just here to tell the user the test failed or that the - user was unable to pressed the hot key during the allowed time. - """ - print("You have failed to perform the required manipulation in time") - # Fail the test when the timeout was reached - self._error = True - # Stop the loop now - self._loop.quit() - - def check(self, timeout): - """ - Run the configured test and return the result - - The result is False if the test has failed. The timeout, when - non-zero, will make the test fail after the specified seconds have - elapsed without conclusive result. - """ - # Setup a timeout if requested - if timeout > 0: - GObject.timeout_add_seconds(timeout, self._on_timeout_expired) - - # helper to listen the functionality is triggered or not - query_thread = SceenSaverStatusHelper(self._loop) - query_thread.start() - - self._loop.run() - query_thread.quit = True - # Return the outcome of the test - return self._error - -def main(): - description = "Wait for the specified hotkey to be pressed." - parser = argparse.ArgumentParser(description=description) - parser.add_argument('--timeout', type=int, default=30) - - args = parser.parse_args() - - # Connect to the system bus, we also get the event - # loop as we need it to start listening for signals. - system_bus, loop = connect_to_system_bus() - - listener = HotkeyFunctionListener(system_bus, loop) - - # Run the actual listener and wait till it either times out or discovers - # the specific hot key pressed. - try: - return listener.check(args.timeout) - except KeyboardInterrupt: - return 1 - -if __name__ == "__main__": - sys.exit(main()) - +from gi.repository import GLib + + +def filter_cb(bus, message): + if message.get_member() == "EventEmitted": + args = message.get_args_list() + if args[0] == "desktop-lock": + print("Lock Screen detected") + mainloop.quit() + elif message.get_member() == "ActiveChanged": + args = message.get_args_list() + if args[0] == True: # noqa: E712 + print("Lock Screen detected") + mainloop.quit() + + +def on_timeout_expired(): + print("You have failed to perform the required manipulation in time") + raise SystemExit(1) + +DBusGMainLoop(set_as_default=True) +bus = dbus.SessionBus() +bus.add_match_string("type='signal',interface='com.ubuntu.Upstart0_6'") +bus.add_match_string("type='signal',interface='org.gnome.ScreenSaver'") +bus.add_message_filter(filter_cb) +mainloop = GLib.MainLoop() +GObject.timeout_add_seconds(30, on_timeout_expired) +mainloop.run() diff --git a/bin/pm_test b/bin/pm_test index 35cd215..481b566 100755 --- a/bin/pm_test +++ b/bin/pm_test @@ -637,7 +637,7 @@ class AutoStartFile(object): [Desktop Entry] Name={pm_operation} test Comment=Verify {pm_operation} works properly -Exec=sudo /usr/bin/python3 {script} -r {repetitions} -w {wakeup} --hardware-delay {hardware_delay} --pm-delay {pm_delay} --min-pm-time {min_pm_time} --max-pm-time {max_pm_time} --append --total {total} --start {start} --pm-timestamp {pm_timestamp} {silent} --log-level={log_level} --log-dir={log_dir} --suspends-before-reboot={suspend_cycles} --checkbox-respawn-cmd={checkbox_respawn} {fwts} {pm_operation} +Exec=sudo {script} -r {repetitions} -w {wakeup} --hardware-delay {hardware_delay} --pm-delay {pm_delay} --min-pm-time {min_pm_time} --max-pm-time {max_pm_time} --append --total {total} --start {start} --pm-timestamp {pm_timestamp} {silent} --log-level={log_level} --log-dir={log_dir} --suspends-before-reboot={suspend_cycles} --checkbox-respawn-cmd={checkbox_respawn} {fwts} {pm_operation} Type=Application X-GNOME-Autostart-enabled=true Hidden=false @@ -673,9 +673,13 @@ Hidden=false """ logging.debug('Writing desktop file ({0!r})...' .format(self.desktop_filename)) - + snap_name = os.getenv('SNAP_NAME') + if snap_name: + script = '/snap/bin/{}.pm-test'.format(snap_name) + else: + script = '/usr/bin/python3 {}'.format(os.path.realpath(__file__)) contents = (self.TEMPLATE - .format(script=os.path.realpath(__file__), + .format(script=script, repetitions=self.args.repetitions - 1, wakeup=self.args.wakeup, hardware_delay=self.args.hardware_delay, diff --git a/bin/removable_storage_test b/bin/removable_storage_test index ddd4e1b..42e197a 100755 --- a/bin/removable_storage_test +++ b/bin/removable_storage_test @@ -99,6 +99,20 @@ def md5_hash_file(path): return md5.hexdigest() +def on_ubuntucore(): + """ + Check if running from on ubuntu core + """ + snap = os.getenv("SNAP") + if snap: + with open(os.path.join(snap, 'meta/snap.yaml')) as f: + for l in f.readlines(): + if l == "confinement: classic\n": + return False + return True + return False + + class DiskTest(): ''' Class to contain various methods for testing removable disks ''' @@ -177,7 +191,7 @@ class DiskTest(): Indirectly sets: self.rem_disks{,_nm,_memory_cards,_memory_cards_nm,_speed} """ - if "SNAP" in os.environ: + if on_ubuntucore(): self._probe_disks_udisks2_cli() else: bus, loop = connect_to_system_bus() diff --git a/bin/removable_storage_watcher b/bin/removable_storage_watcher index 522e7a8..d5a0722 100755 --- a/bin/removable_storage_watcher +++ b/bin/removable_storage_watcher @@ -619,6 +619,10 @@ class UDisks2StorageDeviceListener: and record.value.prop_name == "MountPoints" and record.value.prop_value != []): found.add('mounted') + # On some systems partition are reported as mounted + # filesystems, without 'partition' record + if set(['partition', 'mounted']).issubset(needs): + needs.remove('partition') # Finally memorize the drive the block device belongs to elif (record.value.iface_name == "org.freedesktop.UDisks2.Block" diff --git a/data/palm_rejection.qml b/data/palm_rejection.qml index 1c9b974..3f93f2a 100644 --- a/data/palm_rejection.qml +++ b/data/palm_rejection.qml @@ -1,7 +1,7 @@ /* * This file is part of Checkbox. * - * Copyright 2016 Canonical Ltd. + * Copyright 2018 Canonical Ltd. * Written by: * Maciej Kisielewski <maciej.kisielewski@canonical.com> * @@ -17,10 +17,10 @@ * You should have received a copy of the GNU General Public License * along with Checkbox. If not, see <http://www.gnu.org/licenses/>. */ -import Ubuntu.Components 1.2 -import QtQuick 2.0 -import QtQuick.Layouts 1.1 -import Plainbox 0.1 +import QtQuick 2.5 +import QtQuick.Layouts 1.2 +import QtQuick.Controls 1.4 +import Plainbox 1.0 QmlJob { property var steps: [ @@ -38,8 +38,8 @@ QmlJob { ColumnLayout { anchors.fill: parent - anchors.margins: units.gu(1) - spacing: units.gu(1) + anchors.margins: 10 + spacing: 30 Image { source: steps[currentStep][0] fillMode: Image.PreserveAspectFit @@ -47,34 +47,30 @@ QmlJob { Layout.fillWidth: true } Label { - text: i18n.tr("The cursor <b>SHOULD NOT</b> move with the palm movement?") - fontSize: 'large' + text: "The cursor <b>SHOULD NOT</b> move with the palm movement?" horizontalAlignment: Text.AlignHCenter verticalAlignment: Text.AlignVCenter - Layout.minimumHeight: units.gu(10) + Layout.minimumHeight: 30 Layout.fillHeight: true Layout.fillWidth: true } RowLayout { - Layout.minimumHeight: units.gu(10) + Layout.minimumHeight: 10 Layout.fillWidth: true Button { - text: i18n.tr('Pass') - color: UbuntuColors.green + text: 'Pass' Layout.fillHeight: true Layout.fillWidth: true onClicked: testDone({'outcome': 'pass'}) } Button { - text: i18n.tr('Skip') - color: "#FF9900" + text: 'Skip' Layout.fillHeight: true Layout.fillWidth: true onClicked: testDone({'outcome': 'skip'}) } Button { - text: i18n.tr('Fail') - color: UbuntuColors.red + text: 'Fail' Layout.fillHeight: true Layout.fillWidth: true onClicked: testDone({'outcome': 'fail'}) diff --git a/data/resolution_test.qml b/data/resolution_test.qml index cb4b0b5..50f75e7 100644 --- a/data/resolution_test.qml +++ b/data/resolution_test.qml @@ -19,20 +19,26 @@ * along with Checkbox. If not, see <http://www.gnu.org/licenses/>. */ -import QtQuick 2.0 -import QtQuick.Window 2.0 +import QtQuick 2.5 +import QtQuick.Window 2.2 -Rectangle { - border.color: "lime" - border.width: 15 +Window { color: "transparent" - Text { - anchors.centerIn: parent - text: Screen.width + " x " + Screen.height - font.bold: true - font.pointSize: 80 - color: "lime" - smooth: true + visible: true + visibility: Window.FullScreen + Rectangle { + border.color: "lime" + border.width: 15 + color: "transparent" + anchors.fill: parent + Text { + anchors.centerIn: parent + text: Screen.width + " x " + Screen.height + font.bold: true + font.pointSize: 80 + color: "lime" + smooth: true + } } Timer { interval: 5000 @@ -40,4 +46,3 @@ Rectangle { onTriggered: Qt.quit() } } - diff --git a/data/touch_continuous_move_test.qml b/data/touch_continuous_move_test.qml index 1913719..a234169 100644 --- a/data/touch_continuous_move_test.qml +++ b/data/touch_continuous_move_test.qml @@ -17,8 +17,7 @@ along with Checkbox. If not, see <http://www.gnu.org/licenses/>. */ -import QtQuick 2.0 -import Ubuntu.Components 0.1 +import QtQuick 2.5 Rectangle { width: 500 @@ -36,14 +35,14 @@ Rectangle { } Column { - spacing: units.gu(5) + spacing: 10 anchors.centerIn: parent Text { id: legend anchors.horizontalCenter: parent.horizontalCenter text: "Continuously move your mouse cursor" font.bold: true - font.pointSize: 30 + font.pointSize: 12 } Text { id: countdown diff --git a/data/touch_rotate_test.qml b/data/touch_rotate_test.qml index a3b297f..8bdfe79 100644 --- a/data/touch_rotate_test.qml +++ b/data/touch_rotate_test.qml @@ -17,7 +17,6 @@ along with Checkbox. If not, see <http://www.gnu.org/licenses/>. */ import QtQuick 2.0 -import Ubuntu.Components 0.1 Rectangle{ id: mainWindow @@ -66,7 +65,7 @@ Rectangle{ id: instructions z: 1 // So it appears on top height: mainWindow.height / 3 - anchors.margins: units.gu(1.5) + anchors.margins: 10 anchors.top: mainWindow.top anchors.left: mainWindow.left anchors.right: mainWindow.right @@ -75,8 +74,8 @@ Rectangle{ Text { id: text anchors.fill: instructions - anchors.margins: units.gu(1.5) - font.pointSize: units.gu(1.5) + anchors.margins: 5 + font.pointSize: 12 property var timeout: 30 text: "Using two fingers, rotate the blue rectangle so it fits within the red outline. " + "Press ESC to cancel the test at any time. <b>Test will exit automatically in " + @@ -103,11 +102,11 @@ Rectangle{ Rectangle{ id: goalsize anchors.top: instructions.bottom - anchors.margins: units.gu(8) + anchors.margins: 8 anchors.right: mainWindow.right anchors.left: mainWindow.left anchors.bottom: mainWindow.bottom - anchors.bottomMargin: units.gu(10) + anchors.bottomMargin: 10 border.color: "red" border.width: 4 } diff --git a/data/touch_tap_test.qml b/data/touch_tap_test.qml index eff8a1a..c56c930 100644 --- a/data/touch_tap_test.qml +++ b/data/touch_tap_test.qml @@ -1,4 +1,3 @@ -import Ubuntu.Components 0.1 import QtQuick 2.0 Rectangle { @@ -9,23 +8,12 @@ Rectangle { focus:true Keys.onEscapePressed: {Qt.quit()} - Arguments { - id: args - - Argument { - name: "touchpoints" - help: "minimum TouchPoints" - required: true - valueNames: ["TouchPoints"] - } - } - Text { id: text color: "white" - font.pointSize: units.gu(1.5) + font.pointSize: 15 property var timeout: 15 - text: "<p>Touch the screen with "+args.values.touchpoints+" fingers at the same time</p>" + + text: "<p>Touch the screen with "+Qt.application.arguments[2]+" fingers at the same time</p>" + "<p>Press ESC to cancel the test at any time.</p>" + "<p><b>Test will exit automatically in " + timeout + " seconds </b></p>" @@ -84,7 +72,7 @@ Rectangle { enabled: true anchors.fill: parent - minimumTouchPoints: args.values.touchpoints + minimumTouchPoints: Qt.application.arguments[2] maximumTouchPoints: minimumTouchPoints onReleased: { diff --git a/data/touch_zoom_test.qml b/data/touch_zoom_test.qml index 7976ea8..99eed21 100644 --- a/data/touch_zoom_test.qml +++ b/data/touch_zoom_test.qml @@ -17,7 +17,6 @@ along with Checkbox. If not, see <http://www.gnu.org/licenses/>. */ import QtQuick 2.0 -import Ubuntu.Components 0.1 Rectangle{ id: mainWindow @@ -53,7 +52,7 @@ Rectangle{ Rectangle{ id: instructions height: mainWindow.height / 3 - anchors.margins: units.gu(1.5) + anchors.margins: 5 anchors.top: mainWindow.top anchors.left: mainWindow.left anchors.right: mainWindow.right @@ -62,8 +61,8 @@ Rectangle{ Text { id: text anchors.fill: instructions - anchors.margins: units.gu(1.5) - font.pointSize: units.gu(1.5) + anchors.margins: 5 + font.pointSize: 12 property var timeout: 30 text: "Using two fingers, scale the blue rectangle so it is larger than the red outline. " + "Press ESC to cancel the test at any time. <b>Test will exit automatically in " + @@ -91,8 +90,8 @@ Rectangle{ id: goalsize anchors.top: instructions.bottom anchors.bottom: mainWindow.bottom - anchors.bottomMargin: units.gu(5) - anchors.margins: units.gu(8) + anchors.bottomMargin: 5 + anchors.margins: 8 anchors.right: mainWindow.right anchors.left: mainWindow.left border.color: "red" diff --git a/debian/.git-dpm b/debian/.git-dpm index 22cee2d..c4b01cd 100644 --- a/debian/.git-dpm +++ b/debian/.git-dpm @@ -1,8 +1,8 @@ # see git-dpm(1) from git-dpm package -4654c789a2629e1c437ee760afa3f5f9461315b5 -4654c789a2629e1c437ee760afa3f5f9461315b5 -4654c789a2629e1c437ee760afa3f5f9461315b5 -4654c789a2629e1c437ee760afa3f5f9461315b5 -plainbox-provider-checkbox_0.42.0.orig.tar.gz -786b9714b7f8b3ee7064deabbf94ff9754d1aef8 -1737599 +b3f24fb38ac39af2a93399556f622db0216577a5 +b3f24fb38ac39af2a93399556f622db0216577a5 +b3f24fb38ac39af2a93399556f622db0216577a5 +b3f24fb38ac39af2a93399556f622db0216577a5 +plainbox-provider-checkbox_0.43.0.orig.tar.gz +dafb26f938f6649039461ef12875dd5985643c73 +1738426 @@ -5,7 +5,7 @@ from plainbox.provider_manager import N_ setup( name='plainbox-provider-checkbox', namespace='com.canonical.certification', - version="0.42.0", + version="0.43.0", description=N_("Checkbox provider"), gettext_domain='plainbox-provider-checkbox', strict=False, deprecated=False, diff --git a/units/audio/test-plan.pxu b/units/audio/test-plan.pxu index b425c57..a147682 100644 --- a/units/audio/test-plan.pxu +++ b/units/audio/test-plan.pxu @@ -5,18 +5,27 @@ _description: Audio tests See Monitor / Graphic test plans for hybrid-graphic monitor audio tests include: - audio/alsa_info_collect - audio/alsa_info_attachment - audio/list_devices - audio/speaker-headphone-plug-detection certification-status=blocker - audio/microphone-plug-detection certification-status=blocker - audio/playback_headphones certification-status=blocker - audio/alsa_record_playback_external certification-status=blocker - audio/playback_auto certification-status=blocker - audio/alsa_record_playback_internal certification-status=blocker - audio/channels - audio/external-linein - audio/external-lineout +nested_part: + com.canonical.certification::audio-cert-manual + com.canonical.certification::audio-cert-automated + +id: audio-cert-manual +unit: test plan +_name: Audio tests (Manual) +_description: + Audio tests + See Monitor / Graphic test plans for hybrid-graphic monitor audio tests (Manual) +include: + audio/speaker-headphone-plug-detection certification-status=blocker + audio/microphone-plug-detection certification-status=blocker + audio/playback_headphones certification-status=blocker + audio/alsa_record_playback_external certification-status=blocker + audio/playback_auto certification-status=blocker + audio/alsa_record_playback_internal certification-status=blocker + audio/channels + audio/external-linein + audio/external-lineout + id: after-suspend-audio-cert-full unit: test plan diff --git a/units/bluetooth/test-plan.pxu b/units/bluetooth/test-plan.pxu index 7fa002e..ae2c305 100644 --- a/units/bluetooth/test-plan.pxu +++ b/units/bluetooth/test-plan.pxu @@ -1,13 +1,24 @@ id: bluetooth-cert-full unit: test plan _name: Bluetooth tests -_description: Bluetooth tests +_description: + Bluetooth tests include: - bluetooth/detect-output certification-status=blocker - bluetooth/audio-a2dp certification-status=blocker - bluetooth/HID - bluetooth4/HOGP-mouse certification-status=non-blocker - bluetooth4/HOGP-keyboard certification-status=non-blocker +nested_part: + com.canonical.certification::bluetooth-cert-manual + com.canonical.certification::bluetooth-cert-automated + +id: bluetooth-cert-manual +unit: test plan +_name: Bluetooth tests (Manual) +_description: + Bluetooth tests (Manual) +include: + bluetooth/audio-a2dp certification-status=blocker + bluetooth/HID + bluetooth4/HOGP-mouse certification-status=non-blocker + bluetooth4/HOGP-keyboard certification-status=non-blocker + id: bluetooth-cert-automated unit: test plan diff --git a/units/camera/test-plan.pxu b/units/camera/test-plan.pxu index 2cf5550..e4302be 100644 --- a/units/camera/test-plan.pxu +++ b/units/camera/test-plan.pxu @@ -1,12 +1,22 @@ id: camera-cert-full unit: test plan _name: Camera tests -_description: Camera tests +_description: + Camera tests include: - camera/detect certification-status=blocker - camera/still certification-status=blocker - camera/display certification-status=blocker - camera/multiple-resolution-images certification-status=blocker +nested_part: + com.canonical.certification::camera-cert-manual + com.canonical.certification::camera-cert-automated + +id: camera-cert-manual +unit: test plan +_name: Camera tests (Manual) +_description: + Camera tests (Manual) +include: + camera/still certification-status=blocker + camera/display certification-status=blocker + id: after-suspend-camera-cert-full unit: test plan diff --git a/units/cpu/jobs.pxu b/units/cpu/jobs.pxu index bdea273..bf23af7 100644 --- a/units/cpu/jobs.pxu +++ b/units/cpu/jobs.pxu @@ -2,8 +2,7 @@ plugin: shell category_id: com.canonical.plainbox::cpu id: cpu/scaling_test estimated_duration: 150.0 -requires: - package.name == 'fwts' or executable.name == 'fwts' +requires: executable.name == 'fwts' user: root environ: PLAINBOX_SESSION_SHARE LD_LIBRARY_PATH SNAP command: @@ -32,8 +31,7 @@ plugin: shell category_id: com.canonical.plainbox::cpu id: cpu/maxfreq_test estimated_duration: 0.6 -requires: - package.name == 'fwts' or executable.name == 'fwts' +requires: executable.name == 'fwts' cpuinfo.platform in ("i386", "x86_64") user: root environ: LD_LIBRARY_PATH SNAP @@ -117,8 +115,7 @@ _description: plugin:shell id: cpu/cstates estimated_duration: 10.0 -requires: - package.name == 'fwts' or executable.name == 'fwts' +requires: executable.name == 'fwts' user: root _summary: Run C-States tests diff --git a/units/cpu/test-plan.pxu b/units/cpu/test-plan.pxu index 2678c31..c20efe8 100644 --- a/units/cpu/test-plan.pxu +++ b/units/cpu/test-plan.pxu @@ -1,11 +1,23 @@ id: cpu-cert-full unit: test plan _name: CPU tests -_description: CPU tests +_description: + CPU tests +include: +nested_part: + com.canonical.certification::cpu-cert-manual + com.canonical.certification::cpu-cert-automated + +id: cpu-cert-manual +unit: test plan +_name: CPU tests (Manual) +_description: + CPU tests (Manual) include: nested_part: cpu-cert-automated + id: cpu-cert-automated unit: test plan _name: CPU tests (automated) diff --git a/units/esata/test-plan.pxu b/units/esata/test-plan.pxu index b7e59cd..e2cc21c 100644 --- a/units/esata/test-plan.pxu +++ b/units/esata/test-plan.pxu @@ -1,11 +1,30 @@ id: esata-cert-full unit: test plan _name: eSATA tests -_description: eSATA tests +_description: + eSATA tests include: - esata/insert certification-status=blocker - esata/storage-test certification-status=blocker - esata/remove certification-status=blocker +nested_part: + com.canonical.certification::esata-cert-manual + com.canonical.certification::esata-cert-automated + +id: esata-cert-manual +unit: test plan +_name: eSATA tests (Manual) +_description: + eSATA tests (Manual) +include: + esata/insert certification-status=blocker + esata/storage-test certification-status=blocker + esata/remove certification-status=blocker + +id: esata-cert-automated +unit: test plan +_name: eSATA tests (Automated) +_description: + eSATA tests (Automated) +include: + id: esata-cert-blockers unit: test plan diff --git a/units/firewire/test-plan.pxu b/units/firewire/test-plan.pxu index ecb6084..66de049 100644 --- a/units/firewire/test-plan.pxu +++ b/units/firewire/test-plan.pxu @@ -1,11 +1,30 @@ id: firewire-cert-full unit: test plan _name: Firewire tests -_description: Firewire tests +_description: + Firewire tests include: - firewire/insert certification-status=blocker - firewire/storage-test certification-status=blocker - firewire/remove certification-status=blocker +nested_part: + com.canonical.certification::firewire-cert-manual + com.canonical.certification::firewire-cert-automated + +id: firewire-cert-manual +unit: test plan +_name: Firewire tests (Manual) +_description: + Firewire tests (Manual) +include: + firewire/insert certification-status=blocker + firewire/storage-test certification-status=blocker + firewire/remove certification-status=blocker + +id: firewire-cert-automated +unit: test plan +_name: Firewire tests (Automated) +_description: + Firewire tests (Automated) +include: + id: firewire-cert-blockers unit: test plan diff --git a/units/firmware/jobs.pxu b/units/firmware/jobs.pxu index 3095273..5c8dcee 100644 --- a/units/firmware/jobs.pxu +++ b/units/firmware/jobs.pxu @@ -4,7 +4,7 @@ plugin: shell category_id: com.canonical.plainbox::firmware id: firmware/fwts_{name} estimated_duration: 1.2 -requires: package.name == 'fwts' +requires: executable.name == 'fwts' user: root command: checkbox-support-fwts_test -t {name} -l $PLAINBOX_SESSION_SHARE/fwts_{name}.log _description: Run {name} test from Firmware Test Suite. @@ -16,7 +16,7 @@ plugin: attachment category_id: com.canonical.plainbox::firmware id: firmware/fwts_{name}.log estimated_duration: 1.2 -requires: package.name == 'fwts' +requires: executable.name == 'fwts' user: root command: [[ -e $PLAINBOX_SESSION_SHARE/fwts_{name}.log ]] && xz -c $PLAINBOX_SESSION_SHARE/fwts_{name}.log _description: Attach log for FWTS {name} test. @@ -26,8 +26,7 @@ plugin:shell category_id: com.canonical.plainbox::firmware id: firmware/fwts_desktop_diagnosis estimated_duration: 10.0 -requires: - package.name == 'fwts' +requires: executable.name == 'fwts' user: root _description: Run Firmware Test Suite (fwts) QA-concerned desktop-specific diagnosis tests. @@ -40,8 +39,7 @@ plugin:shell category_id: com.canonical.plainbox::firmware id: firmware/fwts_desktop_diagnosis_hwe estimated_duration: 5.0 -requires: - package.name == 'fwts' +requires: executable.name == 'fwts' user: root _description: Run Firmware Test Suite (fwts) HWE-concerned desktop-specific diagnosis tests. diff --git a/units/graphics/jobs.pxu b/units/graphics/jobs.pxu index 261a527..08bc7b3 100644 --- a/units/graphics/jobs.pxu +++ b/units/graphics/jobs.pxu @@ -87,7 +87,7 @@ category_id: com.canonical.plainbox::graphics id: graphics/{index}_screen_resolution_{product_slug} requires: device.category == 'VIDEO' - package.name == 'qmlscene' + executable.name == 'qmlscene' command: qmlscene -qt5 --transparent --fullscreen $PLAINBOX_PROVIDER_DATA/resolution_test.qml estimated_duration: 10.0 _summary: Test default resolution for {vendor} {product} @@ -240,7 +240,7 @@ id: graphics/{index}_rotation_{product_slug} depends: graphics/xorg-version command: source graphics_env {driver} {index} - rotation_test_using_dbus + rotation_test estimated_duration: 20.000 _summary: Test rotation for {vendor} {product} _description: @@ -283,7 +283,7 @@ template-filter: graphics_card.driver in ['amdgpu', 'amdgpu-pro'] plugin: shell category_id: com.canonical.plainbox::graphics id: graphics/{index}_valid_opengl_renderer_{product_slug} -requires: package.name == 'mesa-utils' +requires: executable.name == 'glxinfo' command: renderer=`DRI_PRIME=1 glxinfo | grep "OpenGL re"` echo $renderer @@ -298,7 +298,7 @@ template-resource: graphics_card plugin: user-interact-verify category_id: com.canonical.plainbox::graphics id: graphics/{index}_glxgears_{product_slug} -requires: package.name == 'mesa-utils' +requires: executable.name == 'glxgears' command: source graphics_env {driver} {index} glxgears @@ -319,7 +319,7 @@ template-resource: graphics_card plugin: user-interact-verify category_id: com.canonical.plainbox::graphics id: graphics/{index}_glxgears_fullscreen_{product_slug} -requires: package.name == 'mesa-utils' +requires: executable.name == 'glxgears' command: source graphics_env {driver} {index} glxgears -fullscreen @@ -340,7 +340,7 @@ template-resource: graphics_card plugin: shell category_id: com.canonical.plainbox::graphics id: graphics/{index}_3d_window_open_close_{product_slug} -requires: package.name == 'mesa-utils' +requires: executable.name == 'glxgears' command: window_test -t open-close -i 10 estimated_duration: 60.525 _description: Open and close a 3D window multiple times on the {vendor} {product} video card @@ -351,7 +351,7 @@ template-resource: graphics_card plugin: shell category_id: com.canonical.plainbox::graphics id: graphics/{index}_3d_window_suspend_resume_{product_slug} -requires: package.name == 'mesa-utils' +requires: executable.name == 'glxgears' command: window_test -t suspend-resume -i 10 estimated_duration: 121.00 _description: Open, suspend resume and close a 3D window multiple times on the {vendor} {product} video card @@ -362,7 +362,7 @@ template-resource: graphics_card plugin: shell category_id: com.canonical.plainbox::graphics id: graphics/{index}_multi_3d_windows_open_close_{product_slug} -requires: package.name == 'mesa-utils' +requires: executable.name == 'glxgears' command: window_test -t open-close-multi -i 10 -w 4 estimated_duration: 60.000 _description: Open and close 4 3D windows multiple times on the {vendor} {product} video card @@ -374,8 +374,8 @@ plugin: shell category_id: com.canonical.plainbox::graphics id: graphics/{index}_3d_window_move_{product_slug} requires: - package.name == 'mesa-utils' - package.name == 'wmctrl' + executable.name == 'glxgears' + executable.name == 'wmctrl' command: window_test -t move estimated_duration: 50.000 _description: Move a 3D window around the screen on the {vendor} {product} video card diff --git a/units/graphics/legacy.pxu b/units/graphics/legacy.pxu index 5e8fe82..74153cc 100644 --- a/units/graphics/legacy.pxu +++ b/units/graphics/legacy.pxu @@ -47,7 +47,7 @@ category_id: com.canonical.plainbox::graphics id: graphics/screen-resolution requires: device.category == 'VIDEO' - package.name == 'qmlscene' + executable.name == 'qmlscene' command: qmlscene -qt5 --transparent --fullscreen $PLAINBOX_PROVIDER_DATA/resolution_test.qml estimated_duration: 10.0 _summary: Test default resolution @@ -174,7 +174,7 @@ _summary: Test Unity 3D support plugin: user-interact-verify category_id: com.canonical.plainbox::graphics id: graphics/glxgears -requires: package.name == 'mesa-utils' +requires: executable.name == 'glxgears' command: glxgears; true _summary: Test that glxgears works _description: @@ -190,7 +190,7 @@ _description: plugin: shell category_id: com.canonical.plainbox::graphics id: graphics/3d_window_open_close -requires: package.name == 'mesa-utils' +requires: executable.name == 'glxgears' command: window_test -t open-close -i 10 estimated_duration: 60.525 _summary: Test 3D window open/close @@ -199,7 +199,7 @@ _description: Open and close a 3D window multiple times plugin: shell category_id: com.canonical.plainbox::graphics id: graphics/3d_window_suspend_resume -requires: package.name == 'mesa-utils' +requires: executable.name == 'glxgears' command: window_test -t suspend-resume -i 10 estimated_duration: 121.00 _description: Open, suspend resume and close a 3D window multiple times @@ -208,7 +208,7 @@ _summary: Test a 3D window with suspend/resume plugin: shell category_id: com.canonical.plainbox::graphics id: graphics/multi_3d_windows_open_close -requires: package.name == 'mesa-utils' +requires: executable.name == 'glxgears' command: window_test -t open-close-multi -i 10 -w 4 estimated_duration: 60.000 _description: Open and close 4 3D windows multiple times @@ -217,7 +217,7 @@ _summary: Test Multi 3D window open/close plugin: shell category_id: com.canonical.plainbox::graphics id: graphics/3d_window_move -requires: package.name == 'mesa-utils' +requires: executable.name == 'glxgears' command: window_test -t move estimated_duration: 50.000 _description: Move a 3D window around the screen diff --git a/units/graphics/test-plan.pxu b/units/graphics/test-plan.pxu index a85b194..9c1f294 100644 --- a/units/graphics/test-plan.pxu +++ b/units/graphics/test-plan.pxu @@ -1,42 +1,88 @@ id: graphics-integrated-gpu-cert-full unit: test plan _name: Graphics tests (integrated GPU) -_description: Graphics tests (integrated GPU) +_description: + Graphics tests (integrated GPU) include: - miscellanea/chvt - graphics/xorg-version certification-status=blocker - graphics/xorg-failsafe certification-status=blocker - graphics/xorg-process certification-status=blocker - graphics/VESA_drivers_not_in_use certification-status=blocker - graphics/1_maximum_resolution_.* certification-status=blocker - graphics/1_glxgears_.* certification-status=blocker - graphics/1_driver_version_.* certification-status=blocker - graphics/1_compiz_check_.* certification-status=blocker - graphics/1_rotation_.* certification-status=blocker - graphics/1_video_.* certification-status=blocker - graphics/1_minimum_resolution_.* - graphics/1_cycle_resolution_.* certification-status=non-blocker +bootstrap_include: + graphics_card +nested_part: + com.canonical.certification::graphics-integrated-gpu-cert-manual + com.canonical.certification::graphics-integrated-gpu-cert-automated + +id: graphics-integrated-gpu-cert-manual +unit: test plan +_name: Graphics tests (integrated GPU) (Manual) +_description: + Graphics tests (integrated GPU) (Manual) +include: + miscellanea/chvt + graphics/1_maximum_resolution_.* certification-status=blocker + graphics/1_glxgears_.* certification-status=blocker + graphics/1_rotation_.* certification-status=blocker + graphics/1_video_.* certification-status=blocker + graphics/1_cycle_resolution_.* certification-status=non-blocker +bootstrap_include: + graphics_card + +id: graphics-integrated-gpu-cert-automated +unit: test plan +_name: Graphics tests (integrated GPU) (Automated) +_description: + Graphics tests (integrated GPU) (Automated) +include: + graphics/xorg-version certification-status=blocker + graphics/xorg-failsafe certification-status=blocker + graphics/xorg-process certification-status=blocker + graphics/VESA_drivers_not_in_use certification-status=blocker + graphics/1_driver_version_.* certification-status=blocker + graphics/1_compiz_check_.* certification-status=blocker + graphics/1_minimum_resolution_.* bootstrap_include: graphics_card + id: graphics-discrete-gpu-cert-full unit: test plan _name: Graphics tests (discrete GPU) -_description: Graphics tests (discrete GPU) +_description: + Graphics tests (discrete GPU) include: - graphics/2_switch_card_.*_xenial certification-status=blocker - graphics/2_maximum_resolution_.* certification-status=blocker - graphics/2_valid_opengl_renderer_.* certification-status=blocker - graphics/2_glxgears_.* certification-status=blocker - graphics/2_driver_version_.* certification-status=blocker - graphics/2_compiz_check_.* certification-status=blocker - graphics/2_rotation_.* certification-status=blocker - graphics/2_video_.* certification-status=blocker - graphics/2_minimum_resolution_.* - graphics/2_cycle_resolution_.* certification-status=non-blocker +bootstrap_include: + graphics_card +nested_part: + com.canonical.certification::graphics-discrete-gpu-cert-manual + com.canonical.certification::graphics-discrete-gpu-cert-automated + +id: graphics-discrete-gpu-cert-manual +unit: test plan +_name: Graphics tests (discrete GPU) (Manual) +_description: + Graphics tests (discrete GPU) (Manual) +include: + graphics/2_switch_card_.*_xenial certification-status=blocker + graphics/2_maximum_resolution_.* certification-status=blocker + graphics/2_glxgears_.* certification-status=blocker + graphics/2_rotation_.* certification-status=blocker + graphics/2_video_.* certification-status=blocker + graphics/2_cycle_resolution_.* certification-status=non-blocker bootstrap_include: graphics_card +id: graphics-discrete-gpu-cert-automated +unit: test plan +_name: Graphics tests (discrete GPU) (Automated) +_description: + Graphics tests (discrete GPU) (Automated) +include: + graphics/2_valid_opengl_renderer_.* certification-status=blocker + graphics/2_driver_version_.* certification-status=blocker + graphics/2_compiz_check_.* certification-status=blocker + graphics/2_minimum_resolution_.* +bootstrap_include: + graphics_card + + id: after-suspend-graphics-integrated-gpu-cert-full unit: test plan _name: After suspend tests (integrated GPU) @@ -58,6 +104,7 @@ include: suspend/1_video_after_suspend_.*_xenial certification-status=blocker suspend/1_cycle_resolutions_after_suspend_.*_xenial certification-status=non-blocker suspend/1_xrandr_screens_after_suspend.tar.gz_xenial + after-suspend-manual-monitor/1_dim_brightness_.* certification-status=blocker id: after-suspend-graphics-discrete-gpu-cert-full unit: test plan @@ -76,6 +123,7 @@ include: suspend/2_video_after_suspend_.*_xenial certification-status=blocker suspend/2_cycle_resolutions_after_suspend_.*_xenial certification-status=non-blocker suspend/2_xrandr_screens_after_suspend_.*.tar.gz_xenial + after-suspend-manual-monitor/2_dim_brightness_.* certification-status=blocker id: graphics-integrated-gpu-cert-blockers unit: test plan @@ -128,6 +176,7 @@ include: suspend/1_display_after_suspend_.*_xenial certification-status=blocker suspend/1_glxgears_after_suspend_.*_xenial certification-status=blocker suspend/1_video_after_suspend_.*_xenial certification-status=blocker + after-suspend-manual-monitor/1_dim_brightness_.* certification-status=blocker id: after-suspend-graphics-discrete-gpu-cert-blockers unit: test plan @@ -142,3 +191,4 @@ include: suspend/2_display_after_suspend_.*_xenial certification-status=blocker suspend/2_glxgears_after_suspend_.*_xenial certification-status=blocker suspend/2_video_after_suspend_.*_xenial certification-status=blocker + after-suspend-manual-monitor/2_dim_brightness_.* certification-status=blocker diff --git a/units/info/test-plan.pxu b/units/info/test-plan.pxu index c01d47e..e54b443 100644 --- a/units/info/test-plan.pxu +++ b/units/info/test-plan.pxu @@ -1,32 +1,55 @@ id: info-attachment-cert-full unit: test plan _name: Info attachment jobs -_description: Info attachment jobs +_description: + Info attachment jobs include: - acpi_sleep_attachment - codecs_attachment - cpuinfo_attachment - dkms_info_attachment - dmesg_attachment - dmi_attachment - dmidecode_attachment - efi_attachment - info/buildstamp - info/disk_partitions - info/hdparm_.*.txt - info/touchpad_driver - installer_debug.gz - kernel_cmdline_attachment - lsmod_attachment - lspci_attachment - lspci_standard_config_attachment - lsusb_attachment - meminfo_attachment - modinfo_attachment - modprobe_attachment - modules_attachment - sysctl_attachment - sysfs_attachment - udev_attachment +bootstrap_include: + device +nested_part: + com.canonical.certification::info-attachment-cert-manual + com.canonical.certification::info-attachment-cert-automated + +id: info-attachment-cert-manual +unit: test plan +_name: Info attachment jobs (Manual) +_description: + Info attachment jobs (Manual) +include: +bootstrap_include: + device + +id: info-attachment-cert-automated +unit: test plan +_name: Info attachment jobs (Automated) +_description: + Info attachment jobs (Automated) +include: + acpi_sleep_attachment + codecs_attachment + cpuinfo_attachment + dkms_info_attachment + dmesg_attachment + dmi_attachment + dmidecode_attachment + efi_attachment + info/buildstamp + info/disk_partitions + info/hdparm_.*.txt + info/touchpad_driver + installer_debug.gz + kernel_cmdline_attachment + lsmod_attachment + lspci_attachment + lspci_standard_config_attachment + lsusb_attachment + meminfo_attachment + modinfo_attachment + modprobe_attachment + modules_attachment + sysctl_attachment + sysfs_attachment + udev_attachment bootstrap_include: device + diff --git a/units/input/test-plan.pxu b/units/input/test-plan.pxu index 8033883..ef9cf75 100644 --- a/units/input/test-plan.pxu +++ b/units/input/test-plan.pxu @@ -1,12 +1,29 @@ id: input-cert-full unit: test plan _name: Input tests -_description: Input tests +_description: + Input tests include: - input/accelerometer certification-status=non-blocker - input/pointing_.* certification-status=blocker - input/clicking_.* certification-status=blocker - input/keyboard certification-status=blocker +nested_part: + com.canonical.certification::input-cert-manual + com.canonical.certification::input-cert-automated + +id: input-cert-manual +unit: test plan +_name: Input tests (Manual) +_description: + Input tests (Manual) +include: + input/accelerometer certification-status=non-blocker + input/keyboard certification-status=blocker + +id: input-cert-automated +unit: test plan +_name: Input tests (Automated) +_description: + Input tests (Automated) +include: + id: after-suspend-input-cert-full unit: test plan diff --git a/units/keys/test-plan.pxu b/units/keys/test-plan.pxu index 4fd7092..c75a32a 100644 --- a/units/keys/test-plan.pxu +++ b/units/keys/test-plan.pxu @@ -4,17 +4,35 @@ _name: Special keys tests _description: Special keys tests (w/o sleep/hibernate keys, see Power Management test plans) include: - keys/lock-screen certification-status=blocker - keys/super certification-status=blocker - keys/battery-info certification-status=blocker - keys/brightness certification-status=blocker - keys/media-control certification-status=blocker - keys/mute certification-status=blocker - keys/volume certification-status=blocker - keys/video-out certification-status=blocker - keys/wireless certification-status=blocker - keys/keyboard-backlight certification-status=blocker - keys/microphone-mute certification-status=blocker +nested_part: + com.canonical.certification::keys-cert-manual + com.canonical.certification::keys-cert-automated + +id: keys-cert-manual +unit: test plan +_name: Special keys tests (Manual) +_description: + Special keys tests (w/o sleep/hibernate keys, see Power Management test plans) (Manual) +include: + keys/lock-screen certification-status=blocker + keys/super certification-status=blocker + keys/battery-info certification-status=blocker + keys/brightness certification-status=blocker + keys/media-control certification-status=blocker + keys/mute certification-status=blocker + keys/volume certification-status=blocker + keys/video-out certification-status=blocker + keys/wireless certification-status=blocker + keys/keyboard-backlight certification-status=blocker + keys/microphone-mute certification-status=blocker + +id: keys-cert-automated +unit: test plan +_name: Special keys tests (Automated) +_description: + Special keys tests (w/o sleep/hibernate keys, see Power Management test plans) (Automated) +include: + id: after-suspend-keys-cert-full unit: test plan diff --git a/units/led/test-plan.pxu b/units/led/test-plan.pxu index f43a3ca..88d38a6 100644 --- a/units/led/test-plan.pxu +++ b/units/led/test-plan.pxu @@ -2,19 +2,45 @@ id: led-cert-full unit: test plan _name: LED tests _description: + LED tests + Notes: - led/power-blink-suspend and led/suspend are used later in the + power management testplan just after a number of suspend tests. + - led/wireless is redundant given that we have led/wlan and + led/bluetooth already. +include: +nested_part: + com.canonical.certification::led-cert-manual + com.canonical.certification::led-cert-automated + +id: led-cert-manual +unit: test plan +_name: LED tests (Manual) +_description: LED tests Notes: - led/power-blink-suspend and led/suspend are used later in the power management testplan just after a number of suspend tests. - led/wireless is redundant given that we have led/wlan and - led/bluetooth already. + led/bluetooth already. (Manual) include: - led/battery-charged - led/battery-charging - led/battery-low - led/camera - led/caps-lock - led/power - led/touchpad + led/battery-charged + led/battery-charging + led/battery-low + led/camera + led/caps-lock + led/power + led/touchpad + +id: led-cert-automated +unit: test plan +_name: LED tests (Automated) +_description: + LED tests + Notes: - led/power-blink-suspend and led/suspend are used later in the + power management testplan just after a number of suspend tests. + - led/wireless is redundant given that we have led/wlan and + led/bluetooth already. (Automated) +include: + id: after-suspend-led-cert-full unit: test plan diff --git a/units/mediacard/test-plan.pxu b/units/mediacard/test-plan.pxu index 087e556..58d3f07 100644 --- a/units/mediacard/test-plan.pxu +++ b/units/mediacard/test-plan.pxu @@ -1,14 +1,33 @@ id: mediacard-cert-full unit: test plan _name: Mediacard tests -_description: Mediacard tests +_description: + Mediacard tests include: - mediacard/sd-insert certification-status=blocker - mediacard/sd-storage certification-status=blocker - mediacard/sd-remove certification-status=blocker - mediacard/sdhc-insert certification-status=blocker - mediacard/sdhc-storage certification-status=blocker - mediacard/sdhc-remove certification-status=blocker +nested_part: + com.canonical.certification::mediacard-cert-manual + com.canonical.certification::mediacard-cert-automated + +id: mediacard-cert-manual +unit: test plan +_name: Mediacard tests (Manual) +_description: + Mediacard tests (Manual) +include: + mediacard/sd-insert certification-status=blocker + mediacard/sd-storage certification-status=blocker + mediacard/sd-remove certification-status=blocker + mediacard/sdhc-insert certification-status=blocker + mediacard/sdhc-storage certification-status=blocker + mediacard/sdhc-remove certification-status=blocker + +id: mediacard-cert-automated +unit: test plan +_name: Mediacard tests (Automated) +_description: + Mediacard tests (Automated) +include: + id: after-suspend-mediacard-cert-full unit: test plan diff --git a/units/miscellanea/jobs.pxu b/units/miscellanea/jobs.pxu index 10df43d..2071f06 100644 --- a/units/miscellanea/jobs.pxu +++ b/units/miscellanea/jobs.pxu @@ -67,8 +67,7 @@ plugin: shell category_id: com.canonical.plainbox::miscellanea id: miscellanea/fwts_test estimated_duration: 1.2 -requires: - package.name == 'fwts' +requires: executable.name == 'fwts' user: root _description: Run Firmware Test Suite (fwts) automated tests. @@ -93,8 +92,7 @@ command: fwupdate -s plugin: attachment category_id: com.canonical.plainbox::miscellanea id: miscellanea/fwts_results.log -requires: - package.name == 'fwts' +requires: executable.name == 'fwts' command: [[ -e ${PLAINBOX_SESSION_SHARE}/fwts_results.log ]] && xz -c ${PLAINBOX_SESSION_SHARE}/fwts_results.log _description: Attaches the miscellanes/fwts_test results log to the submission. @@ -103,8 +101,7 @@ plugin: attachment category_id: com.canonical.plainbox::miscellanea estimated_duration: 0.5 id: miscellanea/fwts_results_hwe.log.gz -requires: - package.name == 'fwts' +requires: executable.name == 'fwts' command: [ -f $PLAINBOX_SESSION_SHARE/fwts_results_hwe.log ] && gzip -c $PLAINBOX_SESSION_SHARE/fwts_results_hwe.log _description: Attaches the FWTS results log to the submission (to HWE) @@ -292,8 +289,7 @@ _description: plugin:shell id: miscellanea/oops estimated_duration: 10.0 -requires: - package.name == 'fwts' +requires: executable.name == 'fwts' user: root _description: Run Firmware Test Suite (fwts) oops tests. diff --git a/units/miscellanea/test-plan.pxu b/units/miscellanea/test-plan.pxu index adf771f..758386e 100644 --- a/units/miscellanea/test-plan.pxu +++ b/units/miscellanea/test-plan.pxu @@ -1,11 +1,23 @@ id: misc-client-cert-full unit: test plan _name: Miscellaneous client tests -_description: Miscellaneous client tests (fwts, dmi, oops) +_description: + Miscellaneous client tests (fwts, dmi, oops) +include: +nested_part: + com.canonical.certification::misc-client-cert-manual + com.canonical.certification::misc-client-cert-automated + +id: misc-client-cert-manual +unit: test plan +_name: Miscellaneous client tests (Manual) +_description: + Miscellaneous client tests (fwts, dmi, oops) (Manual) include: nested_part: misc-client-cert-automated + id: misc-client-cert-automated unit: test plan _name: Miscellaneous client tests (automated) diff --git a/units/mobilebroadband/test-plan.pxu b/units/mobilebroadband/test-plan.pxu index 166a297..dd086c7 100644 --- a/units/mobilebroadband/test-plan.pxu +++ b/units/mobilebroadband/test-plan.pxu @@ -1,11 +1,23 @@ id: mobilebroadband-cert-full unit: test plan _name: Mobile broadband tests -_description: Mobile broadband tests +_description: + Mobile broadband tests +include: +nested_part: + com.canonical.certification::mobilebroadband-cert-manual + com.canonical.certification::mobilebroadband-cert-automated + +id: mobilebroadband-cert-manual +unit: test plan +_name: Mobile broadband tests (Manual) +_description: + Mobile broadband tests (Manual) include: nested_part: mobilebroadband-cert-automated + id: mobilebroadband-cert-automated unit: test plan _name: Mobile broadband tests (automated) diff --git a/units/monitor/jobs.pxu b/units/monitor/jobs.pxu index 36425db..6c1ac58 100644 --- a/units/monitor/jobs.pxu +++ b/units/monitor/jobs.pxu @@ -129,6 +129,7 @@ requires: dmi.product in ['Notebook','Laptop','Portable','All In One','All-In-On plugin: user-interact-verify category_id: com.canonical.plainbox::monitor user: root +flags: also-after-suspend-manual command: brightness_test _purpose: This test will test changes to screen brightness diff --git a/units/monitor/test-plan.pxu b/units/monitor/test-plan.pxu index 10976c9..344b304 100644 --- a/units/monitor/test-plan.pxu +++ b/units/monitor/test-plan.pxu @@ -1,51 +1,97 @@ id: monitor-integrated-gpu-cert-full unit: test plan _name: Monitor tests (integrated GPU) -_description: Monitor tests (integrated GPU) +_description: + Monitor tests (integrated GPU) include: - monitor/1_powersaving_.* certification-status=blocker - monitor/1_dim_brightness_.* certification-status=blocker - monitor/1_displayport_.* certification-status=blocker - monitor/1_type-c_displayport_.* certification-status=non-blocker - audio/1_playback_displayport_.* certification-status=blocker - audio/1_playback_type-c_displayport_.* certification-status=non-blocker - monitor/1_dvi_.* certification-status=blocker - monitor/1_hdmi_.* certification-status=blocker - audio/1_playback_hdmi_.* certification-status=blocker - monitor/1_thunderbolt_.* certification-status=blocker - monitor/1_thunderbolt3_.* certification-status=non-blocker - audio/1_playback_thunderbolt_.* certification-status=blocker - audio/1_playback_thunderbolt3_.* certification-status=non-blocker - thunderbolt/daisy-chain certification-status=blocker - thunderbolt3/daisy-chain certification-status=non-blocker - monitor/1_vga_.* certification-status=blocker - monitor/1_multi-head_.* certification-status=blocker +bootstrap_include: + graphics_card +nested_part: + com.canonical.certification::monitor-integrated-gpu-cert-manual + com.canonical.certification::monitor-integrated-gpu-cert-automated + +id: monitor-integrated-gpu-cert-manual +unit: test plan +_name: Monitor tests (integrated GPU) (Manual) +_description: + Monitor tests (integrated GPU) (Manual) +include: + monitor/1_powersaving_.* certification-status=blocker + monitor/1_dim_brightness_.* certification-status=blocker + monitor/1_displayport_.* certification-status=blocker + monitor/1_type-c_displayport_.* certification-status=blocker + audio/1_playback_displayport_.* certification-status=blocker + audio/1_playback_type-c_displayport_.* certification-status=blocker + monitor/1_dvi_.* certification-status=blocker + monitor/1_hdmi_.* certification-status=blocker + audio/1_playback_hdmi_.* certification-status=blocker + monitor/1_thunderbolt_.* certification-status=blocker + monitor/1_thunderbolt3_.* certification-status=non-blocker + audio/1_playback_thunderbolt_.* certification-status=blocker + audio/1_playback_thunderbolt3_.* certification-status=non-blocker + thunderbolt/daisy-chain certification-status=blocker + thunderbolt3/daisy-chain certification-status=non-blocker + monitor/1_vga_.* certification-status=blocker + monitor/1_multi-head_.* certification-status=blocker bootstrap_include: graphics_card +id: monitor-integrated-gpu-cert-automated +unit: test plan +_name: Monitor tests (integrated GPU) (Automated) +_description: + Monitor tests (integrated GPU) (Automated) +include: +bootstrap_include: + graphics_card + + id: monitor-discrete-gpu-cert-full unit: test plan _name: Monitor tests (discrete GPU) -_description: Monitor tests (discrete GPU) +_description: + Monitor tests (discrete GPU) +include: +bootstrap_include: + graphics_card +nested_part: + com.canonical.certification::monitor-discrete-gpu-cert-manual + com.canonical.certification::monitor-discrete-gpu-cert-automated + +id: monitor-discrete-gpu-cert-manual +unit: test plan +_name: Monitor tests (discrete GPU) (Manual) +_description: + Monitor tests (discrete GPU) (Manual) +include: + monitor/2_powersaving_.* certification-status=blocker + monitor/2_dim_brightness_.* certification-status=blocker + monitor/2_displayport_.* certification-status=blocker + monitor/2_type-c_displayport_.* certification-status=blocker + audio/2_playback_displayport_.* certification-status=blocker + audio/2_playback_type-c_displayport_.* certification-status=blocker + monitor/2_dvi_.* certification-status=blocker + monitor/2_hdmi_.* certification-status=blocker + audio/2_playback_hdmi_.* certification-status=blocker + monitor/2_thunderbolt_.* certification-status=blocker + monitor/2_thunderbolt3_.* certification-status=non-blocker + audio/2_playback_thunderbolt_.* certification-status=blocker + audio/2_playback_thunderbolt3_.* certification-status=non-blocker + monitor/2_vga_.* certification-status=blocker + monitor/2_multi-head_.* certification-status=blocker +bootstrap_include: + graphics_card + +id: monitor-discrete-gpu-cert-automated +unit: test plan +_name: Monitor tests (discrete GPU) (Automated) +_description: + Monitor tests (discrete GPU) (Automated) include: - monitor/2_powersaving_.* certification-status=blocker - monitor/2_dim_brightness_.* certification-status=blocker - monitor/2_displayport_.* certification-status=blocker - monitor/2_type-c_displayport_.* certification-status=non-blocker - audio/2_playback_displayport_.* certification-status=blocker - audio/2_playback_type-c_displayport_.* certification-status=non-blocker - monitor/2_dvi_.* certification-status=blocker - monitor/2_hdmi_.* certification-status=blocker - audio/2_playback_hdmi_.* certification-status=blocker - monitor/2_thunderbolt_.* certification-status=blocker - monitor/2_thunderbolt3_.* certification-status=non-blocker - audio/2_playback_thunderbolt_.* certification-status=blocker - audio/2_playback_thunderbolt3_.* certification-status=non-blocker - monitor/2_vga_.* certification-status=blocker - monitor/2_multi-head_.* certification-status=blocker bootstrap_include: graphics_card + id: monitor-integrated-gpu-cert-blockers unit: test plan _name: Monitor tests (integrated GPU, certification blockers only) @@ -54,7 +100,9 @@ include: monitor/1_powersaving_.* certification-status=blocker monitor/1_dim_brightness_.* certification-status=blocker monitor/1_displayport_.* certification-status=blocker + monitor/1_type-c_displayport_.* certification-status=blocker audio/1_playback_displayport_.* certification-status=blocker + audio/1_playback_type-c_displayport_.* certification-status=blocker monitor/1_dvi_.* certification-status=blocker monitor/1_hdmi_.* certification-status=blocker audio/1_playback_hdmi_.* certification-status=blocker @@ -74,7 +122,9 @@ include: monitor/2_powersaving_.* certification-status=blocker monitor/2_dim_brightness_.* certification-status=blocker monitor/2_displayport_.* certification-status=blocker + monitor/2_type-c_displayport_.* certification-status=blocker audio/2_playback_displayport_.* certification-status=blocker + audio/2_playback_type-c_displayport_.* certification-status=blocker monitor/2_dvi_.* certification-status=blocker monitor/2_hdmi_.* certification-status=blocker audio/2_playback_hdmi_.* certification-status=blocker diff --git a/units/networking/test-plan.pxu b/units/networking/test-plan.pxu index ade662a..e9e61e2 100644 --- a/units/networking/test-plan.pxu +++ b/units/networking/test-plan.pxu @@ -12,7 +12,7 @@ unit: test plan _name: Networking tests (manual) _description: Networking tests (manual) include: - networking/info.* certification-status=blocker + networking/info_device.* certification-status=blocker id: networking-cert-automated unit: test plan @@ -30,5 +30,5 @@ _description: Networking tests (certification blockers only) include: ethernet/detect certification-status=blocker networking/gateway_ping certification-status=blocker - networking/info.* certification-status=blocker + networking/info_device.* certification-status=blocker networking/ntp certification-status=blocker diff --git a/units/optical/test-plan.pxu b/units/optical/test-plan.pxu index 0cc6ab3..2e53d79 100644 --- a/units/optical/test-plan.pxu +++ b/units/optical/test-plan.pxu @@ -1,14 +1,37 @@ id: optical-cert-full unit: test plan _name: Optical drive tests -_description: Optical drive tests +_description: + Optical drive tests include: - optical/detect certification-status=blocker - optical/read_.* certification-status=blocker - optical/bluray-read_.* certification-status=blocker +bootstrap_include: + device +nested_part: + com.canonical.certification::optical-cert-manual + com.canonical.certification::optical-cert-automated + +id: optical-cert-manual +unit: test plan +_name: Optical drive tests (Manual) +_description: + Optical drive tests (Manual) +include: + optical/read_.* certification-status=blocker + optical/bluray-read_.* certification-status=blocker bootstrap_include: device +id: optical-cert-automated +unit: test plan +_name: Optical drive tests (Automated) +_description: + Optical drive tests (Automated) +include: + optical/detect certification-status=blocker +bootstrap_include: + device + + id: optical-cert-blockers unit: test plan _name: Optical drive tests (certification blockers only) diff --git a/units/power-management/jobs.pxu b/units/power-management/jobs.pxu index ce252fc..be640c3 100644 --- a/units/power-management/jobs.pxu +++ b/units/power-management/jobs.pxu @@ -19,8 +19,7 @@ environ: PLAINBOX_SESSION_SHARE estimated_duration: 25.0 user: root _description: Test ACPI Wakealarm (fwts wakealarm) -requires: - package.name == 'fwts' or executable.name == 'fwts' +requires: executable.name == 'fwts' command: checkbox-support-fwts_test -f aborted -t wakealarm -l $PLAINBOX_SESSION_SHARE/fwts-wakealarm.log plugin: attachment @@ -38,10 +37,9 @@ estimated_duration: 120.0 depends: power-management/fwts_wakealarm user: root environ: PLAINBOX_SESSION_SHARE -requires: - package.name == 'fwts' +requires: executable.name == 'fwts' command: pm_test --silent --checkbox-respawn-cmd $PLAINBOX_SESSION_SHARE/__respawn_checkbox poweroff --log-level=debug --log-dir=$PLAINBOX_SESSION_SHARE -flags: noreturn autorestart +flags: noreturn _description: This test will check the system's ability to power-off and boot. @@ -61,10 +59,9 @@ id: power-management/reboot estimated_duration: 120.0 user: root environ: PLAINBOX_SESSION_SHARE -requires: - package.name == 'fwts' +requires: executable.name == 'fwts' command: pm_test --silent --checkbox-respawn-cmd $PLAINBOX_SESSION_SHARE/__respawn_checkbox reboot --log-level=debug --log-dir=$PLAINBOX_SESSION_SHARE -flags: noreturn autorestart +flags: noreturn _description: This test will check the system's ability to reboot cleanly. @@ -196,7 +193,7 @@ estimated_duration: 140.0 user: root requires: package.name == 'upower' - package.name == 'fwts' + executable.name == 'fwts' depends: power-management/unplug_ac _description: Checks the battery drain during suspend. Reports time and capacity until empty. command: diff --git a/units/stress/jobs.pxu b/units/stress/jobs.pxu index 2a2a7d0..8d25af9 100644 --- a/units/stress/jobs.pxu +++ b/units/stress/jobs.pxu @@ -74,7 +74,7 @@ depends: suspend/suspend_advanced requires: executable.name == 'x-terminal-emulator' -flags: noreturn autorestart +flags: noreturn user: root command: pm_test reboot --checkbox-respawn-cmd $PLAINBOX_SESSION_SHARE/__respawn_checkbox --fwts --log-level=debug --log-dir=$PLAINBOX_SESSION_SHARE --suspends-before-reboot=30 -r 3 --silent @@ -222,10 +222,9 @@ plugin: shell category_id: com.canonical.plainbox::stress id: stress/reboot estimated_duration: 4500.0 -requires: - package.name == 'fwts' +requires: executable.name == 'fwts' command: pm_test --checkbox-respawn-cmd $PLAINBOX_SESSION_SHARE/__respawn_checkbox -r 100 --silent --log-level=notset reboot --log-dir=$PLAINBOX_SESSION_SHARE -flags: noreturn autorestart +flags: noreturn user: root environ: PLAINBOX_SESSION_SHARE _description: @@ -243,11 +242,10 @@ command: plugin: shell category_id: com.canonical.plainbox::stress id: stress/reboot_30 -requires: - package.name == 'fwts' +requires: executable.name == 'fwts' executable.name == 'x-terminal-emulator' command: pm_test --checkbox-respawn-cmd $PLAINBOX_SESSION_SHARE/__respawn_checkbox -r 30 --silent --log-level=notset reboot --log-dir=$PLAINBOX_SESSION_SHARE -flags: noreturn autorestart +flags: noreturn estimated_duration: 2700 user: root environ: PLAINBOX_SESSION_SHARE @@ -267,10 +265,10 @@ category_id: com.canonical.plainbox::stress id: stress/poweroff estimated_duration: 4500.0 requires: - package.name == 'fwts' + executable.name == 'fwts' executable.name == 'x-terminal-emulator' -command: pm_test --checkbox-respawn-cmd $PLAINBOX_SESSION_SHARE/__respawn_checkbox --r 100 --silent --log-level=notset poweroff --log-dir=$PLAINBOX_SESSION_SHARE -flags: noreturn autorestart +command: pm_test --checkbox-respawn-cmd $PLAINBOX_SESSION_SHARE/__respawn_checkbox -r 100 --silent --log-level=notset poweroff --log-dir=$PLAINBOX_SESSION_SHARE +flags: noreturn user: root environ: PLAINBOX_SESSION_SHARE _description: @@ -289,10 +287,10 @@ plugin: shell category_id: com.canonical.plainbox::stress id: stress/poweroff_30 requires: - package.name == 'fwts' + executable.name == 'fwts' executable.name == 'x-terminal-emulator' -command: pm_test --checkbox-respawn-cmd $PLAINBOX_SESSION_SHARE/__respawn_checkbox --r 30 --silent --log-level=notset poweroff --log-dir=$PLAINBOX_SESSION_SHARE -flags: noreturn autorestart +command: pm_test --checkbox-respawn-cmd $PLAINBOX_SESSION_SHARE/__respawn_checkbox -r 30 --wakeup 150 --silent --log-level=notset poweroff --log-dir=$PLAINBOX_SESSION_SHARE +flags: noreturn estimated_duration: 3600 user: root environ: PLAINBOX_SESSION_SHARE diff --git a/units/stress/test-plan.pxu b/units/stress/test-plan.pxu index 5581401..659e04e 100644 --- a/units/stress/test-plan.pxu +++ b/units/stress/test-plan.pxu @@ -1,11 +1,27 @@ id: stress-cert-full unit: test plan _name: Stress tests -_description: Stress tests +_description: + Stress tests include: nested_part: + com.canonical.certification::stress-cert-manual + com.canonical.certification::stress-cert-automated + +id: stress-cert-manual +unit: test plan +_name: Stress tests (Manual) +_description: + Stress tests (Manual) +include: + power-management/suspend_30_cycles certification-status=blocker + power-management/suspend-30-cycle-log-attach + power-management/suspend-30-cycles-time-check certification-status=non-blocker + power-management/hibernate_30_cycles certification-status=non-blocker +nested_part: stress-cert-automated + id: stress-cert-automated unit: test plan _name: Stress tests (automated) diff --git a/units/submission/jobs.pxu b/units/submission/jobs.pxu index a21c897..f6f73bc 100644 --- a/units/submission/jobs.pxu +++ b/units/submission/jobs.pxu @@ -1,11 +1,9 @@ id: dkms_info_json plugin: attachment category_id: com.canonical.plainbox::info -requires: - package.name == 'dkms' command: dkms_info --format json | python3 -m plainbox dev parse dkms-info | \ - jq --indent 4 '.dkms_info' + jq '.dkms_info' _description: Attaches json dumps of installed dkms package information. _summary: Attaches json dumps of installed dkms package information. @@ -24,9 +22,9 @@ requires: user: root command: dmidecode -t bios -t system | python3 -m plainbox dev parse dmidecode | \ - jq --indent 4 '[.[0]."_attributes" + - {"category": .[0]."category"}, .[1]."_attributes" + - {"category": .[1]."category"}]' + jq '[.[0]."_attributes" + + {"category": .[0]."category"}, .[1]."_attributes" + + {"category": .[1]."category"}]' estimated_duration: 1 _description: Attaches dmidecode output _summary: Attaches json dumps of udev_resource raw dmi devices @@ -36,7 +34,7 @@ plugin: attachment category_id: com.canonical.plainbox::info command: find /etc/modprobe.* -name \*.conf | xargs cat | python3 -m plainbox dev parse modprobe | - jq --indent 4 'to_entries | map({"module": .key, "options": .value})' + jq 'to_entries | map({"module": .key, "options": .value})' estimated_duration: 0.015 _description: Attaches the contents of the various modprobe conf files. _summary: Attach the contents of /etc/modprobe.* @@ -47,7 +45,7 @@ plugin: attachment category_id: com.canonical.plainbox::info command: lspci -x | python3 -m plainbox dev parse pci-subsys-id | \ - jq --indent 4 '.pci_subsystem_id' + jq '.pci_subsystem_id' estimated_duration: 0.1 _description: Attaches a hex dump of the standard part of the PCI configuration space for all PCI devices. diff --git a/units/suspend/suspend-graphics.pxu b/units/suspend/suspend-graphics.pxu index e05153a..849e5eb 100644 --- a/units/suspend/suspend-graphics.pxu +++ b/units/suspend/suspend-graphics.pxu @@ -126,7 +126,7 @@ plugin: user-interact-verify category_id: com.canonical.plainbox::suspend id: suspend/{index}_glxgears_after_suspend_{product_slug}_xenial depends: suspend/{index}_suspend_after_switch_to_card_{product_slug}_xenial -requires: package.name == 'mesa-utils' +requires: executable.name == 'glxgears' command: source graphics_env {driver} {index} glxgears diff --git a/units/suspend/suspend.pxu b/units/suspend/suspend.pxu index 6704a2f..34d1957 100644 --- a/units/suspend/suspend.pxu +++ b/units/suspend/suspend.pxu @@ -1232,8 +1232,8 @@ category_id: com.canonical.plainbox::suspend id: suspend/gpu_lockup_after_suspend estimated_duration: 10.0 requires: - package.name == 'wmctrl' - package.name == 'mesa-utils' + executable.name == 'wmctrl' + executable.name == 'glxgears' package.name == 'firefox' package.name == 'unity' depends: suspend/suspend_advanced_auto @@ -2197,7 +2197,7 @@ plugin: user-interact-verify category_id: com.canonical.plainbox::suspend id: suspend/{index}_glxgears_after_suspend_{product_slug} depends: suspend/{index}_suspend_after_switch_to_card_{product_slug} -requires: package.name == 'mesa-utils' +requires: executable.name == 'glxgears' command: glxgears; true _summary: Test that glxgears works for {vendor} {product} after suspend _description: @@ -2215,7 +2215,7 @@ category_id: com.canonical.plainbox::suspend id: suspend/glxgears_after_suspend depends: suspend/suspend_advanced requires: - package.name == 'mesa-utils' + executable.name == 'glxgears' device.category == 'VIDEO' command: glxgears; true _summary: Test that glxgears works after suspend @@ -2292,8 +2292,7 @@ id: suspend/oops_after_suspend depends: suspend/suspend_advanced plugin:shell estimated_duration: 10.0 -requires: - package.name == 'fwts' +requires: executable.name == 'fwts' user: root _description: Run Firmware Test Suite (fwts) oops tests after suspend. diff --git a/units/thunderbolt/test-plan.pxu b/units/thunderbolt/test-plan.pxu index 054bb26..d5dc001 100644 --- a/units/thunderbolt/test-plan.pxu +++ b/units/thunderbolt/test-plan.pxu @@ -1,14 +1,33 @@ id: thunderbolt-cert-full unit: test plan _name: Thunderbolt tests -_description: Thunderbolt tests +_description: + Thunderbolt tests include: - thunderbolt/insert certification-status=blocker - thunderbolt/storage-test certification-status=blocker - thunderbolt/remove certification-status=blocker - thunderbolt3/insert certification-status=non-blocker - thunderbolt3/storage-test certification-status=non-blocker - thunderbolt3/remove certification-status=non-blocker +nested_part: + com.canonical.certification::thunderbolt-cert-manual + com.canonical.certification::thunderbolt-cert-automated + +id: thunderbolt-cert-manual +unit: test plan +_name: Thunderbolt tests (Manual) +_description: + Thunderbolt tests (Manual) +include: + thunderbolt/insert certification-status=blocker + thunderbolt/storage-test certification-status=blocker + thunderbolt/remove certification-status=blocker + thunderbolt3/insert certification-status=non-blocker + thunderbolt3/storage-test certification-status=non-blocker + thunderbolt3/remove certification-status=non-blocker + +id: thunderbolt-cert-automated +unit: test plan +_name: Thunderbolt tests (Automated) +_description: + Thunderbolt tests (Automated) +include: + id: thunderbolt-cert-blockers unit: test plan diff --git a/units/touchpad/test-plan.pxu b/units/touchpad/test-plan.pxu index 7514991..78d7fa6 100644 --- a/units/touchpad/test-plan.pxu +++ b/units/touchpad/test-plan.pxu @@ -1,23 +1,42 @@ id: touchpad-cert-full unit: test plan _name: Touchpad tests -_description: Touchpad tests +_description: + Touchpad tests include: - touchpad/basic certification-status=blocker - touchpad/detected-as-mouse certification-status=blocker - touchpad/palm-rejection certification-status=non-blocker - touchpad/continuous-move certification-status=blocker - touchpad/horizontal certification-status=blocker - touchpad/vertical certification-status=blocker - touchpad/singletouch-automated certification-status=blocker - touchpad/singletouch-selection certification-status=blocker - touchpad/drag-and-drop certification-status=blocker - touchpad/multitouch-automated certification-status=blocker - touchpad/multitouch-manual certification-status=blocker - touchpad/multitouch-rightclick certification-status=blocker - touchpad/multitouch-horizontal certification-status=blocker - touchpad/multitouch-vertical certification-status=blocker - touchpad/multitouch-dash certification-status=non-blocker +nested_part: + com.canonical.certification::touchpad-cert-manual + com.canonical.certification::touchpad-cert-automated + +id: touchpad-cert-manual +unit: test plan +_name: Touchpad tests (Manual) +_description: + Touchpad tests (Manual) +include: + touchpad/basic certification-status=blocker + touchpad/palm-rejection certification-status=non-blocker + touchpad/continuous-move certification-status=blocker + touchpad/horizontal certification-status=blocker + touchpad/vertical certification-status=blocker + touchpad/singletouch-selection certification-status=blocker + touchpad/drag-and-drop certification-status=blocker + touchpad/multitouch-manual certification-status=blocker + touchpad/multitouch-rightclick certification-status=blocker + touchpad/multitouch-horizontal certification-status=blocker + touchpad/multitouch-vertical certification-status=blocker + touchpad/multitouch-dash certification-status=non-blocker + +id: touchpad-cert-automated +unit: test plan +_name: Touchpad tests (Automated) +_description: + Touchpad tests (Automated) +include: + touchpad/detected-as-mouse certification-status=blocker + touchpad/singletouch-automated certification-status=blocker + touchpad/multitouch-automated certification-status=blocker + id: after-suspend-touchpad-cert-full unit: test plan diff --git a/units/touchscreen/jobs.pxu b/units/touchscreen/jobs.pxu index 51b4a4e..9db5677 100644 --- a/units/touchscreen/jobs.pxu +++ b/units/touchscreen/jobs.pxu @@ -140,7 +140,7 @@ _description: Did you see the green circles around the three fingers? command: manage_compiz_plugin unityshell disable - qmlscene -qt5 $PLAINBOX_PROVIDER_DATA/touch_tap_test.qml --touchpoints=3 2>&1 | grep -o PASS + qmlscene -qt5 $PLAINBOX_PROVIDER_DATA/touch_tap_test.qml 3 2>&1 | grep -o PASS EXIT=$? sleep 5 manage_compiz_plugin unityshell enable @@ -163,7 +163,7 @@ _description: Did you see the green circles around the four fingers? command: manage_compiz_plugin unityshell disable - qmlscene -qt5 $PLAINBOX_PROVIDER_DATA/touch_tap_test.qml --touchpoints=4 2>&1 | grep -o PASS + qmlscene -qt5 $PLAINBOX_PROVIDER_DATA/touch_tap_test.qml 4 2>&1 | grep -o PASS EXIT=$? sleep 5 manage_compiz_plugin unityshell enable diff --git a/units/touchscreen/test-plan.pxu b/units/touchscreen/test-plan.pxu index e984af9..1682136 100644 --- a/units/touchscreen/test-plan.pxu +++ b/units/touchscreen/test-plan.pxu @@ -1,14 +1,33 @@ id: touchscreen-cert-full unit: test plan _name: Touchscreen tests -_description: Touchscreen tests +_description: + Touchscreen tests include: - touchscreen/drag-n-drop certification-status=blocker - touchscreen/multitouch-zoom certification-status=blocker - touchscreen/multitouch-rotate - touchscreen/3-touch-tap certification-status=blocker - touchscreen/4-touch-tap certification-status=blocker - touchscreen/multitouch-dash certification-status=non-blocker +nested_part: + com.canonical.certification::touchscreen-cert-manual + com.canonical.certification::touchscreen-cert-automated + +id: touchscreen-cert-manual +unit: test plan +_name: Touchscreen tests (Manual) +_description: + Touchscreen tests (Manual) +include: + touchscreen/drag-n-drop certification-status=blocker + touchscreen/multitouch-zoom certification-status=blocker + touchscreen/multitouch-rotate + touchscreen/3-touch-tap certification-status=blocker + touchscreen/4-touch-tap certification-status=blocker + touchscreen/multitouch-dash certification-status=non-blocker + +id: touchscreen-cert-automated +unit: test plan +_name: Touchscreen tests (Automated) +_description: + Touchscreen tests (Automated) +include: + id: touchscreen-cert-blockers unit: test plan diff --git a/units/usb/test-plan.pxu b/units/usb/test-plan.pxu index 5b4e88b..27a0616 100644 --- a/units/usb/test-plan.pxu +++ b/units/usb/test-plan.pxu @@ -1,22 +1,60 @@ id: usb-cert-full unit: test plan _name: USB tests -_description: USB tests +_description: + USB tests include: - usb/detect certification-status=blocker - usb/HID certification-status=blocker - usb/insert certification-status=blocker - usb/storage-automated certification-status=blocker - usb/remove certification-status=blocker +nested_part: + com.canonical.certification::usb-cert-manual + com.canonical.certification::usb-cert-automated + +id: usb-cert-manual +unit: test plan +_name: USB tests (Manual) +_description: + USB tests (Manual) +include: + usb/HID certification-status=blocker + usb/insert certification-status=blocker + usb/storage-automated certification-status=blocker + usb/remove certification-status=blocker + +id: usb-cert-automated +unit: test plan +_name: USB tests (Automated) +_description: + USB tests (Automated) +include: + usb/detect certification-status=blocker + id: usb3-cert-full unit: test plan _name: USB3 tests -_description: USB3 tests +_description: + USB3 tests include: - usb3/insert certification-status=blocker - usb3/storage-automated certification-status=blocker - usb3/remove certification-status=blocker +nested_part: + com.canonical.certification::usb3-cert-manual + com.canonical.certification::usb3-cert-automated + +id: usb3-cert-manual +unit: test plan +_name: USB3 tests (Manual) +_description: + USB3 tests (Manual) +include: + usb3/insert certification-status=blocker + usb3/storage-automated certification-status=blocker + usb3/remove certification-status=blocker + +id: usb3-cert-automated +unit: test plan +_name: USB3 tests (Automated) +_description: + USB3 tests (Automated) +include: + id: usb-c-cert-full unit: test plan diff --git a/units/usb/usb.pxu b/units/usb/usb.pxu index 52c299d..00a4318 100644 --- a/units/usb/usb.pxu +++ b/units/usb/usb.pxu @@ -46,6 +46,7 @@ _description: Did the device work as expected? id: usb/insert +template-engine: jinja2 _summary: USB 2.0 storage device insertion detected _purpose: Check system can detect USB 2.0 storage when inserted @@ -60,15 +61,16 @@ plugin: user-interact flags: also-after-suspend user: root command: - if [[ -v SNAP ]]; then + {%- if __on_ubuntucore__ %} checkbox-support-run_watcher insertion usb2 - else + {%- else %} removable_storage_watcher --unmounted insert usb - fi + {% endif -%} category_id: com.canonical.plainbox::usb estimated_duration: 120 id: usb3/insert +template-engine: jinja2 requires: usb.usb3 == 'supported' _summary: USB 3.0 storage device insertion detected @@ -84,15 +86,16 @@ _verification: plugin: user-interact user: root command: - if [[ -v SNAP ]]; then + {%- if __on_ubuntucore__ %} checkbox-support-run_watcher insertion usb3 - else + {%- else %} removable_storage_watcher --unmounted -m 500000000 insert usb - fi + {% endif -%} category_id: com.canonical.plainbox::usb estimated_duration: 120 id: usb/remove +template-engine: jinja2 _summary: USB 2.0 storage device removal detected _purpose: Check system can detect removal of a USB 2.0 storage device @@ -108,15 +111,16 @@ depends: usb/insert flags: also-after-suspend user: root command: - if [[ -v SNAP ]]; then + {%- if __on_ubuntucore__ %} checkbox-support-run_watcher removal usb2 - else + {%- else %} removable_storage_watcher --unmounted remove usb - fi + {% endif -%} category_id: com.canonical.plainbox::usb estimated_duration: 120 id: usb3/remove +template-engine: jinja2 _summary: USB 3.0 storage device removal detected _purpose: Check system can detect removal of a USB 3.0 storage device @@ -132,11 +136,11 @@ flags: also-after-suspend depends: usb3/insert user: root command: - if [[ -v SNAP ]]; then + {%- if __on_ubuntucore__ %} checkbox-support-run_watcher removal usb3 - else + {%- else %} removable_storage_watcher --unmounted -m 500000000 remove usb - fi + {% endif -%} category_id: com.canonical.plainbox::usb estimated_duration: 120 @@ -179,6 +183,7 @@ _description: automatically selected result. id: usb/storage-automated +template-engine: jinja2 _summary: USB 2.0 storage device read & write works _purpose: Check system can read/write to USB 2.0 storage correctly @@ -191,15 +196,16 @@ depends: usb/insert flags: also-after-suspend user: root command: - if [[ -v SNAP ]]; then + {%- if __on_ubuntucore__ %} checkbox-support-usb_read_write - else + {%- else %} removable_storage_test -s 268400000 usb - fi + {% endif -%} category_id: com.canonical.plainbox::usb estimated_duration: 300 id: usb3/storage-automated +template-engine: jinja2 _summary: USB 3.0 storage device read & write works _purpose: Check system can read/write to USB 3.0 storage devices correctly @@ -212,11 +218,11 @@ depends: usb3/insert flags: also-after-suspend user: root command: - if [[ -v SNAP ]]; then + {%- if __on_ubuntucore__ %} checkbox-support-usb_read_write - else + {%- else %} removable_storage_test -s 268400000 -m 500000000 usb --driver xhci_hcd - fi + {% endif -%} category_id: com.canonical.plainbox::usb estimated_duration: 300 diff --git a/units/wireless/jobs.pxu b/units/wireless/jobs.pxu index 427596c..363984a 100644 --- a/units/wireless/jobs.pxu +++ b/units/wireless/jobs.pxu @@ -1,6 +1,6 @@ unit: template template-resource: device -template-filter: device.category == 'WIRELESS' +template-filter: device.category == 'WIRELESS' and device.interface != 'UNKNOWN' template-engine: jinja2 template-unit: job id: wireless/wireless_scanning_{{ interface }} @@ -21,7 +21,7 @@ requires: unit: template template-resource: device -template-filter: device.category == 'WIRELESS' +template-filter: device.category == 'WIRELESS' and device.interface != 'UNKNOWN' template-engine: jinja2 template-unit: job id: wireless/wireless_connection_wpa_bg_nm_{{ interface }} @@ -42,7 +42,7 @@ requires: unit: template template-resource: device -template-filter: device.category == 'WIRELESS' +template-filter: device.category == 'WIRELESS' and device.interface != 'UNKNOWN' template-engine: jinja2 template-unit: job id: wireless/wireless_connection_open_bg_nm_{{ interface }} @@ -63,7 +63,7 @@ requires: unit: template template-resource: device -template-filter: device.category == 'WIRELESS' +template-filter: device.category == 'WIRELESS' and device.interface != 'UNKNOWN' template-engine: jinja2 template-unit: job id: wireless/wireless_connection_wpa_n_nm_{{ interface }} @@ -78,14 +78,13 @@ category_id: com.canonical.plainbox::wireless estimated_duration: 30.0 flags: preserve-locale also-after-suspend also-after-suspend-manual requires: - wireless_sta_protocol.{{ interface }}_n == 'supported' {%- if __on_ubuntucore__ %} connections.slot == 'network-manager:service' and connections.plug == '{{ __system_env__["SNAP_NAME"] }}:network-manager' {% endif -%} unit: template template-resource: device -template-filter: device.category == 'WIRELESS' +template-filter: device.category == 'WIRELESS' and device.interface != 'UNKNOWN' template-engine: jinja2 template-unit: job id: wireless/wireless_connection_open_n_nm_{{ interface }} @@ -100,14 +99,13 @@ category_id: com.canonical.plainbox::wireless estimated_duration: 30.0 flags: preserve-locale also-after-suspend also-after-suspend-manual requires: - wireless_sta_protocol.{{ interface }}_n == 'supported' {%- if __on_ubuntucore__ %} connections.slot == 'network-manager:service' and connections.plug == '{{ __system_env__["SNAP_NAME"] }}:network-manager' {% endif -%} unit: template template-resource: device -template-filter: device.category == 'WIRELESS' +template-filter: device.category == 'WIRELESS' and device.interface != 'UNKNOWN' template-engine: jinja2 template-unit: job id: wireless/wireless_connection_wpa_ac_nm_{{ interface }} @@ -129,7 +127,7 @@ requires: unit: template template-resource: device -template-filter: device.category == 'WIRELESS' +template-filter: device.category == 'WIRELESS' and device.interface != 'UNKNOWN' template-engine: jinja2 template-unit: job id: wireless/wireless_connection_open_ac_nm_{{ interface }} diff --git a/units/wireless/test-plan.pxu b/units/wireless/test-plan.pxu index ee921df..74bf271 100644 --- a/units/wireless/test-plan.pxu +++ b/units/wireless/test-plan.pxu @@ -1,11 +1,23 @@ id: wireless-cert-full unit: test plan _name: Wireless tests -_description: Wireless connection tests +_description: + Wireless connection tests +include: +nested_part: + com.canonical.certification::wireless-cert-manual + com.canonical.certification::wireless-cert-automated + +id: wireless-cert-manual +unit: test plan +_name: Wireless tests (Manual) +_description: + Wireless connection tests (Manual) include: nested_part: wireless-cert-automated + id: after-suspend-wireless-cert-full unit: test plan _name: Wireless tests (after suspend) |