Import plainbox-provider-checkbox_0.60.0~rc1.orig.tar.gzupstream-0.60.0_rc1patched-0.60.0_rc1-1

31 files changed, 307 insertions, 86 deletions

diff --git a/units/audio/jobs.pxu b/units/audio/jobs.pxu
index 1fad964..e6b21a6 100644
--- a/units/audio/jobs.pxu
+++ b/units/audio/jobs.pxu
@@ -15,13 +15,18 @@ requires:
 device.category == 'AUDIO'
 package.name == 'linux-firmware'
 command:
+ fail=0
 for fw in /lib/firmware/intel/sof/sof-*.ri;
 do
 # shellcheck disable=SC2016
 if xxd "${fw}" | grep -q '$CPD'; then
- sof_ri_info.py --valid "${fw}"
+ if ! sof_ri_info.py --valid "${fw}";
+ then
+ fail=1
+ fi
 fi
 done
+ exit $fail
 _description: Valid sof firmware signature
 
 plugin: user-interact-verify
diff --git a/units/bluetooth/jobs.pxu b/units/bluetooth/jobs.pxu
index 3f174bf..e1573b5 100644
--- a/units/bluetooth/jobs.pxu
+++ b/units/bluetooth/jobs.pxu
@@ -4,7 +4,7 @@ category_id: com.canonical.plainbox::bluetooth
 _summary: Make sure at least one bluetooth device is detected
 plugin: shell
 command:
- bt_list_adapters.py
+ bt_list_adapters.py && udev_resource.py -f BLUETOOTH
 estimated_duration: 2s
 flags: preserve-locale
 requires: manifest.has_bt_adapter
diff --git a/units/cpu/test-plan.pxu b/units/cpu/test-plan.pxu
index 46e9ac4..f649267 100644
--- a/units/cpu/test-plan.pxu
+++ b/units/cpu/test-plan.pxu
@@ -14,9 +14,6 @@ _name: CPU tests (Manual)
 _description:
 CPU tests (Manual)
 include:
-nested_part:
- cpu-cert-automated
-
 
 id: cpu-cert-automated
 unit: test plan
diff --git a/units/ethernet/jobs.pxu b/units/ethernet/jobs.pxu
index 64a0578..89565bf 100644
--- a/units/ethernet/jobs.pxu
+++ b/units/ethernet/jobs.pxu
@@ -97,6 +97,26 @@ template-resource: device
 template-filter: device.category == 'NETWORK' and device.interface != 'UNKNOWN'
 plugin: shell
 category_id: com.canonical.plainbox::ethernet
+id: ethernet/multi_iperf3_nic_device_server_sru{__index__}_{interface}
+_summary: Multi-NIC Iperf3 SRU testing for NIC {interface}
+estimated_duration: 1200.0
+requires:
+ executable.name == 'iperf3'
+ executable.name == 'ethtool'
+ executable.name == 'nmap'
+user: root
+environ: TEST_TARGET_IPERF
+command: network.py test -i {interface} -t iperf --iperf3 --scan-timeout 3600 --fail-threshold 80 --cpu-load-fail-threshold 90 --runtime 1200
+_description:
+ This is a shorter SRU version of the standard iperf3 based network test for
+ servers. It is intended to ONLY be used for SRU and is not valid for offical
+ certification.
+
+unit: template
+template-resource: device
+template-filter: device.category == 'NETWORK' and device.interface != 'UNKNOWN'
+plugin: shell
+category_id: com.canonical.plainbox::ethernet
 id: ethernet/multi_iperf3_nic_underspeed_device{__index__}_{interface}
 _summary: Underspeed Enabled Multi-NIC Iperf3 stress testing for NIC {interface}
 estimated_duration: 7400.0
diff --git a/units/ethernet/test-plan.pxu b/units/ethernet/test-plan.pxu
index cfc23bd..33c95b4 100644
--- a/units/ethernet/test-plan.pxu
+++ b/units/ethernet/test-plan.pxu
@@ -1,3 +1,38 @@
+id: ethernet-cert-full
+unit: test plan
+_name: Ethernet tests
+_description: Ethernet tests
+include:
+nested_part:
+ ethernet-cert-automated
+ ethernet-cert-manual
+
+id: ethernet-cert-manual
+unit: test plan
+_name: Ethernet tests (manual)
+_description: Ethernet tests (manual)
+include:
+ ethernet/hotplug-.* certification-status=blocker
+bootstrap_include:
+ device
+
+id: ethernet-cert-automated
+unit: test plan
+_name: Ethernet tests (automated)
+_description: Ethernet tests (automated)
+include:
+ ethernet/detect certification-status=blocker
+
+id: ethernet-cert-blockers
+unit: test plan
+_name: Ethernet tests (certification blockers only)
+_description: Ethernet tests (certification blockers only)
+include:
+ ethernet/detect certification-status=blocker
+ ethernet/hotplug-.* certification-status=blocker
+bootstrap_include:
+ device
+
 id: ethernet-full
 unit: test plan
 _name: QA tests for ethernet
@@ -72,6 +107,20 @@ bootstrap_include:
 device
 executable
 
+id: server-ethernet-sru
+unit: test plan
+_name: Server Ethernet SRU Tests
+_description: Automated ethernet tests for server regression testing
+estimated_duration: 4h
+include:
+ ethernet/info_automated_server
+ ethernet/ethtool_info
+ ethernet/ethertool_check_.*
+ ethernet/multi_iperf3_nic_device_server_sru.*
+bootstrap_include:
+ device
+ executable
+
 id: server-ethernet-underspeed
 unit: test plan
 _name: Server Ethernet Tests (Speed Check Disabled)
diff --git a/units/fingerprint/test-plan.pxu b/units/fingerprint/test-plan.pxu
index 42ebd52..ed5ee9a 100644
--- a/units/fingerprint/test-plan.pxu
+++ b/units/fingerprint/test-plan.pxu
@@ -1,15 +1,16 @@
 id: fingerprint-full
 unit: test plan
-_name: QA tests for fingerprint reader
+_name: QA tests for Fingerprint reader
 _description: Fingerprint tests
 estimated_duration: 1m
 include:
 nested_part:
 fingerprint-manual
+ fingerprint-automated
 
 id: fingerprint-manual
 unit: test plan
-_name: Manual fingerprint tests
+_name: Fingerprint reader tests (Manual)
 _description: Manual tests for fingerprint reader
 include:
 fingerprint/detect certification-status=non-blocker
@@ -18,3 +19,9 @@ include:
 fingerprint/verify-match certification-status=non-blocker
 fingerprint/unlock certification-status=non-blocker
 fingerprint/delete certification-status=non-blocker
+
+id: fingerprint-automated
+unit: test plan
+_name: Fingerprint reader tests (Automated)
+_description: Autoamted tests for fingerprint reader
+include:
diff --git a/units/gpio/test-plan.pxu b/units/gpio/test-plan.pxu
index a32be08..91d83b5 100644
--- a/units/gpio/test-plan.pxu
+++ b/units/gpio/test-plan.pxu
@@ -25,6 +25,7 @@ _name: Automated GPIO tests
 _description: Automated GPIO tests for Ubuntu Core devices
 bootstrap_include:
 model_assertion
+ dmi
 include:
 gpio/sysfs_loopback_pairs_.*
 gpio/gpiomem_loopback_pairs_.*
diff --git a/units/gpio/vendor-aaeon.pxu b/units/gpio/vendor-aaeon.pxu
new file mode 100644
index 0000000..7f8d6bc
--- /dev/null
+++ b/units/gpio/vendor-aaeon.pxu
@@ -0,0 +1,16 @@
+
+unit: template
+template-resource: dmi
+template-filter: dmi.category == 'SYSTEM' and dmi.vendor == 'AAEON' and dmi.product in ('UPX-TGL01')
+template-unit: job
+id: gpio/sysfs_loopback_pairs_{vendor}_{product}
+_summary: Test GPIO lines exposed on headers can be controlled via sysfs
+plugin: shell
+user: root
+category_id: gpio
+estimated_duration: 30.0
+flags: preserve-locale also-after-suspend
+imports: from com.canonical.plainbox import manifest
+requires: manifest.gpio_loopback == 'True'
+command:
+ gpio_sysfs_loopback.py {product}
diff --git a/units/graphics/jobs.pxu b/units/graphics/jobs.pxu
index acf18c3..9727bc8 100644
--- a/units/graphics/jobs.pxu
+++ b/units/graphics/jobs.pxu
@@ -263,7 +263,6 @@ template-resource: graphics_card
 plugin: shell
 category_id: com.canonical.plainbox::graphics
 id: graphics/{index}_gl_support_{product_slug}
-requires: package.name == 'nux-tools'
 command: /usr/lib/nux/unity_support_test -p 2>&1
 estimated_duration: 0.131
 _description: Check that {vendor} {product} hardware is able to run a desktop session (OpenGL)
diff --git a/units/info/jobs.pxu b/units/info/jobs.pxu
index d2ed96b..12d7ab1 100644
--- a/units/info/jobs.pxu
+++ b/units/info/jobs.pxu
@@ -454,7 +454,11 @@ category_id: com.canonical.plainbox::info
 id: info/systemd-analyze
 estimated_duration: 0.2
 _summary: System boot-up performance statistics
-command: systemd-analyze
+command:
+ # Wait for boot to complete
+ # https://github.com/systemd/systemd/pull/9796
+ timeout 60 systemctl is-system-running --wait > /dev/null
+ systemd-analyze
 
 id: lstopo_verbose_attachment
 plugin: attachment
diff --git a/units/input/jobs.pxu b/units/input/jobs.pxu
index 5f5cde0..7ae0667 100644
--- a/units/input/jobs.pxu
+++ b/units/input/jobs.pxu
@@ -75,3 +75,19 @@ _steps:
 4. Double-click the left button with your {product}.
 _verification:
 Did these buttons work as expected?
+
+unit: template
+template-resource: dmi
+template-filter: dmi.category == 'CHASSIS' and dmi.product not in ('Tablet', 'Convertible', 'Detachable')
+plugin: shell
+category_id: com.canonical.plainbox::input
+estimated_duration: 2
+id: input/fixed_screen_orientation_on_{product}_{__index__}
+user: root
+requires:
+ executable.name == 'udevadm'
+ executable.name == 'dbus-send'
+command: fixed_screen_orientation.sh
+_summary: Check whether screen orientation is fixed on {product}
+_purpose:
+ This tests whether the display orientation of {product} is fixed, and cannot be altered by tilting {product}.
diff --git a/units/input/test-plan.pxu b/units/input/test-plan.pxu
index ef9cf75..1c87d1f 100644
--- a/units/input/test-plan.pxu
+++ b/units/input/test-plan.pxu
@@ -23,7 +23,9 @@ _name: Input tests (Automated)
 _description:
 Input tests (Automated)
 include:
-
+ input/fixed_screen_orientation_on_.* certification-status=non-blocker
+bootstrap_include:
+ dmi
 
 id: after-suspend-input-cert-full
 unit: test plan
diff --git a/units/kernel-snap/jobs.pxu b/units/kernel-snap/jobs.pxu
index 950d1a6..b03d6a8 100644
--- a/units/kernel-snap/jobs.pxu
+++ b/units/kernel-snap/jobs.pxu
@@ -33,3 +33,22 @@ imports:
 from com.canonical.certification import ubuntu_core_features
 requires:
 ubuntu_core_features.force_kernel_extraction == 'True'
+
+unit: template
+template-resource: bootloader
+template-filter: bootloader.name == 'uboot'
+id: kernel-snap/booted-dtbs-match-current-{name}
+requires:
+ lsb.description == 'Ubuntu Core 18'
+ snap.name == 'snapd' and int(snap.revision) >= 11841
+category_id: kernel-snap
+_summary: The dtbs in the bootloader match those in current kernel snap
+_description:
+ Since snapd version 2.50 DTB files can be distributed in the kernel snap and
+ the extracted in to the boot partition. This checks that if the DTBs are in
+ sync.
+plugin: shell
+user: root
+estimated_duration: 2.0
+command:
+ booted_dtb_tests.py {name}
diff --git a/units/kernel-snap/test-plan.pxu b/units/kernel-snap/test-plan.pxu
index ffaa5d1..585bd01 100644
--- a/units/kernel-snap/test-plan.pxu
+++ b/units/kernel-snap/test-plan.pxu
@@ -14,6 +14,7 @@ _name: Automated Kernel Snap tests
 _description: Automated Kernel Snap tests for Ubuntu Core devices
 include:
 kernel-snap/booted-kernel-matches-current-.*
+ kernel-snap/booted-dtbs-match-current-.*
 bootstrap_include:
 bootloader
 
diff --git a/units/led/test-plan.pxu b/units/led/test-plan.pxu
index fc83aa2..b0d0129 100644
--- a/units/led/test-plan.pxu
+++ b/units/led/test-plan.pxu
@@ -115,5 +115,7 @@ include:
 led/bluetooth
 led/serial
 led/fn
-
-
+ led/sysfs_led_brightness_on_.*
+ led/sysfs_led_brightness_off_.*
+bootstrap_include:
+ dmi
diff --git a/units/led/vendor-aaeon.pxu b/units/led/vendor-aaeon.pxu
new file mode 100644
index 0000000..d4ed7cd
--- /dev/null
+++ b/units/led/vendor-aaeon.pxu
@@ -0,0 +1,39 @@
+
+unit: template
+template-resource: dmi
+template-filter: dmi.category == 'SYSTEM' and dmi.vendor == 'AAEON' and dmi.product in ('UPX-TGL01')
+template-unit: job
+id: led/sysfs_led_brightness_on_{vendor}_{product}
+plugin: user-interact-verify
+category_id: led
+estimated_duration: 30.0
+flags: preserve-locale also-after-suspend
+command:
+ led_sysfs_brightness.py {product} on
+_purpose:
+ Verify that the leds_aaeon driver is working by setting all LEDs to maximum
+ brightness
+_steps:
+ Press ENTER to start the test while watching external LEDs on the SUT
+_verification:
+ The external LEDs should now be on at maximum brightness
+
+unit: template
+template-resource: dmi
+template-filter: dmi.category == 'SYSTEM' and dmi.vendor == 'AAEON' and dmi.product in ('UPX-TGL01')
+template-unit: job
+id: led/sysfs_led_brightness_off_{vendor}_{product}
+plugin: user-interact-verify
+category_id: led
+estimated_duration: 30.0
+flags: preserve-locale also-after-suspend
+depends: led/sysfs_led_brightness_on_{vendor}_{product}
+command:
+ led_sysfs_brightness.py {product} off
+_purpose:
+ Verify that the leds_aaeon driver is working by setting all LEDs to off / 
+ minimum brightness
+_steps:
+ Press ENTER to start the test while watching external LEDs on the SUT
+_verification:
+ The external LEDs should now be off or at minimum brightness
diff --git a/units/miscellanea/jobs.pxu b/units/miscellanea/jobs.pxu
index a1eab66..ac0c9ae 100644
--- a/units/miscellanea/jobs.pxu
+++ b/units/miscellanea/jobs.pxu
@@ -183,7 +183,7 @@ _summary: Test that system is not a pre-release version
 _description:
 Test to verify that the system uses production, rather
 than pre-release, versions of the kernel and the OS.
-command: check-prerelease.py
+command: check_prerelease.py
 requires:
 "Ubuntu Core" not in lsb.description
 
@@ -469,7 +469,7 @@ category_id: com.canonical.plainbox::miscellanea
 estimated_duration: 60
 id: miscellanea/debsums
 user: root
-requires: package.name == 'debsums'
+requires: executable.name == 'debsums'
 command: debsums -c
 _summary: Check the MD5 sums of installed Debian packages
 _description: Verify installed Debian package files against MD5 checksum lists from /var/lib/dpkg/info/*.md5sums.
diff --git a/units/miscellanea/test-plan.pxu b/units/miscellanea/test-plan.pxu
index 97c94fc..1c7605a 100644
--- a/units/miscellanea/test-plan.pxu
+++ b/units/miscellanea/test-plan.pxu
@@ -15,7 +15,6 @@ _description:
 Miscellaneous client tests (fwts, dmi, oops) (Manual)
 include:
 nested_part:
- misc-client-cert-automated
 
 
 id: misc-client-cert-automated
@@ -33,7 +32,8 @@ include:
 miscellanea/oops certification-status=blocker
 miscellanea/oops_results.log
 miscellanea/fan_stress_reaction
- miscellanea/debsums
+ miscellanea/debsums certification-status=blocker
+ miscellanea/check_prerelease certification-status=blocker
 bootstrap_include:
 fwts
 
@@ -50,6 +50,8 @@ include:
 firmware/fwts_uefirtvariable.* certification-status=blocker
 miscellanea/oops certification-status=blocker
 miscellanea/oops_results.log
+ miscellanea/debsums certification-status=blocker
+ miscellanea/check_prerelease certification-status=blocker
 bootstrap_include:
 fwts
 
@@ -67,7 +69,7 @@ mandatory_include:
 miscellanea/efi_boot_mode certification-status=blocker
 miscellanea/reboot_firmware
 miscellanea/efi_pxeboot
- miscellanea/check_prerelease
+ miscellanea/check_prerelease certification-status=blocker
 miscellanea/kernel_taint_test
 miscellanea/cpus_are_not_samples
 miscellanea/ipmi_test certification-status=blocker
@@ -85,7 +87,6 @@ _description:
 mandatory_include:
 miscellanea/submission-resources
 miscellanea/cpuid
- miscellanea/efi_boot_mode certification-status=blocker
 miscellanea/ipmi_test certification-status=blocker
 miscellanea/bmc_info
 miscellanea/dmitest_server
diff --git a/units/mobilebroadband/test-plan.pxu b/units/mobilebroadband/test-plan.pxu
index dd086c7..dc5ded9 100644
--- a/units/mobilebroadband/test-plan.pxu
+++ b/units/mobilebroadband/test-plan.pxu
@@ -14,9 +14,6 @@ _name: Mobile broadband tests (Manual)
 _description:
 Mobile broadband tests (Manual)
 include:
-nested_part:
- mobilebroadband-cert-automated
-
 
 id: mobilebroadband-cert-automated
 unit: test plan
diff --git a/units/networking/test-plan.pxu b/units/networking/test-plan.pxu
index f11d215..c00504f 100644
--- a/units/networking/test-plan.pxu
+++ b/units/networking/test-plan.pxu
@@ -13,7 +13,6 @@ _name: Networking tests (manual)
 _description: Networking tests (manual)
 include:
 networking/info_device.* certification-status=blocker
- ethernet/hotplug-.* certification-status=blocker
 bootstrap_include:
 device
 
@@ -22,7 +21,6 @@ unit: test plan
 _name: Networking tests (automated)
 _description: Networking tests (automated)
 include:
- ethernet/detect certification-status=blocker
 networking/gateway_ping certification-status=blocker
 networking/ntp certification-status=blocker
 
@@ -31,11 +29,9 @@ unit: test plan
 _name: Networking tests (certification blockers only)
 _description: Networking tests (certification blockers only)
 include:
- ethernet/detect certification-status=blocker
 networking/gateway_ping certification-status=blocker
 networking/info_device.* certification-status=blocker
 networking/ntp certification-status=blocker
- ethernet/hotplug-.* certification-status=blocker
 bootstrap_include:
 device
 
diff --git a/units/power-management/jobs.pxu b/units/power-management/jobs.pxu
index 726b4ae..d449564 100644
--- a/units/power-management/jobs.pxu
+++ b/units/power-management/jobs.pxu
@@ -366,15 +366,13 @@ unit: job
 plugin: shell
 requires:
 cpuinfo.cpu_lpi_file in ('low_power_idle_cpu_residency_us', 'package_cstate_show')
- package.name == 'x11-xserver-utils'
 sleep.mem_sleep == 's2idle'
+ package.name == 'evemu-tools'
 command:
 cpu_lpi_file=$(cpuinfo_resource.py | grep cpu_lpi_file | awk '{ print $2 }')
 if [ "$cpu_lpi_file" == "low_power_idle_cpu_residency_us" ]; then
 before=$(cat /sys/devices/system/cpu/cpuidle/low_power_idle_cpu_residency_us)
- xset dpms force off
- sleep 20
- xset dpms force on
+ screenoff.sh 20
 after=$(cat /sys/devices/system/cpu/cpuidle/low_power_idle_cpu_residency_us)
 residency=$((after-before))
 echo "/sys/devices/system/cpu/cpuidle/low_power_idle_cpu_residency_us after/before screen off is $residency"
@@ -382,9 +380,7 @@ command:
 echo "The system doesn't have hardware-based residency counter."
 echo "please check https://www.uefi.org/sites/default/files/resources/Intel_ACPI_Low_Power_S0_Idle.pdf"
 before=$(cat < /sys/kernel/debug/pmc_core/package_cstate_show | grep C10 | awk '{ print $4 }')
- xset dpms force off
- sleep 20
- xset dpms force on
+ screenoff.sh 20
 after=$(cat < /sys/kernel/debug/pmc_core/package_cstate_show | grep C10 | awk '{ print $4 }')
 residency=$((after-before))
 echo "/sys/kernel/debug/pmc_core/package_cstate_show after/before screen off is $residency"
diff --git a/units/power-management/packaging.pxu b/units/power-management/packaging.pxu
new file mode 100644
index 0000000..a03242e
--- /dev/null
+++ b/units/power-management/packaging.pxu
@@ -0,0 +1,4 @@
+# This is to install evemu-tools for the screen lock key event simulation
+unit: packaging meta-data
+os-id: debian
+Depends: evemu-tools
diff --git a/units/power-management/test-plan.pxu b/units/power-management/test-plan.pxu
index 0cc9997..40e5dc1 100644
--- a/units/power-management/test-plan.pxu
+++ b/units/power-management/test-plan.pxu
@@ -1,19 +1,27 @@
-id: power-management-precheck-cert
+id: power-management-precheck-cert-full
 unit: test plan
 _name: Power Management precheck tests
 _description: Power Management precheck tests
 include:
 nested_part:
+ power-management-precheck-cert-manual
 power-management-precheck-cert-automated
 
-id: power-management-reboot-poweroff-cert
+id: power-management-reboot-poweroff-cert-full
 unit: test plan
 _name: Power Management reboot and power off tests
 _description: Power Management reboot and power off tests
 include:
 nested_part:
+ power-management-reboot-poweroff-cert-manual
 power-management-reboot-poweroff-cert-automated
 
+id: power-management-precheck-cert-manual
+unit: test plan
+_name: Power Management precheck tests (Manual)
+_description: Power Management precheck tests (Manual)
+include:
+
 id: power-management-precheck-cert-automated
 unit: test plan
 _name: Power Management precheck tests (automated)
@@ -24,6 +32,12 @@ include:
 power-management/fwts_wakealarm certification-status=blocker
 power-management/fwts_wakealarm-log-attach
 
+id: power-management-reboot-poweroff-cert-manual
+unit: test plan
+_name: Power Management reboot and power off tests (Manual
+_description: Power Management reboot and power off tests (Manual)
+include:
+
 id: power-management-reboot-poweroff-cert-automated
 unit: test plan
 _name: Power Management reboot and power off tests (automated)
diff --git a/units/stress/jobs.pxu b/units/stress/jobs.pxu
index a541e3a..544e7af 100644
--- a/units/stress/jobs.pxu
+++ b/units/stress/jobs.pxu
@@ -510,7 +510,9 @@ plugin:shell
 category_id: com.canonical.plainbox::stress
 id: stress/s2idle_pm-graph_30
 estimated_duration: 10m
-requires: executable.name == 'sleepgraph'
+requires: 
+ executable.name == 'sleepgraph'
+ sleep.mem_sleep == 's2idle'
 user: root
 _summary: Resume from idle by using Intel pm-graph
 command:
@@ -520,6 +522,10 @@ plugin: attachment
 category_id: com.canonical.plainbox::stress
 id: stress/s2idle_pm-graph_30.tar.xz
 estimated_duration: 1
+requires: 
+ sleep.mem_sleep == 's2idle'
+after:
+ stress/s2idle_pm-graph_30
 user: root
 _summary: Attach pm-graph logs (s2idle)
 command:
@@ -529,7 +535,9 @@ plugin:shell
 category_id: com.canonical.plainbox::stress
 id: stress/s3_pm-graph_30
 estimated_duration: 10m
-requires: executable.name == 'sleepgraph'
+requires:
+ executable.name == 'sleepgraph'
+ sleep.mem_sleep == 'deep'
 user: root
 _summary: Resume from suspend by using Intel pm-graph
 command:
@@ -539,6 +547,10 @@ plugin: attachment
 category_id: com.canonical.plainbox::stress
 id: stress/s3_pm-graph_30.tar.xz
 estimated_duration: 1
+requires:
+ sleep.mem_sleep == 'deep'
+after:
+ stress/s3_pm-graph_30
 user: root
 _summary: Attach pm-graph logs (s3)
 command:
diff --git a/units/stress/stress-ng.pxu b/units/stress/stress-ng.pxu
index 8dd6853..adca084 100644
--- a/units/stress/stress-ng.pxu
+++ b/units/stress/stress-ng.pxu
@@ -30,4 +30,11 @@ estimated_duration: 1200.0
 environ: STRESS_NG_CLASSES_TIMEOUT
 command:
 cd /var/tmp || exit $?
- stress-ng --sequential 0 --class {stress-ng-class} --timeout "${{STRESS_NG_CLASSES_TIMEOUT:-30}}"
+ stress-ng --sequential 0 --class {stress-ng-class} --timeout "${{STRESS_NG_CLASSES_TIMEOUT:-30}}" --skip-silent --verbose
+ # Error code definition - 3(no resource), 4(not implemented)
+ EXIT_CODE=$?
+ echo "EXIT_CODE="$EXIT_CODE
+ case $EXIT_CODE in
+ 0|3|4) exit 0;;
+ *) exit 1;;
+ esac
diff --git a/units/suspend/suspend.pxu b/units/suspend/suspend.pxu
index a2830d4..8c25a91 100644
--- a/units/suspend/suspend.pxu
+++ b/units/suspend/suspend.pxu
@@ -189,17 +189,27 @@ requires:
 sleep.mem == 'supported'
 rtc.state == 'supported'
 user: root
-environ: PLAINBOX_SESSION_SHARE
+environ: PLAINBOX_SESSION_SHARE RTC_DEVICE_FILE
 command:
 if type -P fwts >/dev/null; then
- echo "Calling fwts"
- if [[ -v SNAP ]]; then
- export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$SNAP/usr/lib/fwts"
- fi
- set -o pipefail; checkbox-support-fwts_test -f none -l "$PLAINBOX_SESSION_SHARE"/suspend_single -s s3 --s3-sleep-delay=30 --s3-device-check --s3-device-check-delay=45 | tee "$PLAINBOX_SESSION_SHARE"/suspend_single_times.log
+ echo "Calling fwts"
+ if [[ -v SNAP ]]; then
+ export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$SNAP/usr/lib/fwts"
+ fi
+ if fwts --show-tests-categories | grep -q 's3 '; then
+ echo "Calling fwts"
+ set -o pipefail; checkbox-support-fwts_test -f none -l "$PLAINBOX_SESSION_SHARE"/suspend_single -s s3 --s3-sleep-delay=30 --s3-device-check --s3-device-check-delay=45 | tee "$PLAINBOX_SESSION_SHARE"/suspend_single_times.log
+ else
+ echo "Calling rtcwake"
+ if [ -z "$RTC_DEVICE_FILE" ]; then
+ rtcwake -m mem -s 30
+ else
+ rtcwake -d "$RTC_DEVICE_FILE" -m mem -s 30
+ fi
+ fi
 else
- echo "Calling sleep_test.py"
- set -o pipefail; sleep_test.py -p | tee "$PLAINBOX_SESSION_SHARE"/2_suspend_single_times.log
+ echo "Calling sleep_test.py"
+ set -o pipefail; sleep_test.py -p | tee "$PLAINBOX_SESSION_SHARE"/2_suspend_single_times.log
 fi
 estimated_duration: 90.0
 _summary: Manual test of suspend function
@@ -263,7 +273,7 @@ _summary: Automated test of suspend function
 _description:
 This is the automated version of suspend/suspend_advanced.
 user: root
-environ: PLAINBOX_SESSION_SHARE
+environ: PLAINBOX_SESSION_SHARE RTC_DEVICE_FILE
 command:
 if [[ -v SNAP ]]; then
 export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$SNAP/usr/lib/fwts"
@@ -274,7 +284,11 @@ command:
 set -o pipefail; checkbox-support-fwts_test -f none -l "$PLAINBOX_SESSION_SHARE"/suspend_single -s s3 --s3-sleep-delay=30 --s3-device-check --s3-device-check-delay=45 | tee "$PLAINBOX_SESSION_SHARE"/suspend_single_times.log
 else
 echo "Calling rtcwake"
- rtcwake -m mem -s 30
+ if [ -z "$RTC_DEVICE_FILE" ]; then
+ rtcwake -m mem -s 30
+ else
+ rtcwake -d "$RTC_DEVICE_FILE" -m mem -s 30
+ fi
 fi
 estimated_duration: 90.000
 
diff --git a/units/ubuntucore/jobs.pxu b/units/ubuntucore/jobs.pxu
index 8cae699..04c30b3 100644
--- a/units/ubuntucore/jobs.pxu
+++ b/units/ubuntucore/jobs.pxu
@@ -65,7 +65,7 @@ _steps:
 {% endif -%}
 1. Check the current revision of core{{release}}
 $ snap list core{{release}}
- 2. Update to edge. 
+ 2. Update to edge.
 Note that system will automatically reboot in 1 minute after doing this command.
 Please execute the command in step 3 in 1 minute to stop the automatic reboot.
 $ sudo snap refresh core{{release}} --edge
@@ -98,7 +98,7 @@ _steps:
 {% endif -%}
 1. Check the current revision of core{{release}}
 $ snap list core{{release}}
- 2. Update to edge. 
+ 2. Update to edge.
 Note that system will automatically reboot in 1 minute after doing this command.
 Please execute the command in step 3 in 1 minute to stop the automatic reboot.
 $ sudo snap refresh core{{release}} --edge --ignore-validation
@@ -129,3 +129,26 @@ _verification:
 plugin: manual
 category_id: ubuntucore
 estimated_duration: 120
+
+unit: template
+template-resource: model_assertion
+template-unit: job
+plugin: manual
+id: ubuntucore/kernel-failboot-{kernel}
+_summary: Reboot with failboot kernel to verify system rollback function
+_purpose:
+ Check if system will rollback to previous(functional) kernel
+_description:
+ This test checks the kernel rollback function.
+ When booting a kernel that fails to boot, system should be able to rollback
+ to previous one.
+ To perform this test, a failboot kernel should be in edge/kernel-failboot-test
+ for testing.
+_steps:
+ 1. Refresh kernel to the failboot branch
+ sudo snap refresh {kernel} --channel=edge/kernel-failboot-test
+ 2. Wait for system reboot after kernel snap refresh
+_verification:
+ Check if system could boot, and kernel version is the one before switched to
+ "failboot" kernel
+category_id: ubuntucore
diff --git a/units/usb/test-plan.pxu b/units/usb/test-plan.pxu
index e4a3df1..690204e 100644
--- a/units/usb/test-plan.pxu
+++ b/units/usb/test-plan.pxu
@@ -278,14 +278,6 @@ unit: test plan
 _name: Automated USB-C tests
 _description: Automated USB-C tests for Ubuntu Core devices
 include:
- usb-c/c-to-a-adapter/hid
- usb-c/c-to-a-adapter/insert
- usb-c/c-to-a-adapter/storage-automated
- usb-c/c-to-a-adapter/remove
- usb-c/hid
- usb-c/insert
- usb-c/storage-automated
- usb-c/remove
 
 id: after-suspend-usb-full
 unit: test plan
diff --git a/units/watchdog/jobs.pxu b/units/watchdog/jobs.pxu
index 0e16f82..c2bd868 100644
--- a/units/watchdog/jobs.pxu
+++ b/units/watchdog/jobs.pxu
@@ -9,32 +9,7 @@ command: udev_resource.py -f WATCHDOG
 id: watchdog/systemd-config
 _summary: Check if the hardware watchdog is properly configured
 template-engine: jinja2
-command:
- inbuilt=$(systemctl show -p RuntimeWatchdogUSec | awk -F= '{print $2}')
- external=$(systemctl is-active watchdog.service)
- {%- if __on_ubuntucore__ %}
- if [ "$inbuilt" == "0" ]; then
- echo "systemd watchdog should be enabled but reset timeout: $inbuilt"
- exit 1
- fi
- if [ "$external" == "active" ]; then
- echo "found unexpected active watchdog.service unit"
- exit 1
- fi
- echo "systemd watchdog enabled, reset timeout: $inbuilt"
- echo "watchdog.service is not active"
- {%- else %}
- if [ "$inbuilt" != "0" ]; then
- echo "systemd watchdog should not be enabled but reset timeout: $inbuilt"
- exit 1
- fi
- if [ "$external" != "active" ]; then
- echo "watchdog.service unit does not report as active"
- exit 1
- fi
- echo "systemd watchdog disabled"
- echo "watchdog.service active"
- {% endif -%}
+command: watchdog_config_test.py
 category_id: com.canonical.plainbox::power-management
 flags: simple
 imports: from com.canonical.plainbox import manifest
diff --git a/units/wireless/jobs.pxu b/units/wireless/jobs.pxu
index 4827af1..95ddf31 100644
--- a/units/wireless/jobs.pxu
+++ b/units/wireless/jobs.pxu
@@ -551,3 +551,15 @@ command:
 estimated_duration: 2.0
 depends: wireless/nm_connection_save_{interface}
 flags: preserve-locale also-after-suspend also-after-suspend-manual
+
+unit: template
+template-resource: device
+template-filter: device.driver == 'iwlwifi'
+id: wireless/check_iwlwifi_microcode_crash_{interface}
+_summary: Check there have been no iwlwifi crashes
+plugin: shell
+command: check-iwlwifi-microcode-sw-error.sh
+category_id: com.canonical.plainbox::wireless
+estimated_duration: 30.0
+flags: preserve-locale also-after-suspend
+requires: package.name == 'systemd'
diff --git a/units/wireless/test-plan.pxu b/units/wireless/test-plan.pxu
index b7448cb..d0458cc 100644
--- a/units/wireless/test-plan.pxu
+++ b/units/wireless/test-plan.pxu
@@ -14,9 +14,6 @@ _name: Wireless tests (Manual)
 _description:
 Wireless connection tests (Manual)
 include:
-nested_part:
- wireless-cert-automated
-
 
 id: after-suspend-wireless-cert-full
 unit: test plan
@@ -44,6 +41,7 @@ include:
 wireless/wireless_connection_wpa_ax_nm_.* certification-status=blocker
 wireless/wireless_connection_open_ax_nm_.* certification-status=blocker
 wireless/nm_connection_restore_.*
+ wireless/check_iwlwifi_microcode_crash_.*
 
 id: after-suspend-wireless-cert-automated
 unit: test plan
@@ -62,6 +60,7 @@ include:
 after-suspend-wireless/wireless_connection_wpa_ax_nm_.* certification-status=blocker
 after-suspend-wireless/wireless_connection_open_ax_nm_.* certification-status=blocker
 after-suspend-wireless/nm_connection_restore_.*
+ after-suspend-wireless/check_iwlwifi_microcode_crash_.*
 
 id: after-suspend-manual-wireless-cert-automated
 unit: test plan
@@ -160,6 +159,7 @@ include:
 wireless/wireless_connection_wpa_ac_np_.*
 wireless/wireless_connection_wpa_bg_np_.*
 wireless/wireless_connection_wpa_n_np_.*
+ wireless/check_iwlwifi_microcode_crash_.*
 bootstrap_include:
 device
 
@@ -357,6 +357,7 @@ include:
 after-suspend-wireless/wireless_connection_wpa_ac_np_.*
 after-suspend-wireless/wireless_connection_wpa_bg_np_.*
 after-suspend-wireless/wireless_connection_wpa_n_np_.*
+ after-suspend-wireless/check_iwlwifi_microcode_crash_.*
 bootstrap_include:
 device