26 files changed, 351 insertions, 155 deletions

diff --git a/.bumpversion.cfg b/.bumpversion.cfg
index 718a0cb..3d4c6ab 100644
--- a/.bumpversion.cfg
+++ b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.59.0.dev0
+current_version = 0.60.0.dev0
 files = manage.py
 parse = (?P<major>\d+)\.(?P<minor>\d+)(\.(?P<patch>\d+))?((?P<release>\.?[a-z]+)(?P<N>\d+))?
 serialize = 
diff --git a/bin/check-iwlwifi-microcode-sw-error.sh b/bin/check-iwlwifi-microcode-sw-error.sh
new file mode 100755
index 0000000..e1b47fc
--- /dev/null
+++ b/bin/check-iwlwifi-microcode-sw-error.sh
@@ -0,0 +1,12 @@
+#!/bin/bash
+
+while read -r line
+do
+ bootidx=$(echo "$line" | cut -d " " -f1)
+ if journalctl -k -b "$bootidx" | grep -q "Microcode SW error detected"; then
+ echo "Boot $line, Microcode SW error detected"
+ exit 1
+ fi
+done < <(journalctl --list-boots)
+
+echo "No Microcode SW error detected"
diff --git a/bin/ipmi_test.py b/bin/ipmi_test.py
index 91f3524..f481134 100755
--- a/bin/ipmi_test.py
+++ b/bin/ipmi_test.py
@@ -73,7 +73,7 @@ class FreeIpmiTest:
 # min. ipmi version to pass
 self._ipmi_ver = 2.0
 # subprocess call timeout (s)
- self._subproc_timeout = 10
+ self._subproc_timeout = 15
 # raised subproc exceptions to handle
 # (decoupled from self._process_exc())
 self._sub_process_excs = (
diff --git a/bin/kernel_taint_test.py b/bin/kernel_taint_test.py
index 259c05c..e8ddc2a 100755
--- a/bin/kernel_taint_test.py
+++ b/bin/kernel_taint_test.py
@@ -44,10 +44,10 @@ def find_taints(taint_file):
 f = open(taint_file, "r")
 taints = int(f.read())
 except OSError:
- taints = -1
- print("Kernel taint file ({}) not found!".format(taint_file))
+ raise SystemExit(
+ "Kernel taint file ({}) not found!".format(taint_file))
 print("Kernel taint value is {}".format(taints))
- return(taints)
+ return taints
 
 
 def get_modules():
@@ -57,30 +57,60 @@ def get_modules():
 for line in lsmod_output:
 if line and 'Module' not in line:
 modules.append(line.split()[0])
- return(modules)
+ return modules
 
 
-def print_out_of_tree_modules(modules):
- print("* Modules not in-tree:")
+def process_out_of_tree_modules(modules):
+ mod_list = []
+ modules = remove_ignored_modules(modules)
 for mod in modules:
 cmd = 'modinfo -F intree %s' % mod
 if not check_output(shlex.split(cmd),
 universal_newlines=True):
- print(" %s" % mod)
+ mod_list.append(mod)
+ return mod_list
 
 
-def print_GPL_incompatible_modules(modules):
- print("* Modules with GPL Incompatible Licenses:")
+def process_GPL_incompatible_modules(modules):
+ mod_list = []
+ modules = remove_ignored_modules(modules)
 for mod in modules:
 cmd = 'modinfo -F license %s' % mod
 license = check_output(shlex.split(cmd),
 universal_newlines=True).strip()
 if "GPL" not in license and "MIT" not in license:
- print(" %s: %s" % (mod, license))
+ mod_list.append((mod, license))
+ return mod_list
+
+
+def remove_ignored_modules(modules):
+ # Remove modules we know will fail, but accept
+ ignored_modules = ['icp',
+ 'spl',
+ 'zavl',
+ 'zcommon',
+ 'zfs',
+ 'zlua',
+ 'znvpair',
+ 'zunicode',
+ 'zzstd']
+ for ignore_mod in ignored_modules:
+ try:
+ modules.remove(ignore_mod)
+ except ValueError:
+ pass
+ return modules
 
 
-def report_failures(taints):
- """Report the failure code and its meaning(s)."""
+def main():
+ """Print out the tainted state code and its meaning(s)."""
+ parser = ArgumentParser()
+ parser.add_argument('--taint-file',
+ default="/proc/sys/kernel/tainted",
+ help='The file that holds the taint information')
+ args = parser.parse_args()
+ taints = find_taints(args.taint_file)
+
 # Below meaning strings are taken from
 # https://www.kernel.org/doc/html/latest/admin-guide/tainted-kernels.html
 taint_meanings = ["proprietary module was loaded",
@@ -107,35 +137,37 @@ def report_failures(taints):
 modules = get_modules()
 print("Taint bit value: {} ({})".format(i, taint_meanings[i]))
 if i == 0: # List GPL incompatible modules and licenses
- print_GPL_incompatible_modules(modules)
- if i == 12: # List out-of-tree modules
- print_out_of_tree_modules(modules)
- if i == 11:
+ proprietary_modules = process_GPL_incompatible_modules(modules)
+ if proprietary_modules:
+ print("* Modules with GPL Incompatible Licenses:")
+ for mod in proprietary_modules:
+ print(" %s: %s" % (mod[0], mod[1]))
+ count += 1
+ else:
+ print("* Proprietary modules found, "
+ "but they are expected and OK")
+ elif i == 11:
 print("* Firmware workarounds are expected and OK")
- continue
- count += 1
-
- if taints == 0:
- print("No kernel taints detected.")
-
- if taints and count == 0:
- # we found only taint 11
- return count
+ elif i == 12: # List out-of-tree modules
+ out_of_tree_modules = process_out_of_tree_modules(modules)
+ if len(out_of_tree_modules) > 0:
+ print("* Modules not in-tree:")
+ for mod in out_of_tree_modules:
+ print(" %s" % mod)
+ count += 1
+ else:
+ print("* Out of Tree modules found, "
+ "but they are expected and OK")
+ else:
+ count += 1
+
+ if count == 0:
+ # else case below contains expected issue in case 0 / 11 / 12
+ if not taints:
+ print("No kernel taints detected.")
+ return 0
 else:
- return taints
-
-
-def main():
- parser = ArgumentParser()
- parser.add_argument('--taint-file',
- default="/proc/sys/kernel/tainted",
- help='The file that holds the taint information')
- args = parser.parse_args()
- taints = find_taints(args.taint_file)
- if taints < 0:
- return taints
-
- return(report_failures(taints))
+ return 1
 
 
 if __name__ == '__main__':
diff --git a/bin/led_sysfs_brightness.py b/bin/led_sysfs_brightness.py
new file mode 100755
index 0000000..a18f34d
--- /dev/null
+++ b/bin/led_sysfs_brightness.py
@@ -0,0 +1,46 @@
+#!/usr/bin/env python3
+#
+# Written by:
+# Kunyang Fan <kunyang_fan@aaeon.com.tw>
+
+import os
+import sys
+
+
+def led_brightness_write(led, brightness):
+ print("{} brightness -> {}".format(led, brightness), flush=True)
+ # test led devices exist
+ if not os.path.exists('/sys/class/leds/{}/brightness'.format(led)):
+ raise SystemExit('External LED {} not exist'.format(led))
+
+ with open('/sys/class/leds/{}/brightness'.format(led), 'wt') as f:
+ f.write('{}\n'.format(brightness))
+
+
+def led_arrays(model_name):
+ led_data = os.path.expandvars(
+ '$PLAINBOX_PROVIDER_DATA/led-brightness.{}.in'.format(model_name))
+ if not os.path.exists(led_data):
+ raise SystemExit(
+ "ERROR: no led information found at: {}".format(led_data))
+ with open(led_data, 'r') as f:
+ for line in f:
+ if line.startswith('#'):
+ continue
+ yield line.strip()
+
+
+def main():
+ if len(sys.argv) < 3:
+ raise SystemExit('Usage: led_syfs_brightness.py MODEL_NAME on/off')
+ model_name = sys.argv[1]
+ if sys.argv[2] == 'on':
+ brightness = 255
+ else:
+ brightness = 0
+ for led in led_arrays(model_name):
+ led_brightness_write(led, brightness)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/bin/ubuntucore_image_checks.py b/bin/ubuntucore_image_checks.py
index e381e61..ff14784 100755
--- a/bin/ubuntucore_image_checks.py
+++ b/bin/ubuntucore_image_checks.py
@@ -6,6 +6,7 @@
 # Jonathan Cave <jonathan.cave@canonical.com>
 
 import io
+import os
 import sys
 
 from checkbox_support.snap_utils.snapd import Snapd
@@ -91,10 +92,14 @@ class ModelInfo():
 print('PASS')
 
 def test_model_grade(self):
+ MODEL_GRADE = os.environ.get('MODEL_GRADE', 'secured')
 if not self.grade:
 raise SystemExit('ERROR: failed to get model grade info')
 if self.grade == 'dangerous':
 raise SystemExit('ERROR: model grade must not be dangerous')
+ if self.grade != MODEL_GRADE:
+ raise SystemExit('ERROR: model grade is "{}",'.format(self.grade) +
+ ' but "{}" is expected'.format(MODEL_GRADE))
 print('PASS')
 
 
diff --git a/bin/watchdog_config_test.py b/bin/watchdog_config_test.py
new file mode 100755
index 0000000..a650976
--- /dev/null
+++ b/bin/watchdog_config_test.py
@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+# Copyright 2021 Canonical Ltd.
+# All rights reserved.
+#
+# Written by:
+# Vic Liu <vic.liu@canonical.com>
+#
+# Checkbox is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3,
+# as published by the Free Software Foundation.
+#
+# Checkbox is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Checkbox. If not, see <http://www.gnu.org/licenses/>.
+
+import subprocess
+
+from checkbox_support.snap_utils.system import on_ubuntucore
+from checkbox_support.snap_utils.system import get_series
+
+
+def get_systemd_wdt_usec():
+ """
+ Return value of systemd-watchdog RuntimeWatchdogUSec
+ """
+ cmd = ['systemctl', 'show', '-p', 'RuntimeWatchdogUSec']
+ try:
+ result = subprocess.check_output(cmd, universal_newlines=True)
+ except Exception as err:
+ raise SystemExit("Error: {}".format(err))
+
+ if result:
+ runtime_watchdog_usec = result.split("=")[1].strip()
+ return runtime_watchdog_usec
+ else:
+ raise SystemExit(
+ "Unexpected failure occurred when executing: {}".format(cmd))
+
+
+def watchdog_service_check():
+ """
+ Check if the watchdog service is configured correctly
+ """
+ cmd = ['systemctl', 'is-active', 'watchdog.service', '--quiet']
+ try:
+ return not subprocess.run(cmd).returncode
+ except Exception as err:
+ raise SystemExit("Error: {}".format(err))
+
+
+def main():
+ runtime_watchdog_usec = get_systemd_wdt_usec()
+ systemd_wdt_configured = (runtime_watchdog_usec != "0")
+ wdt_service_configured = watchdog_service_check()
+ ubuntu_version = int(get_series().split(".")[0])
+ watchdog_config_ready = True
+
+ if (ubuntu_version >= 20) or (on_ubuntucore()):
+ if not systemd_wdt_configured:
+ print("systemd watchdog should be enabled but reset timeout: "
+ "{}".format(runtime_watchdog_usec))
+ watchdog_config_ready = False
+ if wdt_service_configured:
+ print("found unexpected active watchdog.service unit")
+ watchdog_config_ready = False
+ if watchdog_config_ready:
+ print("systemd watchdog enabled, reset timeout: {}".format(
+ runtime_watchdog_usec))
+ print("watchdog.service is not active")
+ else:
+ if systemd_wdt_configured:
+ print("systemd watchdog should not be enabled but reset timeout: "
+ "{}".format(runtime_watchdog_usec))
+ watchdog_config_ready = False
+ if not wdt_service_configured:
+ print("watchdog.service unit does not report as active")
+ watchdog_config_ready = False
+ if watchdog_config_ready:
+ print("systemd watchdog disabled")
+ print("watchdog.service active")
+
+ raise SystemExit(not watchdog_config_ready)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/bin/wwan_tests.py b/bin/wwan_tests.py
index 9bbf1e7..5790ad8 100755
--- a/bin/wwan_tests.py
+++ b/bin/wwan_tests.py
@@ -282,7 +282,7 @@ class ThreeGppConnection():
 pass
 _destroy_3gpp_connection()
 _wwan_radio_off()
- return ret_code
+ sys.exit(ret_code)
 
 
 class CountModems():
@@ -334,7 +334,7 @@ class SimPresent():
 mm = MMDbus()
 mm_id = mm.equipment_id_to_mm_id(args.hw_id)
 if not mm.sim_present(mm_id):
- return 1
+ sys.exit(1)
 
 
 class SimInfo():
diff --git a/data/gpio-loopback.UPX-TGL01.in b/data/gpio-loopback.UPX-TGL01.in
new file mode 100644
index 0000000..e705ad8
--- /dev/null
+++ b/data/gpio-loopback.UPX-TGL01.in
@@ -0,0 +1,2 @@
+# GPIO IDs as presented by sysfs that have paired together:
+22,27
diff --git a/data/led-brightness.UPX-TGL01.in b/data/led-brightness.UPX-TGL01.in
new file mode 100644
index 0000000..d4b4a9c
--- /dev/null
+++ b/data/led-brightness.UPX-TGL01.in
@@ -0,0 +1,4 @@
+# The LED IDs as presented by sysfs to test
+led:0:
+led:1:
+led:2: \ No newline at end of file
diff --git a/manage.py b/manage.py
index f946e4a..d404260 100755
--- a/manage.py
+++ b/manage.py
@@ -5,7 +5,7 @@ from plainbox.provider_manager import N_
 setup(
 name='plainbox-provider-checkbox',
 namespace='com.canonical.certification',
- version="0.59.0.dev0",
+ version="0.60.0.dev0",
 description=N_("Checkbox provider"),
 gettext_domain='plainbox-provider-checkbox',
 strict=False, deprecated=False,
diff --git a/units/bluetooth/jobs.pxu b/units/bluetooth/jobs.pxu
index 3f174bf..e1573b5 100644
--- a/units/bluetooth/jobs.pxu
+++ b/units/bluetooth/jobs.pxu
@@ -4,7 +4,7 @@ category_id: com.canonical.plainbox::bluetooth
 _summary: Make sure at least one bluetooth device is detected
 plugin: shell
 command:
- bt_list_adapters.py
+ bt_list_adapters.py && udev_resource.py -f BLUETOOTH
 estimated_duration: 2s
 flags: preserve-locale
 requires: manifest.has_bt_adapter
diff --git a/units/firmware/jobs.pxu b/units/firmware/jobs.pxu
index eb8397c..e03648d 100644
--- a/units/firmware/jobs.pxu
+++ b/units/firmware/jobs.pxu
@@ -105,47 +105,3 @@ plugin: attachment
 depends: firmware/fwts_dump
 command:
 [ -f "$PLAINBOX_SESSION_SHARE/acpidump.log" ] && gzip -c "$PLAINBOX_SESSION_SHARE/acpidump.log"
-
-
-id: firmware/tcglog-required-algs-sha256
-category_id: com.canonical.plainbox::firmware
-_summary: Test that the SHA256 algorithm is present in the TCG event log
-_description:
- Presence of support for the SHA256 algorithm is a requirement for enabling FDE
- support in Ubuntu Core 20 systems
-plugin: shell
-user: root
-command: tcglog-check -required-algs sha256
-imports: from com.canonical.plainbox import manifest
-requires:
- cpuinfo.platform == 'x86_64'
- manifest.has_tpm2_chip == 'True'
- executable.name == 'tcglog-check'
-
-id: firmware/tcglog-require-pe-image-digests
-category_id: com.canonical.plainbox::firmware
-_summary: Test format of digests for EV_EFI_BOOT_SERVICES_APPLICATION events
-_description:
- Digests for EV_EFI_BOOT_SERVICES_APPLICATION events associated with PE images
- must be PE image digests rather than file digests. This test is a requirement
- for enabling FDE support in Ubuntu Core 20 systems
-plugin: shell
-user: root
-command: tcglog-check -require-pe-image-digests
-imports: from com.canonical.plainbox import manifest
-requires:
- cpuinfo.platform == 'x86_64'
- manifest.has_tpm2_chip == 'True'
- executable.name == 'tcglog-check'
-
-id: firmware/tcglog-dump-attachment
-category_id: com.canonical.plainbox::firmware
-_summary: Attach a dump of the TCG Event log for debugging
-plugin: attachment
-user: root
-command: tcglog-dump
-imports: from com.canonical.plainbox import manifest
-requires:
- cpuinfo.platform == 'x86_64'
- manifest.has_tpm2_chip == 'True'
- executable.name == 'tcglog-dump'
diff --git a/units/firmware/test-plan.pxu b/units/firmware/test-plan.pxu
index 7db724d..4f63d19 100644
--- a/units/firmware/test-plan.pxu
+++ b/units/firmware/test-plan.pxu
@@ -11,33 +11,6 @@ bootstrap_include:
 fwts
 
 
-id: firmware-uc20-fde-full
-unit: test plan
-_name: Test firmware compatibility with UC20 FDE
-_description: Test firmware compatibility with UC20 FDE
-include:
-nested_part:
- firmware-uc20-fde-manual
- firmware-uc20-fde-automated
-
-
-id: firmware-uc20-fde-manual
-unit: test plan
-_name: Test firmware compatibility with UC20 FDE (manual)
-_description: Test firmware compatibility with UC20 FDE (manual)
-include:
-
-
-id: firmware-uc20-fde-automated
-unit: test plan
-_name: Test firmware compatibility with UC20 FDE (automated)
-_description: Test firmware compatibility with UC20 FDE (automated)
-include:
- firmware/tcglog-required-algs-sha256
- firmware/tcglog-require-pe-image-digests
- firmware/tcglog-dump-attachment
-
-
 id: iot-fwts-full
 unit: test plan
 _name: Test fwts diagnosis with iot project
diff --git a/units/gpio/test-plan.pxu b/units/gpio/test-plan.pxu
index a32be08..91d83b5 100644
--- a/units/gpio/test-plan.pxu
+++ b/units/gpio/test-plan.pxu
@@ -25,6 +25,7 @@ _name: Automated GPIO tests
 _description: Automated GPIO tests for Ubuntu Core devices
 bootstrap_include:
 model_assertion
+ dmi
 include:
 gpio/sysfs_loopback_pairs_.*
 gpio/gpiomem_loopback_pairs_.*
diff --git a/units/gpio/vendor-aaeon.pxu b/units/gpio/vendor-aaeon.pxu
new file mode 100644
index 0000000..7f8d6bc
--- /dev/null
+++ b/units/gpio/vendor-aaeon.pxu
@@ -0,0 +1,16 @@
+
+unit: template
+template-resource: dmi
+template-filter: dmi.category == 'SYSTEM' and dmi.vendor == 'AAEON' and dmi.product in ('UPX-TGL01')
+template-unit: job
+id: gpio/sysfs_loopback_pairs_{vendor}_{product}
+_summary: Test GPIO lines exposed on headers can be controlled via sysfs
+plugin: shell
+user: root
+category_id: gpio
+estimated_duration: 30.0
+flags: preserve-locale also-after-suspend
+imports: from com.canonical.plainbox import manifest
+requires: manifest.gpio_loopback == 'True'
+command:
+ gpio_sysfs_loopback.py {product}
diff --git a/units/image/jobs.pxu b/units/image/jobs.pxu
index d185550..d5aa7d4 100644
--- a/units/image/jobs.pxu
+++ b/units/image/jobs.pxu
@@ -71,16 +71,18 @@ command:
 estimated_duration: 2.0
 flags: preserve-locale
 
-id: image/model-grade-not-dangerous
+id: image/model-grade
 category_id: image
-_summary: Check that the model grade is not missing or set to dangerous
+_summary: Check that the model grade is correctly set
 requires:
 lsb.distributor_id == "Ubuntu Core" and int(lsb.release) >= 20
 _description:
 Images with the 'dangerous' grade (the lowest of all available grades)
 results in certain security measures to be relaxed.
 Images that require strict security related implementations must
- have the model grade set to a grade higher than 'dangerous'.
+ have the model grade set to non-dangerous grades - either the highest
+ grade of 'secured', or a grade passed to checkbox for checking via the
+ MODEL_GRADE configuration variable.
 plugin: shell
 command:
 ubuntucore_image_checks.py model-grade
diff --git a/units/info/jobs.pxu b/units/info/jobs.pxu
index 847b285..d2ed96b 100644
--- a/units/info/jobs.pxu
+++ b/units/info/jobs.pxu
@@ -362,6 +362,8 @@ command:
 cat /writable/system-data/etc/buildstamp
 elif [ -e /var/lib/snapd/seed/seed.yaml ]; then
 echo && date -r /var/lib/snapd/seed/seed.yaml -R
+ else
+ exit 1
 fi
 {% else -%}
 if [ -s /.disk/info ]; then # Ubuntu Classic
diff --git a/units/led/test-plan.pxu b/units/led/test-plan.pxu
index fc83aa2..b0d0129 100644
--- a/units/led/test-plan.pxu
+++ b/units/led/test-plan.pxu
@@ -115,5 +115,7 @@ include:
 led/bluetooth
 led/serial
 led/fn
-
-
+ led/sysfs_led_brightness_on_.*
+ led/sysfs_led_brightness_off_.*
+bootstrap_include:
+ dmi
diff --git a/units/led/vendor-aaeon.pxu b/units/led/vendor-aaeon.pxu
new file mode 100644
index 0000000..d4ed7cd
--- /dev/null
+++ b/units/led/vendor-aaeon.pxu
@@ -0,0 +1,39 @@
+
+unit: template
+template-resource: dmi
+template-filter: dmi.category == 'SYSTEM' and dmi.vendor == 'AAEON' and dmi.product in ('UPX-TGL01')
+template-unit: job
+id: led/sysfs_led_brightness_on_{vendor}_{product}
+plugin: user-interact-verify
+category_id: led
+estimated_duration: 30.0
+flags: preserve-locale also-after-suspend
+command:
+ led_sysfs_brightness.py {product} on
+_purpose:
+ Verify that the leds_aaeon driver is working by setting all LEDs to maximum
+ brightness
+_steps:
+ Press ENTER to start the test while watching external LEDs on the SUT
+_verification:
+ The external LEDs should now be on at maximum brightness
+
+unit: template
+template-resource: dmi
+template-filter: dmi.category == 'SYSTEM' and dmi.vendor == 'AAEON' and dmi.product in ('UPX-TGL01')
+template-unit: job
+id: led/sysfs_led_brightness_off_{vendor}_{product}
+plugin: user-interact-verify
+category_id: led
+estimated_duration: 30.0
+flags: preserve-locale also-after-suspend
+depends: led/sysfs_led_brightness_on_{vendor}_{product}
+command:
+ led_sysfs_brightness.py {product} off
+_purpose:
+ Verify that the leds_aaeon driver is working by setting all LEDs to off / 
+ minimum brightness
+_steps:
+ Press ENTER to start the test while watching external LEDs on the SUT
+_verification:
+ The external LEDs should now be off or at minimum brightness
diff --git a/units/miscellanea/test-plan.pxu b/units/miscellanea/test-plan.pxu
index 97c94fc..2f4efec 100644
--- a/units/miscellanea/test-plan.pxu
+++ b/units/miscellanea/test-plan.pxu
@@ -85,7 +85,6 @@ _description:
 mandatory_include:
 miscellanea/submission-resources
 miscellanea/cpuid
- miscellanea/efi_boot_mode certification-status=blocker
 miscellanea/ipmi_test certification-status=blocker
 miscellanea/bmc_info
 miscellanea/dmitest_server
diff --git a/units/suspend/suspend.pxu b/units/suspend/suspend.pxu
index a2830d4..cc5414d 100644
--- a/units/suspend/suspend.pxu
+++ b/units/suspend/suspend.pxu
@@ -263,7 +263,7 @@ _summary: Automated test of suspend function
 _description:
 This is the automated version of suspend/suspend_advanced.
 user: root
-environ: PLAINBOX_SESSION_SHARE
+environ: PLAINBOX_SESSION_SHARE RTC_DEVICE_FILE
 command:
 if [[ -v SNAP ]]; then
 export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$SNAP/usr/lib/fwts"
@@ -274,7 +274,11 @@ command:
 set -o pipefail; checkbox-support-fwts_test -f none -l "$PLAINBOX_SESSION_SHARE"/suspend_single -s s3 --s3-sleep-delay=30 --s3-device-check --s3-device-check-delay=45 | tee "$PLAINBOX_SESSION_SHARE"/suspend_single_times.log
 else
 echo "Calling rtcwake"
- rtcwake -m mem -s 30
+ if [ -z "$RTC_DEVICE_FILE" ]; then
+ rtcwake -m mem -s 30
+ else
+ rtcwake -d "$RTC_DEVICE_FILE" -m mem -s 30
+ fi
 fi
 estimated_duration: 90.000
 
diff --git a/units/ubuntucore/jobs.pxu b/units/ubuntucore/jobs.pxu
index 8cae699..04c30b3 100644
--- a/units/ubuntucore/jobs.pxu
+++ b/units/ubuntucore/jobs.pxu
@@ -65,7 +65,7 @@ _steps:
 {% endif -%}
 1. Check the current revision of core{{release}}
 $ snap list core{{release}}
- 2. Update to edge. 
+ 2. Update to edge.
 Note that system will automatically reboot in 1 minute after doing this command.
 Please execute the command in step 3 in 1 minute to stop the automatic reboot.
 $ sudo snap refresh core{{release}} --edge
@@ -98,7 +98,7 @@ _steps:
 {% endif -%}
 1. Check the current revision of core{{release}}
 $ snap list core{{release}}
- 2. Update to edge. 
+ 2. Update to edge.
 Note that system will automatically reboot in 1 minute after doing this command.
 Please execute the command in step 3 in 1 minute to stop the automatic reboot.
 $ sudo snap refresh core{{release}} --edge --ignore-validation
@@ -129,3 +129,26 @@ _verification:
 plugin: manual
 category_id: ubuntucore
 estimated_duration: 120
+
+unit: template
+template-resource: model_assertion
+template-unit: job
+plugin: manual
+id: ubuntucore/kernel-failboot-{kernel}
+_summary: Reboot with failboot kernel to verify system rollback function
+_purpose:
+ Check if system will rollback to previous(functional) kernel
+_description:
+ This test checks the kernel rollback function.
+ When booting a kernel that fails to boot, system should be able to rollback
+ to previous one.
+ To perform this test, a failboot kernel should be in edge/kernel-failboot-test
+ for testing.
+_steps:
+ 1. Refresh kernel to the failboot branch
+ sudo snap refresh {kernel} --channel=edge/kernel-failboot-test
+ 2. Wait for system reboot after kernel snap refresh
+_verification:
+ Check if system could boot, and kernel version is the one before switched to
+ "failboot" kernel
+category_id: ubuntucore
diff --git a/units/watchdog/jobs.pxu b/units/watchdog/jobs.pxu
index 0e16f82..c2bd868 100644
--- a/units/watchdog/jobs.pxu
+++ b/units/watchdog/jobs.pxu
@@ -9,32 +9,7 @@ command: udev_resource.py -f WATCHDOG
 id: watchdog/systemd-config
 _summary: Check if the hardware watchdog is properly configured
 template-engine: jinja2
-command:
- inbuilt=$(systemctl show -p RuntimeWatchdogUSec | awk -F= '{print $2}')
- external=$(systemctl is-active watchdog.service)
- {%- if __on_ubuntucore__ %}
- if [ "$inbuilt" == "0" ]; then
- echo "systemd watchdog should be enabled but reset timeout: $inbuilt"
- exit 1
- fi
- if [ "$external" == "active" ]; then
- echo "found unexpected active watchdog.service unit"
- exit 1
- fi
- echo "systemd watchdog enabled, reset timeout: $inbuilt"
- echo "watchdog.service is not active"
- {%- else %}
- if [ "$inbuilt" != "0" ]; then
- echo "systemd watchdog should not be enabled but reset timeout: $inbuilt"
- exit 1
- fi
- if [ "$external" != "active" ]; then
- echo "watchdog.service unit does not report as active"
- exit 1
- fi
- echo "systemd watchdog disabled"
- echo "watchdog.service active"
- {% endif -%}
+command: watchdog_config_test.py
 category_id: com.canonical.plainbox::power-management
 flags: simple
 imports: from com.canonical.plainbox import manifest
diff --git a/units/wireless/jobs.pxu b/units/wireless/jobs.pxu
index 4827af1..95ddf31 100644
--- a/units/wireless/jobs.pxu
+++ b/units/wireless/jobs.pxu
@@ -551,3 +551,15 @@ command:
 estimated_duration: 2.0
 depends: wireless/nm_connection_save_{interface}
 flags: preserve-locale also-after-suspend also-after-suspend-manual
+
+unit: template
+template-resource: device
+template-filter: device.driver == 'iwlwifi'
+id: wireless/check_iwlwifi_microcode_crash_{interface}
+_summary: Check there have been no iwlwifi crashes
+plugin: shell
+command: check-iwlwifi-microcode-sw-error.sh
+category_id: com.canonical.plainbox::wireless
+estimated_duration: 30.0
+flags: preserve-locale also-after-suspend
+requires: package.name == 'systemd'
diff --git a/units/wireless/test-plan.pxu b/units/wireless/test-plan.pxu
index b7448cb..d0458cc 100644
--- a/units/wireless/test-plan.pxu
+++ b/units/wireless/test-plan.pxu
@@ -14,9 +14,6 @@ _name: Wireless tests (Manual)
 _description:
 Wireless connection tests (Manual)
 include:
-nested_part:
- wireless-cert-automated
-
 
 id: after-suspend-wireless-cert-full
 unit: test plan
@@ -44,6 +41,7 @@ include:
 wireless/wireless_connection_wpa_ax_nm_.* certification-status=blocker
 wireless/wireless_connection_open_ax_nm_.* certification-status=blocker
 wireless/nm_connection_restore_.*
+ wireless/check_iwlwifi_microcode_crash_.*
 
 id: after-suspend-wireless-cert-automated
 unit: test plan
@@ -62,6 +60,7 @@ include:
 after-suspend-wireless/wireless_connection_wpa_ax_nm_.* certification-status=blocker
 after-suspend-wireless/wireless_connection_open_ax_nm_.* certification-status=blocker
 after-suspend-wireless/nm_connection_restore_.*
+ after-suspend-wireless/check_iwlwifi_microcode_crash_.*
 
 id: after-suspend-manual-wireless-cert-automated
 unit: test plan
@@ -160,6 +159,7 @@ include:
 wireless/wireless_connection_wpa_ac_np_.*
 wireless/wireless_connection_wpa_bg_np_.*
 wireless/wireless_connection_wpa_n_np_.*
+ wireless/check_iwlwifi_microcode_crash_.*
 bootstrap_include:
 device
 
@@ -357,6 +357,7 @@ include:
 after-suspend-wireless/wireless_connection_wpa_ac_np_.*
 after-suspend-wireless/wireless_connection_wpa_bg_np_.*
 after-suspend-wireless/wireless_connection_wpa_n_np_.*
+ after-suspend-wireless/check_iwlwifi_microcode_crash_.*
 bootstrap_include:
 device