Import plainbox-provider-checkbox_0.48.0~rc1.orig.tar.gzupstream-0.48.0_rc1patched-0.48.0_rc1-1

18 files changed, 323 insertions, 110 deletions

diff --git a/units/dock/jobs.pxu b/units/dock/jobs.pxu
index 34decb6..c425402 100644
--- a/units/dock/jobs.pxu
+++ b/units/dock/jobs.pxu
@@ -478,18 +478,22 @@ id: dock/mac-address-pass-through
 category_id: dock-network
 plugin: manual
 depends: dock/cold-plug
-estimated_duration: 60
-_summary: MAC address pass through function test
-_purpose:
- Test MAC address pass through function with docking station (Dell Dock specific function, TB15, WD15) 
-_steps:
- Skip this test if your docking station does not support MAC address pass through.
- 1. Make sure the dock is connected to network using an Ethernet cable
- 2. Run 'ifconfig' to get system specific MAC address and IP
- 3. Ping the IP address from another system
- 4. Run 'arp -a IP' from another system to get the MAC address of the test unit through dock
-_verification:
- Is the MAC address got from arp matching with specific MAC address?
+estimated_duration: 180
+_summary: MAC address passthrough function test
+_purpose:
+ Test MAC address passthrough function with dock station.
+_steps:
+ Skip this test if your dock station does not support MAC address passthrough.
+ 1. Find Passthrough MAC Address in BIOS setup
+ 2. Check "Passthrough MAC Address" is enabled
+ 2. Connect dock to system
+ 3. Boot to OS
+ 4. Run "ip a" in terminal to get dock ethernet MAC address
+_verification:
+ 1. Is the Passthrough MAC Address the same as MAC address found in step 4?
+ 2. If BIOS doesn't have Passthrough MAC Address in BIOS setup, a bug should be filed.
+ 3. If dock ethernet MAC address is not the same as BIOS Passthrough MAC Address, a 
+ bug should be filed.
 
 ### USB Tests ###
 
diff --git a/units/ethernet/jobs.pxu b/units/ethernet/jobs.pxu
index cf1c02b..b5f068a 100644
--- a/units/ethernet/jobs.pxu
+++ b/units/ethernet/jobs.pxu
@@ -2,13 +2,15 @@ plugin: shell
 category_id: com.canonical.plainbox::ethernet
 id: ethernet/detect
 flags: also-after-suspend
-command: network_device_info
+command: network_device_info.py detect
 estimated_duration: 2.0
 _summary:
- Report info about available network devices
+ Detect if at least one ethernet device is detected
 _description:
 Test to detect and return information about available network controllers on
 the system under test.
+imports: from com.canonical.plainbox import manifest
+requires: manifest.has_ethernet_adapter == 'True'
 
 plugin: shell
 category_id: com.canonical.plainbox::ethernet
@@ -82,6 +84,25 @@ template-resource: device
 template-filter: device.category == 'NETWORK' and device.interface != 'UNKNOWN'
 plugin: shell
 category_id: com.canonical.plainbox::ethernet
+id: ethernet/multi_iperf3_nic_underspeed_device{__index__}_{interface}
+_summary: Underspeed Enabled Multi-NIC Iperf3 stress testing for NIC {interface}
+estimated_duration: 7400.0
+requires:
+ package.name == 'iperf3' or executable.name == 'iperf3'
+ package.name == 'ethtool' or executable.name == 'ethtool'
+ package.name == 'nmap' or executable.name == 'nmap'
+user: root
+environ: TEST_TARGET_IPERF
+command: network test -i {interface} -t iperf --iperf3 --scan-timeout 3600 --fail-threshold 80 --cpu-load-fail-threshold 90 --runtime 900 --num_runs 4 --underspeed-ok
+_description:
+ This is the standard Multi-NIC Iperf3 test with the speed check disabled
+ for retesting systems that are incorrectly reporting supported speeds.
+
+unit: template
+template-resource: device
+template-filter: device.category == 'NETWORK' and device.interface != 'UNKNOWN'
+plugin: shell
+category_id: com.canonical.plainbox::ethernet
 id: ethernet/ethertool_check_device{__index__}_{interface}
 _summary: ethtool check for NIC {interface}
 estimated_duration: 330.0
diff --git a/units/ethernet/manifest.pxu b/units/ethernet/manifest.pxu
new file mode 100644
index 0000000..6936344
--- /dev/null
+++ b/units/ethernet/manifest.pxu
@@ -0,0 +1,4 @@
+unit: manifest entry
+id: has_ethernet_adapter
+_name: An Ethernet Adapter
+value-type: bool
diff --git a/units/ethernet/test-plan.pxu b/units/ethernet/test-plan.pxu
index 6a6af73..ec2310f 100644
--- a/units/ethernet/test-plan.pxu
+++ b/units/ethernet/test-plan.pxu
@@ -55,3 +55,27 @@ include:
 after-suspend-ethernet/ping_.*
 bootstrap_include:
 device
+
+id: server-ethernet
+unit: test plan
+_name: Server Ethernet Tests
+_description: Automated ethernet tests for server certification
+estimated_duration: 4h
+include:
+ ethernet/detect
+ ethernet/info_automated
+ ethernet/ethtool_info
+ ethernet/ethertool_check_.*
+ ethernet/multi_iperf3_nic_device.*
+
+id: server-ethernet-underspeed
+unit: test plan
+_name: Server Ethernet Tests (Speed Check Disabled)
+_description: Automated ethernet tests for server certification without speed check
+estimated_duration: 4h
+include:
+ ethernet/detect
+ ethernet/info_automated
+ ethernet/ethtool_info
+ ethernet/ethertool_check_.*
+ ethernet/multi_iperf3_nic_underspeed_device.*
diff --git a/units/gpgpu/category.pxu b/units/gpgpu/category.pxu
new file mode 100644
index 0000000..e07d1b4
--- /dev/null
+++ b/units/gpgpu/category.pxu
@@ -0,0 +1,3 @@
+unit: category
+id: gpgpu
+_name: GPGPU Compute Tests
diff --git a/units/gpgpu/jobs.pxu b/units/gpgpu/jobs.pxu
new file mode 100644
index 0000000..2c823a0
--- /dev/null
+++ b/units/gpgpu/jobs.pxu
@@ -0,0 +1,8 @@
+id: gpgpu/gpu-burn
+category_id: gpgpu
+plugin: shell
+estimated_duration: 300
+requires:
+ package.name == 'cuda'
+_summary: GPGPU stress testing
+command: cd /opt/gpu-burn/ && ./gpu_burn 1800
diff --git a/units/gpgpu/test-plan.pxu b/units/gpgpu/test-plan.pxu
new file mode 100644
index 0000000..eaf0db8
--- /dev/null
+++ b/units/gpgpu/test-plan.pxu
@@ -0,0 +1,8 @@
+id: gpgpu-tests
+unit: test plan
+_name: GPGPU Compute Testing
+_description:
+ Tests for GPGPU Computations (non-graphical)
+mandatory_include:
+ gpgpu/gpu-burn
+include:
diff --git a/units/graphics/test-plan.pxu b/units/graphics/test-plan.pxu
index 5b58844..9085cb8 100644
--- a/units/graphics/test-plan.pxu
+++ b/units/graphics/test-plan.pxu
@@ -60,7 +60,7 @@ _name: Graphics tests (discrete GPU) (Manual)
 _description:
 Graphics tests (discrete GPU) (Manual)
 include:
- graphics/2_switch_card_.*_xenial certification-status=blocker
+ graphics/2_auto_switch_card_.* certification-status=blocker
 graphics/2_maximum_resolution_.* certification-status=blocker
 graphics/2_glxgears_.* certification-status=blocker
 graphics/2_rotation_.* certification-status=blocker
@@ -88,44 +88,44 @@ unit: test plan
 _name: After suspend tests (integrated GPU)
 _description: After suspend tests (integrated GPU)
 include:
- graphics/1_switch_card_.*_xenial certification-status=blocker
- suspend/1_resolution_before_suspend_.*_xenial certification-status=blocker
+ graphics/1_auto_switch_card_.* certification-status=blocker
+ suspend/1_resolution_before_suspend_.*_auto certification-status=blocker
 # The following after suspend jobs will automatically select the right suspend job
 # depending on the amount of graphic cards available on the SUT:
 # suspend/suspend_advanced (one GPU)
- # or suspend/{{ index }}_suspend_after_switch_to_card_{{ product_slug }}_xenial (two GPUs)
- suspend/1_suspend-time-check_.*_xenial certification-status=non-blocker
- suspend/1_suspend-single-log-attach_.*_xenial
- power-management/lid certification-status=blocker
- power-management/lid_close certification-status=blocker
- power-management/lid_open certification-status=blocker
- suspend/1_compiz_check_after_suspend_.*_xenial certification-status=blocker
- suspend/1_driver_version_after_suspend_.*_xenial certification-status=blocker
- suspend/1_resolution_after_suspend_.*_xenial certification-status=blocker
- suspend/1_display_after_suspend_.*_xenial certification-status=blocker
- suspend/1_glxgears_after_suspend_.*_xenial certification-status=blocker
- suspend/1_video_after_suspend_.*_xenial certification-status=blocker
- suspend/1_cycle_resolutions_after_suspend_.*_xenial certification-status=non-blocker
- suspend/1_xrandr_screens_after_suspend.tar.gz_xenial
+ # or suspend/{{ index }}_suspend_after_switch_to_card_{{ product_slug }}_auto (two GPUs)
+ suspend/1_suspend-time-check_.*_auto certification-status=non-blocker
+ suspend/1_suspend-single-log-attach_.*_auto
+ power-management/lid certification-status=blocker
+ power-management/lid_close certification-status=blocker
+ power-management/lid_open certification-status=blocker
+ suspend/1_compiz_check_after_suspend_.*_auto certification-status=blocker
+ suspend/1_driver_version_after_suspend_.*_auto certification-status=blocker
+ suspend/1_resolution_after_suspend_.*_auto certification-status=blocker
+ suspend/1_display_after_suspend_.*_auto certification-status=blocker
+ suspend/1_glxgears_after_suspend_.*_auto certification-status=blocker
+ suspend/1_video_after_suspend_.*_auto certification-status=blocker
+ suspend/1_cycle_resolutions_after_suspend_.*_auto certification-status=non-blocker
+ suspend/1_xrandr_screens_after_suspend.tar.gz_auto
 
 id: after-suspend-graphics-discrete-gpu-cert-full
 unit: test plan
 _name: After suspend tests (discrete GPU)
 _description: After suspend tests (discrete GPU)
 include:
- suspend/2_resolution_before_suspend_.*_xenial certification-status=blocker
- suspend/2_suspend_after_switch_to_card_.*_xenial certification-status=blocker
- suspend/2_suspend-time-check_.*_xenial certification-status=non-blocker
- suspend/2_suspend-single-log-attach_.*_xenial
- suspend/2_compiz_check_after_suspend_.*_xenial certification-status=blocker
- suspend/2_driver_version_after_suspend_.*_xenial certification-status=blocker
- suspend/2_resolution_after_suspend_.*_xenial certification-status=blocker
- suspend/2_display_after_suspend_.*_xenial certification-status=blocker
- suspend/2_glxgears_after_suspend_.*_xenial certification-status=blocker
- suspend/2_video_after_suspend_.*_xenial certification-status=blocker
- suspend/2_cycle_resolutions_after_suspend_.*_xenial certification-status=non-blocker
- suspend/2_xrandr_screens_after_suspend_.*.tar.gz_xenial
- after-suspend-manual-monitor/2_dim_brightness_.* certification-status=blocker
+ suspend/2_resolution_before_suspend_.*_auto certification-status=blocker
+ suspend/2_suspend_after_switch_to_card_.*_auto certification-status=blocker
+ suspend/2_suspend-time-check_.*_auto certification-status=non-blocker
+ suspend/2_suspend-single-log-attach_.*_auto
+ suspend/2_compiz_check_after_suspend_.*_auto certification-status=blocker
+ suspend/2_driver_version_after_suspend_.*_auto certification-status=blocker
+ suspend/2_resolution_after_suspend_.*_auto certification-status=blocker
+ suspend/2_display_after_suspend_.*_auto certification-status=blocker
+ suspend/2_glxgears_after_suspend_.*_auto certification-status=blocker
+ suspend/2_video_after_suspend_.*_auto certification-status=blocker
+ suspend/2_cycle_resolutions_after_suspend_.*_auto certification-status=non-blocker
+ suspend/2_xrandr_screens_after_suspend_.*.tar.gz_auto
+ after-suspend-manual-monitor/2_dim_brightness_.* certification-status=blocker
 
 id: graphics-integrated-gpu-cert-blockers
 unit: test plan
@@ -150,7 +150,7 @@ unit: test plan
 _name: Graphics tests (discrete GPU, certification blockers only)
 _description: Graphics tests (discrete GPU, certification blockers only)
 include:
- graphics/2_switch_card_.*_xenial certification-status=blocker
+ graphics/2_auto_switch_card_.* certification-status=blocker
 graphics/2_maximum_resolution_.* certification-status=blocker
 graphics/2_valid_opengl_renderer_.* certification-status=blocker
 graphics/2_glxgears_.* certification-status=blocker
@@ -166,18 +166,18 @@ unit: test plan
 _name: After suspend tests (integrated GPU, certification blockers only)
 _description: After suspend tests (integrated GPU, certification blockers only)
 include:
- graphics/1_switch_card_.*_xenial certification-status=blocker
- suspend/1_resolution_before_suspend_.*_xenial certification-status=blocker
+ graphics/1_auto_switch_card_.* certification-status=blocker
+ suspend/1_resolution_before_suspend_.*_auto certification-status=blocker
 suspend/suspend_advanced certification-status=blocker
 power-management/lid certification-status=blocker
 power-management/lid_close certification-status=blocker
 power-management/lid_open certification-status=blocker
- suspend/1_compiz_check_after_suspend_.*_xenial certification-status=blocker
- suspend/1_driver_version_after_suspend_.*_xenial certification-status=blocker
- suspend/1_resolution_after_suspend_.*_xenial certification-status=blocker
- suspend/1_display_after_suspend_.*_xenial certification-status=blocker
- suspend/1_glxgears_after_suspend_.*_xenial certification-status=blocker
- suspend/1_video_after_suspend_.*_xenial certification-status=blocker
+ suspend/1_compiz_check_after_suspend_.*_auto certification-status=blocker
+ suspend/1_driver_version_after_suspend_.*_auto certification-status=blocker
+ suspend/1_resolution_after_suspend_.*_auto certification-status=blocker
+ suspend/1_display_after_suspend_.*_auto certification-status=blocker
+ suspend/1_glxgears_after_suspend_.*_auto certification-status=blocker
+ suspend/1_video_after_suspend_.*_auto certification-status=blocker
 after-suspend-manual-monitor/1_dim_brightness_.* certification-status=blocker
 
 id: after-suspend-graphics-discrete-gpu-cert-blockers
@@ -185,12 +185,12 @@ unit: test plan
 _name: After suspend tests (discrete GPU, certification blockers only)
 _description: After suspend tests (discrete GPU, certification blockers only)
 include:
- suspend/2_resolution_before_suspend_.*_xenial certification-status=blocker
- suspend/2_suspend_after_switch_to_card_.*_xenial certification-status=blocker
- suspend/2_compiz_check_after_suspend_.*_xenial certification-status=blocker
- suspend/2_driver_version_after_suspend_.*_xenial certification-status=blocker
- suspend/2_resolution_after_suspend_.*_xenial certification-status=blocker
- suspend/2_display_after_suspend_.*_xenial certification-status=blocker
- suspend/2_glxgears_after_suspend_.*_xenial certification-status=blocker
- suspend/2_video_after_suspend_.*_xenial certification-status=blocker
- after-suspend-manual-monitor/2_dim_brightness_.* certification-status=blocker
+ suspend/2_resolution_before_suspend_.*_auto certification-status=blocker
+ suspend/2_suspend_after_switch_to_card_.*_auto certification-status=blocker
+ suspend/2_compiz_check_after_suspend_.*_auto certification-status=blocker
+ suspend/2_driver_version_after_suspend_.*_auto certification-status=blocker
+ suspend/2_resolution_after_suspend_.*_auto certification-status=blocker
+ suspend/2_display_after_suspend_.*_auto certification-status=blocker
+ suspend/2_glxgears_after_suspend_.*_auto certification-status=blocker
+ suspend/2_video_after_suspend_.*_auto certification-status=blocker
+ after-suspend-manual-monitor/2_dim_brightness_.* certification-status=blocker
diff --git a/units/info/jobs.pxu b/units/info/jobs.pxu
index 3170566..93ccd92 100644
--- a/units/info/jobs.pxu
+++ b/units/info/jobs.pxu
@@ -310,7 +310,7 @@ _description: Lists the device driver and version for all audio devices.
 plugin: attachment
 category_id: com.canonical.plainbox::info
 id: info/network_devices
-command: network_device_info
+command: network_device_info.py info
 estimated_duration: 0.550
 _description: Provides information about network devices
 
diff --git a/units/led/test-plan.pxu b/units/led/test-plan.pxu
index 0329717..40f5697 100644
--- a/units/led/test-plan.pxu
+++ b/units/led/test-plan.pxu
@@ -25,10 +25,13 @@ include:
 led/battery-charged
 led/battery-charging
 led/battery-low
- camera/led_.*
- led/caps-lock
- led/power
- led/touchpad
+ camera/led_.* certification-status=blocker
+ led/caps-lock certification-status=blocker
+ led/numeric-keypad certification-status=blocker
+ led/power certification-status=blocker
+ led/bluetooth certification-status=non-blocker
+ led/wlan certification-status=non-blocker
+ led/wlan-disabled certification-status=non-blocker
 bootstrap_include:
 device
 
@@ -52,10 +55,13 @@ include:
 suspend/led_after_suspend/battery-charged
 suspend/led_after_suspend/battery-charging
 suspend/led_after_suspend/battery-low
- after-suspend-manual-camera/led_.*
- suspend/led_after_suspend/caps-lock
- suspend/led_after_suspend/power
- suspend/led_after_suspend/touchpad
+ after-suspend-manual-camera/led_.* certification-status=blocker
+ suspend/led_after_suspend/caps-lock certification-status=blocker
+ suspend/led_after_suspend/numeric-keypad certification-status=blocker
+ suspend/led_after_suspend/power certification-status=blocker
+ suspend/led_after_suspend/bluetooth certification-status=non-blocker
+ suspend/led_after_suspend/wlan certification-status=non-blocker
+ suspend/led_after_suspend/wlan-disabled certification-status=non-blocker
 bootstrap_include:
 device
 
diff --git a/units/oob-management/jobs.pxu b/units/oob-management/jobs.pxu
index 5615a65..cf3ebd1 100644
--- a/units/oob-management/jobs.pxu
+++ b/units/oob-management/jobs.pxu
@@ -113,14 +113,14 @@ plugin: shell
 template-engine: jinja2
 requires:
 {%- if __on_ubuntucore__ %}
- # TODO: name is a guess until snap provided
+ {# TODO: name is a guess until snap provided #}
 snap.name == 'lms'
 {%- else %}
 package.name == 'lms'
 {% endif -%}
 command:
 {%- if __on_ubuntucore__ %}
- # TODO: name is a guess until snap provided
+ {# TODO: name is a guess until snap provided #}
 SERVICE_NAME="snap.lms.lms.service"
 {%- else %}
 SERVICE_NAME="lms.service"
diff --git a/units/stress/jobs.pxu b/units/stress/jobs.pxu
index 261a0a3..756d537 100644
--- a/units/stress/jobs.pxu
+++ b/units/stress/jobs.pxu
@@ -71,7 +71,7 @@ id: power-management/suspend_30_cycles_with_reboots
 estimated_duration: 5400.0
 depends:
 power-management/rtc
- suspend/suspend_advanced
+ suspend/suspend_advanced_auto
 requires:
 executable.name == 'x-terminal-emulator'
 flags: noreturn
@@ -79,8 +79,16 @@ user: root
 environ: PM_TEST_DRY_RUN
 command:
 pm_test reboot --checkbox-respawn-cmd $PLAINBOX_SESSION_SHARE/__respawn_checkbox --fwts --log-level=debug --log-dir=$PLAINBOX_SESSION_SHARE --suspends-before-reboot=30 -r 3 --silent --check-hardware-list
+_summary: 30 suspend/resume cycles and 1 reboot, 3 times (automated stress test)
 _description:
 This is an automated stress test that will run a sequence of '30 suspend/resume cycles and one reboot' 3 times.
+_siblings: [
+ { "id": "power-management/suspend_30_cycles_with_coldboots",
+ "command": "pm_test poweroff --checkbox-respawn-cmd $PLAINBOX_SESSION_SHARE/__respawn_checkbox --fwts --log-level=debug --log-dir=$PLAINBOX_SESSION_SHARE --suspends-before-reboot=30 -r 3 --silent --check-hardware-list",
+ "_description": "This is an automated stress test that will run a sequence of '30 suspend/resume cycles and one poweroff' 3 times.",
+ "_summary": "30 suspend/resume cycles and 1 poweroff, 3 times (automated stress test)"
+ }
+ ]
 
 plugin: shell
 category_id: com.canonical.plainbox::stress
@@ -118,8 +126,17 @@ id: power-management/suspend-30-cycles-log-check-with-reboots
 depends: power-management/suspend_30_cycles_with_reboots
 estimated_duration: 1.0
 command: [ -e $PLAINBOX_SESSION_SHARE/pm_test.reboot.3.log ] && sleep_test_log_check -v s3 $PLAINBOX_SESSION_SHARE/pm_test.reboot.3.log
+_summary: 30 suspend/resume cycles and 1 reboot, 3 times (check logs for errors)
 _description:
 Automated check of the '30 cycle suspend and 1 reboot times 3' logs for errors detected by fwts.
+_siblings: [
+ { "id": "power-management/suspend-30-cycles-log-check-with-coldboots",
+ "depends": "power-management/suspend_30_cycles_with_coldboots",
+ "command": "[ -e $PLAINBOX_SESSION_SHARE/pm_test.poweroff.3.log ] && sleep_test_log_check -v s3 $PLAINBOX_SESSION_SHARE/pm_test.poweroff.3.log",
+ "_description": "Automated check of the '30 cycle suspend and 1 poweroff times 3' logs for errors detected by fwts.",
+ "_summary": "30 suspend/resume cycles and 1 poweroff, 3 times (check logs for errors)"
+ }
+ ]
 
 plugin: attachment
 category_id: com.canonical.plainbox::stress
@@ -136,8 +153,17 @@ id: power-management/suspend-30-cycle-log-attach-with-reboots
 estimated_duration: 1.0
 depends: power-management/suspend_30_cycles_with_reboots
 command: [ -e $PLAINBOX_SESSION_SHARE/pm_test.reboot.3.log ] && cat $PLAINBOX_SESSION_SHARE/pm_test.reboot.3.log
+_summary: 30 suspend/resume cycles and 1 reboot, 3 times (attach logs)
 _description:
- Attaches the log from the '30 cycle Suspend/Resume and one reboot times 3' test if it exists
+ Attaches the log from the '30 cycle suspend/resume and one reboot times 3' test if it exists
+_siblings: [
+ { "id": "power-management/suspend-30-cycle-log-attach-with-coldboots",
+ "depends": "power-management/suspend_30_cycles_with_coldboots",
+ "command": "[ -e $PLAINBOX_SESSION_SHARE/pm_test.poweroff.3.log ] && cat $PLAINBOX_SESSION_SHARE/pm_test.poweroff.3.log",
+ "_description": "Attaches the log from the '30 cycle Suspend/Resume and one poweroff times 3' test if it exists",
+ "_summary": "30 suspend/resume cycles and 1 poweroff, 3 times (attach logs)"
+ }
+ ]
 
 plugin: shell
 category_id: com.canonical.plainbox::stress
@@ -154,8 +180,17 @@ id: power-management/suspend-30-cycles-time-check-with-reboots
 estimated_duration: 1.0
 depends: power-management/suspend_30_cycles_with_reboots
 command: [ -e $PLAINBOX_SESSION_SHARE/pm_test.reboot.3.log ] && sleep_time_check $PLAINBOX_SESSION_SHARE/pm_test.reboot.3.log
+_summary: 30 suspend/resume cycles and 1 reboot, 3 times (check logs for timing issues)
 _description:
- Checks the sleep times to ensure that a machine suspends and resumes within a given threshold
+ Checks the sleep times to ensure that a machine suspends and resumes within a given threshold (warm boots)
+_siblings: [
+ { "id": "power-management/suspend-30-cycles-time-check-with-coldboots",
+ "depends": "power-management/suspend_30_cycles_with_coldboots",
+ "command": "[ -e $PLAINBOX_SESSION_SHARE/pm_test.poweroff.3.log ] && sleep_time_check $PLAINBOX_SESSION_SHARE/pm_test.poweroff.3.log",
+ "_description": "Checks the sleep times to ensure that a machine suspends and resumes within a given threshold (cold boots)",
+ "_summary": "30 suspend/resume cycles and 1 poweroff, 3 times (check logs for timing issues)"
+ }
+ ]
 
 plugin: shell
 category_id: com.canonical.plainbox::stress
diff --git a/units/stress/test-plan.pxu b/units/stress/test-plan.pxu
index 2f6ff8b..8e7929a 100644
--- a/units/stress/test-plan.pxu
+++ b/units/stress/test-plan.pxu
@@ -51,6 +51,16 @@ include:
 power-management/suspend-30-cycle-log-attach-with-reboots
 power-management/suspend-30-cycles-time-check-with-reboots
 
+id: stress-suspend-30-cycles-with-coldboots-automated
+unit: test plan
+_name: Suspend stress tests (with coldboots)
+_description: Suspend stress tests (with coldboots)
+include:
+ power-management/suspend_30_cycles_with_coldboots
+ power-management/suspend-30-cycles-log-check-with-coldboots
+ power-management/suspend-30-cycle-log-attach-with-coldboots
+ power-management/suspend-30-cycles-time-check-with-coldboots
+
 id: stress-hibernate-30-cycles-automated
 unit: test plan
 _name: Hibernate stress tests
diff --git a/units/suspend/suspend-graphics.pxu b/units/suspend/suspend-graphics.pxu
index 43a7246..be820a7 100644
--- a/units/suspend/suspend-graphics.pxu
+++ b/units/suspend/suspend-graphics.pxu
@@ -3,8 +3,8 @@ template-resource: graphics_card
 template-filter: graphics_card.prime_gpu_offload == 'Off'
 plugin: shell
 category_id: com.canonical.plainbox::suspend
-id: suspend/{index}_resolution_before_suspend_{product_slug}_xenial
-after: graphics/{index}_switch_card_{product_slug}_xenial
+id: suspend/{index}_resolution_before_suspend_{product_slug}_auto
+after: graphics/{index}_auto_switch_card_{product_slug}
 estimated_duration: 1.2
 _description: Record the current resolution before suspending.
 command:
@@ -16,11 +16,11 @@ template-resource: graphics_card
 template-filter: graphics_card.prime_gpu_offload == 'Off'
 plugin: user-interact-verify
 category_id: com.canonical.plainbox::suspend
-id: suspend/{index}_suspend_after_switch_to_card_{product_slug}_xenial
+id: suspend/{index}_suspend_after_switch_to_card_{product_slug}_auto
 requires:
 sleep.mem == 'supported'
 rtc.state == 'supported'
-after: graphics/{index}_switch_card_{product_slug}_xenial
+after: graphics/{index}_auto_switch_card_{product_slug}
 user: root
 environ: PLAINBOX_SESSION_SHARE
 command:
@@ -52,12 +52,12 @@ template-filter: graphics_card.prime_gpu_offload == 'Off'
 template-engine: jinja2
 plugin: shell
 category_id: com.canonical.plainbox::suspend
-id: suspend/{{ index }}_resolution_after_suspend_{{ product_slug }}_xenial
+id: suspend/{{ index }}_resolution_after_suspend_{{ product_slug }}_auto
 estimated_duration: 1.2
 depends:
- suspend/{{ index }}_resolution_before_suspend_{{ product_slug }}_xenial
+ suspend/{{ index }}_resolution_before_suspend_{{ product_slug }}_auto
 {%- if gpu_count > "1" %}
- suspend/{{ index }}_suspend_after_switch_to_card_{{ product_slug }}_xenial
+ suspend/{{ index }}_suspend_after_switch_to_card_{{ product_slug }}_auto
 {%- else %}
 suspend/suspend_advanced
 {%- endif %}
@@ -72,10 +72,10 @@ template-filter: graphics_card.prime_gpu_offload == 'Off'
 template-engine: jinja2
 plugin: manual
 category_id: com.canonical.plainbox::suspend
-id: suspend/{{ index }}_display_after_suspend_{{ product_slug }}_xenial
+id: suspend/{{ index }}_display_after_suspend_{{ product_slug }}_auto
 depends:
 {%- if gpu_count > "1" %}
- suspend/{{ index }}_suspend_after_switch_to_card_{{ product_slug }}_xenial
+ suspend/{{ index }}_suspend_after_switch_to_card_{{ product_slug }}_auto
 {%- else %}
 suspend/suspend_advanced
 {%- endif %}
@@ -94,11 +94,11 @@ template-filter: graphics_card.prime_gpu_offload == 'Off'
 template-engine: jinja2
 plugin: user-interact-verify
 category_id: com.canonical.plainbox::suspend
-id: suspend/{{ index }}_cycle_resolutions_after_suspend_{{ product_slug }}_xenial
+id: suspend/{{ index }}_cycle_resolutions_after_suspend_{{ product_slug }}_auto
 requires: package.name == 'xorg'
 depends:
 {%- if gpu_count > "1" %}
- suspend/{{ index }}_suspend_after_switch_to_card_{{ product_slug }}_xenial
+ suspend/{{ index }}_suspend_after_switch_to_card_{{ product_slug }}_auto
 {%- else %}
 suspend/suspend_advanced
 {%- endif %}
@@ -119,8 +119,8 @@ template-resource: graphics_card
 template-filter: graphics_card.prime_gpu_offload == 'Off'
 plugin: attachment
 category_id: com.canonical.plainbox::suspend
-id: suspend/{index}_xrandr_screens_after_suspend.tar.gz_xenial
-depends: suspend/{index}_cycle_resolutions_after_suspend_{product_slug}_xenial
+id: suspend/{index}_xrandr_screens_after_suspend.tar.gz_auto
+depends: suspend/{index}_cycle_resolutions_after_suspend_{product_slug}_auto
 command: [ -f $PLAINBOX_SESSION_SHARE/{index}_xrandr_screens_after_suspend.tgz ] && cat $PLAINBOX_SESSION_SHARE/{index}_xrandr_screens_after_suspend.tgz
 _description: This attaches screenshots from the suspend/cycle_resolutions_after_suspend test to the results submission.
 
@@ -130,10 +130,10 @@ template-filter: graphics_card.prime_gpu_offload == 'Off'
 template-engine: jinja2
 plugin: shell
 category_id: com.canonical.plainbox::suspend
-id: suspend/{{ index }}_compiz_check_after_suspend_{{ product_slug }}_xenial
+id: suspend/{{ index }}_compiz_check_after_suspend_{{ product_slug }}_auto
 depends:
 {%- if gpu_count > "1" %}
- suspend/{{ index }}_suspend_after_switch_to_card_{{ product_slug }}_xenial
+ suspend/{{ index }}_suspend_after_switch_to_card_{{ product_slug }}_auto
 {%- else %}
 suspend/suspend_advanced
 {%- endif %}
@@ -150,10 +150,10 @@ template-resource: graphics_card
 template-engine: jinja2
 plugin: user-interact-verify
 category_id: com.canonical.plainbox::suspend
-id: suspend/{{ index }}_glxgears_after_suspend_{{ product_slug }}_xenial
+id: suspend/{{ index }}_glxgears_after_suspend_{{ product_slug }}_auto
 depends:
 {%- if gpu_count > "1" %}
- suspend/{{ index }}_suspend_after_switch_to_card_{{ product_slug }}_xenial
+ suspend/{{ index }}_suspend_after_switch_to_card_{{ product_slug }}_auto
 {%- else %}
 suspend/suspend_advanced
 {%- endif %}
@@ -176,10 +176,10 @@ _description:
 unit: template
 template-resource: graphics_card
 template-engine: jinja2
-id: suspend/{{ index }}_video_after_suspend_{{ product_slug }}_xenial
+id: suspend/{{ index }}_video_after_suspend_{{ product_slug }}_auto
 depends:
 {%- if gpu_count > "1" %}
- suspend/{{ index }}_suspend_after_switch_to_card_{{ product_slug }}_xenial
+ suspend/{{ index }}_suspend_after_switch_to_card_{{ product_slug }}_auto
 {%- else %}
 suspend/suspend_advanced
 {%- endif %}
@@ -206,10 +206,10 @@ template-filter: graphics_card.prime_gpu_offload == 'Off'
 template-engine: jinja2
 plugin: shell
 category_id: com.canonical.plainbox::suspend
-id: suspend/{{ index }}_driver_version_after_suspend_{{ product_slug }}_xenial
+id: suspend/{{ index }}_driver_version_after_suspend_{{ product_slug }}_auto
 depends:
 {%- if gpu_count > "1" %}
- suspend/{{ index }}_suspend_after_switch_to_card_{{ product_slug }}_xenial
+ suspend/{{ index }}_suspend_after_switch_to_card_{{ product_slug }}_auto
 {%- else %}
 suspend/suspend_advanced
 {%- endif %}
@@ -226,10 +226,10 @@ template-filter: graphics_card.prime_gpu_offload == 'Off'
 template-engine: jinja2
 plugin: attachment
 category_id: com.canonical.plainbox::suspend
-id: suspend/{{ index }}_suspend-single-log-attach_{{ product_slug }}_xenial
+id: suspend/{{ index }}_suspend-single-log-attach_{{ product_slug }}_auto
 depends:
 {%- if gpu_count > "1" %}
- suspend/{{ index }}_suspend_after_switch_to_card_{{ product_slug }}_xenial
+ suspend/{{ index }}_suspend_after_switch_to_card_{{ product_slug }}_auto
 {%- else %}
 suspend/suspend_advanced
 {%- endif %}
@@ -243,10 +243,10 @@ template-filter: graphics_card.prime_gpu_offload == 'Off'
 template-engine: jinja2
 plugin: shell
 category_id: com.canonical.plainbox::suspend
-id: suspend/{{ index }}_suspend-time-check_{{ product_slug }}_xenial
+id: suspend/{{ index }}_suspend-time-check_{{ product_slug }}_auto
 depends:
 {%- if gpu_count > "1" %}
- suspend/{{ index }}_suspend_after_switch_to_card_{{ product_slug }}_xenial
+ suspend/{{ index }}_suspend_after_switch_to_card_{{ product_slug }}_auto
 {%- else %}
 suspend/suspend_advanced
 {%- endif %}
diff --git a/units/suspend/test-plan.pxu b/units/suspend/test-plan.pxu
index e7359c0..edc6aaa 100644
--- a/units/suspend/test-plan.pxu
+++ b/units/suspend/test-plan.pxu
@@ -22,8 +22,8 @@ include:
 keys/sleep certification-status=blocker
 suspend/oops_after_suspend certification-status=blocker
 suspend/oops_results_after_suspend.log
- led/power-blink-suspend
- led/suspend
+ led/power-blink-suspend certification-status=blocker
+ led/suspend certification-status=blocker
 
 id: before-suspend-reference-cert-automated
 unit: test plan
diff --git a/units/wireless/jobs.pxu b/units/wireless/jobs.pxu
index 52b2b01..b3d6604 100644
--- a/units/wireless/jobs.pxu
+++ b/units/wireless/jobs.pxu
@@ -7,7 +7,7 @@ id: wireless/wireless_scanning_{{ interface }}
 _summary: Test system can discover Wi-Fi networks on {{ interface }}
 command:
 net_driver_info $NET_DRIVER_INFO
- wifi_nmcli_test scan {{ interface }}
+ wifi_nmcli_test.py scan {{ interface }}
 plugin: shell
 category_id: com.canonical.plainbox::wireless
 estimated_duration: 6
@@ -31,7 +31,7 @@ _purpose:
 plugin: shell
 command:
 net_driver_info $NET_DRIVER_INFO
- wifi_nmcli_test secured {{ interface }} "$WPA_BG_SSID" "$WPA_BG_PSK"
+ wifi_nmcli_test.py secured {{ interface }} "$WPA_BG_SSID" "$WPA_BG_PSK"
 category_id: com.canonical.plainbox::wireless
 estimated_duration: 30.0
 flags: preserve-locale also-after-suspend also-after-suspend-manual
@@ -52,7 +52,7 @@ _purpose:
 plugin: shell
 command:
 net_driver_info $NET_DRIVER_INFO
- wifi_nmcli_test open {{ interface }} "$OPEN_BG_SSID"
+ wifi_nmcli_test.py open {{ interface }} "$OPEN_BG_SSID"
 category_id: com.canonical.plainbox::wireless
 estimated_duration: 30.0
 flags: preserve-locale also-after-suspend also-after-suspend-manual
@@ -73,7 +73,7 @@ _purpose:
 plugin: shell
 command:
 net_driver_info $NET_DRIVER_INFO
- wifi_nmcli_test secured {{ interface }} "$WPA_N_SSID" "$WPA_N_PSK"
+ wifi_nmcli_test.py secured {{ interface }} "$WPA_N_SSID" "$WPA_N_PSK"
 category_id: com.canonical.plainbox::wireless
 estimated_duration: 30.0
 flags: preserve-locale also-after-suspend also-after-suspend-manual
@@ -94,7 +94,7 @@ _purpose:
 plugin: shell
 command:
 net_driver_info $NET_DRIVER_INFO
- wifi_nmcli_test open {{ interface }} "$OPEN_N_SSID"
+ wifi_nmcli_test.py open {{ interface }} "$OPEN_N_SSID"
 category_id: com.canonical.plainbox::wireless
 estimated_duration: 30.0
 flags: preserve-locale also-after-suspend also-after-suspend-manual
@@ -115,7 +115,7 @@ _purpose:
 plugin: shell
 command:
 net_driver_info $NET_DRIVER_INFO
- wifi_nmcli_test secured {{ interface }} "$WPA_AC_SSID" "$WPA_AC_PSK"
+ wifi_nmcli_test.py secured {{ interface }} "$WPA_AC_SSID" "$WPA_AC_PSK"
 category_id: com.canonical.plainbox::wireless
 estimated_duration: 30.0
 flags: preserve-locale also-after-suspend also-after-suspend-manual
@@ -137,7 +137,7 @@ _purpose:
 plugin: shell
 command:
 net_driver_info $NET_DRIVER_INFO
- wifi_nmcli_test open {{ interface }} "$OPEN_AC_SSID"
+ wifi_nmcli_test.py open {{ interface }} "$OPEN_AC_SSID"
 category_id: com.canonical.plainbox::wireless
 estimated_duration: 30.0
 flags: preserve-locale also-after-suspend also-after-suspend-manual
@@ -442,3 +442,31 @@ command:
 estimated_duration: 330.0
 _description:
 Tests the performance of a system's wireless connection through the iperf tool, using UDP packets.
+
+
+unit: template
+template-resource: device
+template-filter: device.category == 'WIRELESS' and device.interface != 'UNKNOWN'
+id: wireless/nm_connection_save_{interface}
+category_id: com.canonical.plainbox::wireless
+_summary: Save any NetworkManager 802.11 configurations prior to testing
+plugin: shell
+user: root
+command:
+ wifi_nmcli_backup.py save
+estimated_duration: 2.0
+flags: preserve-locale also-after-suspend also-after-suspend-manual
+
+unit: template
+template-resource: device
+template-filter: device.category == 'WIRELESS' and device.interface != 'UNKNOWN'
+id: wireless/nm_connection_restore_{interface}
+category_id: com.canonical.plainbox::wireless
+_summary: Restore any NetworkManager 802.11 configurations after testing
+plugin: shell
+user: root
+command:
+ wifi_nmcli_backup.py restore
+estimated_duration: 2.0
+depends: wireless/nm_connection_save_{interface}
+flags: preserve-locale also-after-suspend also-after-suspend-manual
diff --git a/units/wireless/nm-hotspot.pxu b/units/wireless/nm-hotspot.pxu
new file mode 100644
index 0000000..5ffbe2b
--- /dev/null
+++ b/units/wireless/nm-hotspot.pxu
@@ -0,0 +1,42 @@
+
+unit: template
+template-resource: device
+template-filter: device.category == 'WIRELESS' and device.interface != 'UNKNOWN'
+template-engine: jinja2
+template-unit: job
+id: wireless/nmcli_wifi_ap_bg_{{ interface }}
+_summary: Create 802.11b/g Wi-Fi AP on {{ interface }} using NetworkManager
+command:
+ net_driver_info $NET_DRIVER_INFO
+ wifi_nmcli_test.py ap {{ interface }} bg
+plugin: shell
+category_id: com.canonical.plainbox::wireless
+estimated_duration: 10
+flags: preserve-locale also-after-suspend
+requires:
+ {%- if __on_ubuntucore__ %}
+ connections.slot == 'network-manager:service' and connections.plug == '{{ __system_env__["SNAP_NAME"] }}:network-manager'
+ {%- else %}
+ executable.name == 'nmcli'
+ {% endif -%}
+
+unit: template
+template-resource: device
+template-filter: device.category == 'WIRELESS' and device.interface != 'UNKNOWN'
+template-engine: jinja2
+template-unit: job
+id: wireless/nmcli_wifi_ap_a_{{ interface }}
+_summary: Create 802.11a Wi-Fi AP on {{ interface }} using NetworkManager
+command:
+ net_driver_info $NET_DRIVER_INFO
+ wifi_nmcli_test.py ap {{ interface }} a
+plugin: shell
+category_id: com.canonical.plainbox::wireless
+estimated_duration: 10
+flags: preserve-locale also-after-suspend
+requires:
+ {%- if __on_ubuntucore__ %}
+ connections.slot == 'network-manager:service' and connections.plug == '{{ __system_env__["SNAP_NAME"] }}:network-manager'
+ {%- else %}
+ executable.name == 'nmcli'
+ {% endif -%} \ No newline at end of file
diff --git a/units/wireless/test-plan.pxu b/units/wireless/test-plan.pxu
index 74bf271..8b4aafe 100644
--- a/units/wireless/test-plan.pxu
+++ b/units/wireless/test-plan.pxu
@@ -30,7 +30,10 @@ id: wireless-cert-automated
 unit: test plan
 _name: Wireless tests
 _description: Wireless connection tests
+bootstrap_include:
+ device
 include:
+ wireless/nm_connection_save_.*
 wireless/wireless_scanning_.* certification-status=blocker
 wireless/wireless_connection_wpa_bg_nm_.* certification-status=blocker
 wireless/wireless_connection_open_bg_nm_.* certification-status=blocker
@@ -38,36 +41,48 @@ include:
 wireless/wireless_connection_open_n_nm_.* certification-status=blocker
 wireless/wireless_connection_wpa_ac_nm_.* certification-status=blocker
 wireless/wireless_connection_open_ac_nm_.* certification-status=blocker
+ wireless/nm_connection_restore_.*
 
 id: after-suspend-wireless-cert-automated
 unit: test plan
 _name: Wireless tests (after suspend, automated)
 _description: Wireless connection tests (after suspend, automated)
+bootstrap_include:
+ device
 include:
+ after-suspend-wireless/nm_connection_save_.*
 after-suspend-wireless/wireless_connection_wpa_bg_nm_.* certification-status=blocker
 after-suspend-wireless/wireless_connection_open_bg_nm_.* certification-status=blocker
 after-suspend-wireless/wireless_connection_wpa_n_nm_.* certification-status=blocker
 after-suspend-wireless/wireless_connection_open_n_nm_.* certification-status=blocker
 after-suspend-wireless/wireless_connection_wpa_ac_nm_.* certification-status=blocker
 after-suspend-wireless/wireless_connection_open_ac_nm_.* certification-status=blocker
+ after-suspend-wireless/nm_connection_restore_.*
 
 id: after-suspend-manual-wireless-cert-automated
 unit: test plan
 _name: Wireless tests (after manual suspend, automated)
 _description: Wireless connection tests (after manual suspend, automated)
+bootstrap_include:
+ device
 include:
+ after-suspend-manual-wireless/nm_connection_save_.*
 after-suspend-manual-wireless/wireless_connection_wpa_bg_nm_.* certification-status=blocker
 after-suspend-manual-wireless/wireless_connection_open_bg_nm_.* certification-status=blocker
 after-suspend-manual-wireless/wireless_connection_wpa_n_nm_.* certification-status=blocker
 after-suspend-manual-wireless/wireless_connection_open_n_nm_.* certification-status=blocker
 after-suspend-manual-wireless/wireless_connection_wpa_ac_nm_.* certification-status=blocker
 after-suspend-manual-wireless/wireless_connection_open_ac_nm_.* certification-status=blocker
+ after-suspend-manual-wireless/nm_connection_restore_.*
 
 id: wireless-cert-blockers
 unit: test plan
 _name: Wireless tests (certification blockers only)
 _description: Wireless connection tests (certification blockers only)
+bootstrap_include:
+ device
 include:
+ wireless/nm_connection_save_.*
 wireless/wireless_scanning_.* certification-status=blocker
 wireless/wireless_connection_wpa_bg_nm_.* certification-status=blocker
 wireless/wireless_connection_open_bg_nm_.* certification-status=blocker
@@ -75,16 +90,21 @@ include:
 wireless/wireless_connection_open_n_nm_.* certification-status=blocker
 wireless/wireless_connection_wpa_ac_nm_.* certification-status=blocker
 wireless/wireless_connection_open_ac_nm_.* certification-status=blocker
+ wireless/nm_connection_restore_.*
 
 id: after-suspend-wireless-cert-blockers
 unit: test plan
 _name: Wireless tests (after manual suspend, certification blockers only)
 _description:
 Wireless connection tests (after manual suspend, certification blockers only)
+bootstrap_include:
+ device
 include:
+ after-suspend-manual-wireless/nm_connection_save_.*
 after-suspend-manual-wireless/wireless_connection_wpa_bg_nm_.* certification-status=blocker
 after-suspend-manual-wireless/wireless_connection_open_bg_nm_.* certification-status=blocker
 after-suspend-manual-wireless/wireless_connection_wpa_n_nm_.* certification-status=blocker
 after-suspend-manual-wireless/wireless_connection_open_n_nm_.* certification-status=blocker
 after-suspend-manual-wireless/wireless_connection_wpa_ac_nm_.* certification-status=blocker
 after-suspend-manual-wireless/wireless_connection_open_ac_nm_.* certification-status=blocker
+ after-suspend-manual-wireless/nm_connection_restore_.*