diff options
Diffstat (limited to 'units')
35 files changed, 947 insertions, 183 deletions
diff --git a/units/audio/jobs.pxu b/units/audio/jobs.pxu index 0745e5c..126209e 100644 --- a/units/audio/jobs.pxu +++ b/units/audio/jobs.pxu @@ -679,6 +679,8 @@ flags: also-after-suspend command: alsa_test loopback -d 5 category_id: com.canonical.plainbox::audio estimated_duration: 5 +imports: from com.canonical.plainbox import manifest +requires: manifest.has_audio_loopback_connector == 'True' id: audio/alsa-loopback _summary: Captured sound matches played one diff --git a/units/audio/manifest.pxu b/units/audio/manifest.pxu index 95417f6..d3e285b 100644 --- a/units/audio/manifest.pxu +++ b/units/audio/manifest.pxu @@ -13,3 +13,9 @@ unit: manifest entry id: has_audio_capture _name: Audio capture value-type: bool + +unit: manifest entry +id: has_audio_loopback_connector +prompt: Does this device have the following?: +_name: Audio Loopback Connector +value-type: bool diff --git a/units/bluetooth/test-plan.pxu b/units/bluetooth/test-plan.pxu index 69497ae..9de4d37 100644 --- a/units/bluetooth/test-plan.pxu +++ b/units/bluetooth/test-plan.pxu @@ -17,7 +17,6 @@ include: bluetooth/audio-a2dp certification-status=blocker bluetooth4/HOGP-mouse certification-status=blocker bluetooth4/HOGP-keyboard certification-status=blocker - bluetooth/file-transfer certification-status=non-blocker id: bluetooth-cert-automated unit: test plan diff --git a/units/camera/jobs.pxu b/units/camera/jobs.pxu index 0605d18..d7ad2aa 100644 --- a/units/camera/jobs.pxu +++ b/units/camera/jobs.pxu @@ -6,7 +6,19 @@ requires: device.category == 'CAPTURE' command: camera_test detect -_description: This Automated test attempts to detect a camera. +_summary: This Automated test attempts to detect a camera. +user: root + +plugin: shell +category_id: com.canonical.plainbox::camera +id: camera/detect-rpi +estimated_duration: 1.0 +imports: from com.canonical.plainbox import manifest +requires: + manifest.has_rpi_camera == 'True' +command: + udev_resource -f MMAL | grep "category: MMAL" +_summary: Detect presence of a MMAL camera. unit: template template-resource: device @@ -103,3 +115,19 @@ _description: Takes multiple pictures based on the resolutions supported by the camera and validates their size and that they are of a valid format. +unit: template +template-resource: device +template-filter: device.category == 'MMAL' and device.name != '' +template-unit: job +plugin: shell +category_id: com.canonical.plainbox::camera +id: camera/multiple-resolution-images-rpi_{name} +_summary: Webcam multiple resolution capture test for Pi Camera +estimated_duration: 16.0 +depends: camera/detect-rpi +command: + camera_test_rpi.py --device /dev/{name} +_description: + Takes multiple pictures based on the resolutions supported by the camera and + validates their size and that they are of a valid format. +user: root \ No newline at end of file diff --git a/units/camera/manifest.pxu b/units/camera/manifest.pxu new file mode 100644 index 0000000..de12798 --- /dev/null +++ b/units/camera/manifest.pxu @@ -0,0 +1,4 @@ +unit: manifest entry +id: has_rpi_camera +_name: RaspberryPi Camera Module +value-type: bool \ No newline at end of file diff --git a/units/camera/test-plan.pxu b/units/camera/test-plan.pxu index 578a7ce..c3b8378 100644 --- a/units/camera/test-plan.pxu +++ b/units/camera/test-plan.pxu @@ -60,3 +60,29 @@ include: after-suspend-manual-camera/display_.* certification-status=blocker bootstrap_include: device + +id: camera-full +unit: test plan +_name: Camera tests +_description: Camera tests for Ubuntu Core devices +include: +nested_part: + camera-automated + camera-manual + +id: camera-manual +unit: test plan +_name: Manual camera tests +_description: Manual camera tests for Ubuntu Core devices +include: + +id: camera-automated +unit: test plan +_name: Automated camera tests +_description: Automated camera tests for Ubuntu Core devices +estimated_duration: 1h30m +include: + camera/multiple-resolution-images-_.* + camera/multiple-resolution-images-rpi_.* +bootstrap_include: + device diff --git a/units/cpu/jobs.pxu b/units/cpu/jobs.pxu index a027aa9..dccf2e4 100644 --- a/units/cpu/jobs.pxu +++ b/units/cpu/jobs.pxu @@ -2,7 +2,9 @@ plugin: shell category_id: com.canonical.plainbox::cpu id: cpu/scaling_test estimated_duration: 150.0 -requires: executable.name == 'fwts' +requires: + executable.name == 'fwts' + 'userspace' in cpuinfo.governors user: root environ: PLAINBOX_SESSION_SHARE LD_LIBRARY_PATH SNAP command: diff --git a/units/dock/jobs.pxu b/units/dock/jobs.pxu index c425402..c457102 100644 --- a/units/dock/jobs.pxu +++ b/units/dock/jobs.pxu @@ -152,6 +152,23 @@ _verification: Was the desktop displayed correctly on the screen connected using a "USB Type-C to VGA" adapter in every mode? +id: dock/monitor-thunderbolt3 +category_id: dock-display +depends: dock/cold-plug +imports: from com.canonical.plainbox import manifest +requires: manifest.has_thunderbolt3 == 'True' +estimated_duration: 120.0 +_summary: Thunderbolt3 port test on the dock +plugin: manual +_purpose: + This test will check the Thunderbolt3 port(s) on the dock. +_steps: + Skip this test if the dock does not have a Thunderbolt3 port. + 1. Connect a display (if not already connected) to the Thunderbolt3 port on the dock + 2. Repeat step 1 for each additional Thunderbolt3 port, if any +_verification: + Was the desktop displayed correctly on both screens? + id: dock/monitor-dual-head category_id: dock-display depends: dock/cold-plug @@ -159,9 +176,9 @@ estimated_duration: 120.0 _summary: Dual monitors test while docked plugin: manual _purpose: - This test verifies that dual-monitor output works using the dock. You will need two external monitors to perform this test. + This test verifies that dual-monitor output works using the dock. You will need two external monitors to perform this test. Depends on the specification of the device to use 4K monitor or FHD monitor. _steps: - Skip this test if your video card or the dock do not support multiple monitors. Use 4K monitors to run this test if the video card and dock both support. + Skip this test if your video card or the dock do not support multiple monitors. 1. If your dock provides more than one monitor outputs, connect two monitors 2. Open the "Displays" tool (open the dash and search for "Displays") 3. Configure your output to provide one desktop across all the monitors @@ -169,14 +186,14 @@ _steps: _verification: Was the stretched desktop displayed correctly across all the screens? -id: dock/monitor-tripple-head +id: dock/monitor-triple-head category_id: dock-display depends: dock/cold-plug estimated_duration: 120.0 _summary: Tripple monitors test while docked plugin: manual _purpose: - This test verifies that tripple-monitor output works using the dock. You will need three external monitors to perform this test. + This test verifies that triple-monitor output works using the dock. You will need three external monitors to perform this test. Depends on the specification of the device to use 4K monitor or FHD monitor. _steps: Skip this test if your video card or the dock do not support multiple monitors. 1. If your dock provides more than one monitor outputs, connect three monitors @@ -225,6 +242,7 @@ requires: package.name == 'pulseaudio-utils' _summary: HDMI audio test plugin: user-interact-verify +flags: also-after-suspend-manual estimated_duration: 30.0 command: audio_settings store --verbose --file=$PLAINBOX_SESSION_SHARE/pulseaudio_settings @@ -252,6 +270,7 @@ requires: package.name == 'pulseaudio-utils' _summary: DisplayPort audio test plugin: user-interact-verify +flags: also-after-suspend-manual estimated_duration: 30.0 command: audio_settings store --verbose --file=$PLAINBOX_SESSION_SHARE/pulseaudio_settings @@ -269,9 +288,96 @@ _steps: _verification: Did you hear the sound from the DisplayPort device? +id: dock/audio-playback-type-c-hdmi +category_id: dock-audio +depends: dock/cold-plug +requires: + device.category == 'AUDIO' + package.name == 'alsa-base' + package.name == 'gir1.2-gst-plugins-base-0.10' or package.name == 'gir1.2-gst-plugins-base-1.0' + package.name == 'pulseaudio-utils' +_summary: HDMI audio test +plugin: user-interact-verify +estimated_duration: 30.0 +command: + audio_settings store --verbose --file=$PLAINBOX_SESSION_SHARE/pulseaudio_settings + audio_settings set --verbose --device=hdmi --volume=50 + gst_pipeline_test -t 2 --device hdmi 'audiotestsrc wave=sine freq=512 ! audioconvert ! audioresample ! autoaudiosink' + EXIT_CODE=$? + audio_settings restore --verbose --file=$PLAINBOX_SESSION_SHARE/pulseaudio_settings + exit $EXIT_CODE +_purpose: + Dock USB Type-C HDMI audio interface verification +_steps: + 1. Plug an external HDMI device with sound on a USB Type-C port using a "USB Typce-C to HDMI" adapter on Dock (use only one HDMI/DisplayPort/Thunderbolt interface at a time for this test) + 2. Go to the Sound settings and make sure the correct Output is selected + 3. Click the Test button +_verification: + Did you hear the sound from the HDMI device? + +id: dock/audio-playback-type-c-displayport +category_id: dock-audio +depends: dock/cold-plug +requires: + device.category == 'AUDIO' + package.name == 'alsa-base' + package.name == 'gir1.2-gst-plugins-base-0.10' or package.name == 'gir1.2-gst-plugins-base-1.0' + package.name == 'pulseaudio-utils' +_summary: HDMI audio test +plugin: user-interact-verify +flags: also-after-suspend-manual +estimated_duration: 30.0 +command: + audio_settings store --verbose --file=$PLAINBOX_SESSION_SHARE/pulseaudio_settings + audio_settings set --verbose --device=hdmi --volume=50 + gst_pipeline_test -t 2 --device hdmi 'audiotestsrc wave=sine freq=512 ! audioconvert ! audioresample ! autoaudiosink' + EXIT_CODE=$? + audio_settings restore --verbose --file=$PLAINBOX_SESSION_SHARE/pulseaudio_settings + exit $EXIT_CODE +_purpose: + Dock USB Type-C Displayport audio interface verification +_steps: + 1. Plug an external Displayport device with sound on a USB Type-C port using a "USB Typce-C to Displayport" adapter on Dock (use only one HDMI/DisplayPort/Thunderbolt interface at a time for this test) + 2. Go to the Sound settings and make sure the correct Output is selected + 3. Click the Test button +_verification: + Did you hear the sound from the Displayport device? + +id: dock/audio-playback-thunderbolt3 +category_id: dock-audio +depends: dock/cold-plug +imports: from com.canonical.plainbox import manifest +requires: + device.category == 'AUDIO' + package.name == 'alsa-base' + package.name == 'gir1.2-gst-plugins-base-0.10' or package.name == 'gir1.2-gst-plugins-base-1.0' + package.name == 'pulseaudio-utils' + manifest.has_thunderbolt3 == 'True' +_summary: DisplayPort audio test +plugin: user-interact-verify +flags: also-after-suspend-manual +estimated_duration: 30.0 +command: + audio_settings store --verbose --file=$PLAINBOX_SESSION_SHARE/pulseaudio_settings + audio_settings set --verbose --device=hdmi --volume=50 + gst_pipeline_test -t 2 --device hdmi 'audiotestsrc wave=sine freq=512 ! audioconvert ! audioresample ! autoaudiosink' + EXIT_CODE=$? + audio_settings restore --verbose --file=$PLAINBOX_SESSION_SHARE/pulseaudio_settings + exit $EXIT_CODE +_purpose: + Dock Thunderbolt3 audio interface verification +_steps: + 1. Plug an external Thunderbolt3 device with sound to the dock (use only one HDMI/DisplayPort/Thunderbolt3 interface at a time for this test) + 2. Go to the Sound settings and make sure the correct Output is selected + 3. Click the Test button +_verification: + Did you hear the sound from the Thunderbolt3 device? + + id: dock/audio-playback-headphones category_id: dock-audio plugin: user-interact-verify +flags: also-after-suspend-manual estimated_duration: 30.0 _summary: Headphones output test depends: dock/audio-list-devices @@ -292,7 +398,8 @@ _purpose: (Skip this test if there is no headphone connector on the dock) _steps: 1. Connect a pair of headphones to the dock - 2. Click the Test button to play a sound to your audio device + 2. Go to the Sound settings and make sure the correct Output is selected + 3. Click the Test button to play a sound to your audio device _verification: Did you hear a sound through the headphones and did the sound play without any distortion, clicks or other strange noises from your headphones? @@ -300,6 +407,7 @@ plugin: user-interact-verify category_id: dock-audio id: dock/audio-alsa-record-playback-external estimated_duration: 30.0 +flags: also-after-suspend-manual _summary: External microphone plugged to the dock to record sound test depends: dock/audio-playback-headphones requires: @@ -319,8 +427,9 @@ _purpose: (Skip this test if the dock does not have a microphone connector) _steps: 1. Connect a microphone to the dock's microphone port - 2. Click "Test", then speak into the external microphone - 3. After a few seconds, your speech will be played back to you + 2. Go to the Sound settings and make sure the correct Output is selected + 3. Click "Test", then speak into the external microphone + 4. After a few seconds, your speech will be played back to you _verification: Did you hear your speech played back? @@ -883,6 +992,27 @@ _description: Did a notification show and was the connection correctly established? plugin: user-interact +id: dock/hotplug-usb-insert +category_id: dock-hotplug +depends: dock/hot-plug +estimated_duration: 10.0 +_summary: USB drive insertion test +command: removable_storage_watcher insert usb +_purpose: + This test will check that the system correctly detects the insertion of + a USB storage device plugged on the dock +_steps: + 1. Click "Test" and insert a USB storage device, preferably a HDD, + in one of the dock's port. + Although a USB pen drive may be used it might cause performance + related tests to fail. (Note: this test will time-out after 20 + seconds.) + 2. Do not unplug the device after the test. +_verification: + The verification of this test is automated. Do not change the + automatically selected result. + +plugin: user-interact id: dock/hotplug-usb3-insert category_id: dock-hotplug depends: dock/hot-plug @@ -903,6 +1033,17 @@ _verification: automatically selected result. plugin: shell +id: dock/hotplug-usb-storage-automated +category_id: dock-hotplug +depends: dock/hotplug-usb-insert +user: root +estimated_duration: 45.0 +_summary: USB drive storage test +command: removable_storage_test -s 268400000 usb +_description: + This test is automated and executes after the dock/usb_insert test is run. + +plugin: shell id: dock/hotplug-usb3-storage-automated category_id: dock-hotplug depends: dock/hotplug-usb3-insert @@ -914,6 +1055,23 @@ _description: This test is automated and executes after the dock/usb3_insert test is run. plugin: user-interact +id: dock/hotplug-usb-remove +category_id: dock-hotplug +depends: dock/hotplug-usb-insert +estimated_duration: 10.0 +_summary: USB drive removal test +command: removable_storage_watcher remove usb +_purpose: + This test will check that the system correctly detects the removal of + a USB storage device +_steps: + 1. Click "Test" and remove the USB device from the dock. + (Note: this test will time-out after 20 seconds.) +_verification: + The verification of this test is automated. Do not change the + automatically selected result. + +plugin: user-interact id: dock/hotplug-usb3-remove category_id: dock-hotplug depends: dock/hotplug-usb3-insert @@ -930,35 +1088,117 @@ _verification: The verification of this test is automated. Do not change the automatically selected result. -id: dock/hotplug-monitor-displayport +id: dock/hotplug-playback-type-c-displayport category_id: dock-hotplug depends: dock/hot-plug -estimated_duration: 120.0 -_summary: DisplayPort test on the dock -plugin: manual +requires: + device.category == 'AUDIO' + package.name == 'alsa-base' + package.name == 'gir1.2-gst-plugins-base-0.10' or package.name == 'gir1.2-gst-plugins-base-1.0' + package.name == 'pulseaudio-utils' +_summary: HDMI audio test +plugin: user-interact-verify +flags: also-after-suspend-manual +estimated_duration: 30.0 +command: + audio_settings store --verbose --file=$PLAINBOX_SESSION_SHARE/pulseaudio_settings + audio_settings set --verbose --device=hdmi --volume=50 + gst_pipeline_test -t 2 --device hdmi 'audiotestsrc wave=sine freq=512 ! audioconvert ! audioresample ! autoaudiosink' + EXIT_CODE=$? + audio_settings restore --verbose --file=$PLAINBOX_SESSION_SHARE/pulseaudio_settings + exit $EXIT_CODE _purpose: - This test will check the DisplayPort(s) on the dock after hot plugged in. + Dock USB Type-C Displayport audio interface verification _steps: - Skip this test if the dock does not have a DisplayPort. - 1. Connect a display (if not already connected) to the DisplayPort on the dock - 2. Repeat step 1 for each additional DisplayPorts, if any + 1. Plug an external Displayport device with sound on a USB Type-C port using a "USB Typce-C to Displayport" adapter on Dock (use only one HDMI/DisplayPort/Thunderbolt interface at a time for this test) + 2. Go to the Sound settings and make sure the correct Output is selected + 3. Click the Test button _verification: - Was the desktop displayed correctly on both screens? + Did you hear the sound from the Displayport device? -id: dock/hotplug-monitor-hdmi +id: dock/hotplug-playback-thumderbolt3 category_id: dock-hotplug depends: dock/hot-plug -estimated_duration: 120.0 -_summary: HDMI port test on the dock -plugin: manual +requires: + device.category == 'AUDIO' + package.name == 'alsa-base' + package.name == 'gir1.2-gst-plugins-base-0.10' or package.name == 'gir1.2-gst-plugins-base-1.0' + package.name == 'pulseaudio-utils' +_summary: HDMI audio test +plugin: user-interact-verify +flags: also-after-suspend-manual +estimated_duration: 30.0 +command: + audio_settings store --verbose --file=$PLAINBOX_SESSION_SHARE/pulseaudio_settings + audio_settings set --verbose --device=hdmi --volume=50 + gst_pipeline_test -t 2 --device hdmi 'audiotestsrc wave=sine freq=512 ! audioconvert ! audioresample ! autoaudiosink' + EXIT_CODE=$? + audio_settings restore --verbose --file=$PLAINBOX_SESSION_SHARE/pulseaudio_settings + exit $EXIT_CODE _purpose: - This test will check the HDMI port(s) on the dock after hot plugged in. + Dock Thunderbolt3 audio interface verification _steps: - Skip this test if the dock does not have a HDMI port. - 1. Connect a display (if not already connected) to the HDMI port on the dock - 2. Repeat step 1 for each additional HDMI port, if any + 1. Plug an external Thunderbolt3 device with sound (Use only one HDMI/DisplayPort/Thunderbolt3 interface at a time for this test) + 2. Go to the Sound settings and make sure the correct Output is selected + 3. Click the Test button _verification: - Was the desktop displayed correctly on both screens? + Did you hear the sound from the HDMI device? + +id: dock/hotplug-playback-displayport +category_id: dock-hotplug +depends: dock/hot-plug +requires: + device.category == 'AUDIO' + package.name == 'alsa-base' + package.name == 'gir1.2-gst-plugins-base-0.10' or package.name == 'gir1.2-gst-plugins-base-1.0' + package.name == 'pulseaudio-utils' +_summary: DisplayPort audio test +plugin: user-interact-verify +flags: also-after-suspend-manual +estimated_duration: 30.0 +command: + audio_settings store --verbose --file=$PLAINBOX_SESSION_SHARE/pulseaudio_settings + audio_settings set --verbose --device=hdmi --volume=50 + gst_pipeline_test -t 2 --device hdmi 'audiotestsrc wave=sine freq=512 ! audioconvert ! audioresample ! autoaudiosink' + EXIT_CODE=$? + audio_settings restore --verbose --file=$PLAINBOX_SESSION_SHARE/pulseaudio_settings + exit $EXIT_CODE +_purpose: + Dock DisplayPort audio interface verification +_steps: + 1. Plug an external DisplayPort device with sound to the dock (use only one HDMI/DisplayPort/Thunderbolt interface at a time for this test) + 2. Go to the Sound settings and make sure the correct Output is selected + 3. Click the Test button +_verification: + Did you hear the sound from the DisplayPort device? + +id: dock/hotplug-playback-hdmi +category_id: dock-hotplug +depends: dock/hot-plug +requires: + device.category == 'AUDIO' + package.name == 'alsa-base' + package.name == 'gir1.2-gst-plugins-base-0.10' or package.name == 'gir1.2-gst-plugins-base-1.0' + package.name == 'pulseaudio-utils' +_summary: HDMI audio test +plugin: user-interact-verify +flags: also-after-suspend-manual +estimated_duration: 30.0 +command: + audio_settings store --verbose --file=$PLAINBOX_SESSION_SHARE/pulseaudio_settings + audio_settings set --verbose --device=hdmi --volume=50 + gst_pipeline_test -t 2 --device hdmi 'audiotestsrc wave=sine freq=512 ! audioconvert ! audioresample ! autoaudiosink' + EXIT_CODE=$? + audio_settings restore --verbose --file=$PLAINBOX_SESSION_SHARE/pulseaudio_settings + exit $EXIT_CODE +_purpose: + Dock HDMI audio interface verification +_steps: + 1. Plug an external HDMI device with sound to the dock (use only one HDMI/DisplayPort/Thunderbolt interface at a time for this test) + 2. Go to the Sound settings and make sure the correct Output is selected + 3. Click the Test button +_verification: + Did you hear the sound from the HDMI device? ### Network ### @@ -1234,6 +1474,41 @@ _verification: Was the desktop displayed correctly on the screen connected using a "USB Type-C to VGA" adapter in every mode? +id: dock/monitor-dual-head-after-suspend +category_id: dock-display +depends: suspend/suspend_advanced +estimated_duration: 120.0 +_summary: Dual monitors test while docked after suspend +plugin: manual +_purpose: + This test verifies that dual-monitor output works using the dock after suspend. You will need two external monitors to perform this test. Depends on the specification of the device to use 4K monitor or FHD monitor. +_steps: + Skip this test if your video card or the dock do not support multiple monitors. + 1. If your dock provides more than one monitor outputs, connect two monitors + 2. Open the "Displays" tool (open the dash and search for "Displays") + 3. Configure your output to provide one desktop across all the monitors + 4. Open any application and drag its window from one monitor to the next. +_verification: + Was the stretched desktop displayed correctly across all the screens? + +id: dock/monitor-triple-head-after-suspend +category_id: dock-display +depends: suspend/suspend_advanced +estimated_duration: 120.0 +_summary: Tripple monitors test while docked after suspend +plugin: manual +_purpose: + This test verifies that triple-monitor output works using the dock after suspend. You will need three external monitors to perform this test. Depends on the specification of the device to use 4K monitor or FHD monitor. +_steps: + Skip this test if your video card or the dock do not support multiple monitors. + 1. If your dock provides more than one monitor outputs, connect three monitors + 2. Open the "Displays" tool (open the dash and search for "Displays") + 3. Turn off internal monitor + 4. Configure your output to provide one desktop across all the monitors + 5. Open any application and drag its window from one monitor to the next. +_verification: + Was the stretched desktop displayed correctly across all the screens? + ### Suspend, Undocked, Resume ### id: dock/suspend-undock-resume @@ -1650,52 +1925,84 @@ _verification: Are all kinds of discs properly written? plugin: user-interact -id: dock/thunderbolt-insert +id: dock/thunderbolt3-insert category_id: dock estimated_duration: 40.0 depends: dock/cold-plug +imports: from com.canonical.plainbox import manifest +requires: manifest.has_thunderbolt3 == 'True' +flags: also-after-suspend-manual command: removable_storage_watcher insert --timeout 40 scsi -_summary: Thunderbolt storage insertion detection +_summary: Thunderbolt3 storage insertion detection _purpose: - This test will check if the connection of a Thunderbolt HDD to the dock could be detected + This test will check if the connection of a Thunderbolt3 HDD to the dock could be detected _steps: 1. Click 'Test' to begin the test. This test will timeout and fail if the insertion has not been detected within 40 seconds. - 2. Plug a Thunderbolt HDD into an available Thunderbolt port on the dock; + 2. Plug a Thunderbolt3 HDD into an available Thunderbolt3 port on the dock; if it's not mounted automatically, please click the HDD icon to mount it. _verification: The verification of this test is automated. Do not change the automatically selected result plugin: shell -id: dock/thunderbolt-storage-test +id: dock/thunderbolt3-storage-test category_id: dock estimated_duration: 45.0 user: root -depends: dock/thunderbolt-insert +depends: dock/thunderbolt3-insert +imports: from com.canonical.plainbox import manifest +requires: manifest.has_thunderbolt3 == 'True' +flags: also-after-suspend-manual command: removable_storage_test -s 268400000 scsi -_summary: Thunderbolt storage test +_summary: Thunderbolt3 storage test _description: This is an automated test which performs read/write operations on an attached Thunderbolt HDD plugin: user-interact -id: dock/thunderbolt-remove +id: dock/thunderbolt3-remove category_id: dock estimated_duration: 20.0 -depends: dock/thunderbolt-insert +depends: dock/thunderbolt3-insert +imports: from com.canonical.plainbox import manifest +requires: manifest.has_thunderbolt3 == 'True' +flags: also-after-suspend-manual command: removable_storage_watcher remove scsi -_summary: Thunderbolt storage removal detection +_summary: Thunderbolt3 storage removal detection _purpose: - This test will check the system can detect the removal of a Thunderbolt HDD + This test will check the system can detect the removal of a Thunderbolt3 HDD _steps: 1. Click 'Test' to begin the test. This test will timeout and fail if the removal has not been detected within 20 seconds. - 2. Remove the previously attached Thunderbolt HDD from the Thunderbolt port. + 2. Remove the previously attached Thunderbolt3 HDD from the Thunderbolt3 port. _verification: The verification of this test is automated. Do not change the automatically selected result +plugin: user-interact-verify +category_id: dock +id: dock/thunderbolt3-daisy-chain +flags: also-after-suspend-manual +estimated_duration: 45.0 +imports: from com.canonical.plainbox import manifest +requires: manifest.has_thunderbolt3 == 'True' +command: removable_storage_test -s 268400000 scsi +_summary: Daisy-chain testing for Thunderbolt 3 storage and display device +_description: +_purpose: + This test will check if your system can support daisy-chaining of a storage + and a monitor over Thunderbolt 3 port + _steps: + 1. Connect your Thunderbolt monitor to your systems + 2. Connect and mount your Thunderbolt HDD to another Thunderbolt 3 port of the + monitor (you can do this with HDD first as well) + 3. Click 'Test' to perform the storage test on the Thunderbolt HDD + _verification: + 1. The verification for storage is automated, please select the result combine + with the result for the display. + 2. Was the desktop displayed correctly on the Thunderbolt-connected screen? + ### Hot Plug after suspend ### id: dock/hot-plug-after-suspend @@ -1728,6 +2035,27 @@ _description: Did a notification show and was the connection correctly established? plugin: user-interact +id: dock/hotplug-usb-insert-after-suspend +category_id: dock-hotplug +depends: dock/hot-plug-after-suspend +estimated_duration: 10.0 +_summary: USB drive insertion after resuming +command: removable_storage_watcher insert usb +_purpose: + This test will check that the system correctly detects the insertion of + a USB storage device plugged on the dock after suspend +_steps: + 1. Click "Test" and insert a USB storage device, preferably a HDD, + in one of the dock's port. + Although a USB pen drive may be used it might cause performance + related tests to fail. (Note: this test will time-out after 20 + seconds.) + 2. Do not unplug the device after the test. +_verification: + The verification of this test is automated. Do not change the + automatically selected result. + +plugin: user-interact id: dock/hotplug-usb3-insert-after-suspend category_id: dock-hotplug depends: dock/hot-plug-after-suspend @@ -1748,6 +2076,17 @@ _verification: automatically selected result. plugin: shell +id: dock/hotplug-usb-storage-automated-after-suspend +category_id: dock-hotplug +depends: dock/hotplug-usb-insert-after-suspend +user: root +estimated_duration: 45.0 +_summary: USB drive storage test after resuming +command: removable_storage_test -s 268400000 usb +_description: + This test is automated and executes after the dock/usb-insert test is run. + +plugin: shell id: dock/hotplug-usb3-storage-automated-after-suspend category_id: dock-hotplug depends: dock/hotplug-usb3-insert-after-suspend @@ -1759,6 +2098,23 @@ _description: This test is automated and executes after the dock/usb3_insert test is run. plugin: user-interact +id: dock/hotplug-usb-remove-after-suspend +category_id: dock-hotplug +depends: dock/hotplug-usb-insert-after-suspend +estimated_duration: 10.0 +_summary: USB drive removal after resuming +command: removable_storage_watcher remove usb +_purpose: + This test will check that the system correctly detects the removal of + a USB storage device after suspend +_steps: + 1. Click "Test" and remove the USB device from the dock. + (Note: this test will time-out after 20 seconds.) +_verification: + The verification of this test is automated. Do not change the + automatically selected result. + +plugin: user-interact id: dock/hotplug-usb3-remove-after-suspend category_id: dock-hotplug depends: dock/hotplug-usb3-insert-after-suspend @@ -1775,36 +2131,6 @@ _verification: The verification of this test is automated. Do not change the automatically selected result. -id: dock/hotplug-monitor-displayport-after-suspend -category_id: dock-hotplug -depends: dock/hot-plug-after-suspend -estimated_duration: 120.0 -_summary: DisplayPort test on the dock -plugin: manual -_purpose: - This test will check the DisplayPort(s) on the dock after hot plugged in. -_steps: - Skip this test if the dock does not have a DisplayPort. - 1. Connect a display (if not already connected) to the DisplayPort on the dock - 2. Repeat step 1 for each additional DisplayPorts, if any -_verification: - Was the desktop displayed correctly on both screens? - -id: dock/hotplug-monitor-hdmi-after-suspend -category_id: dock-hotplug -depends: dock/hot-plug-after-suspend -estimated_duration: 120.0 -_summary: HDMI port test on the dock -plugin: manual -_purpose: - This test will check the HDMI port(s) on the dock after hot plugged in. -_steps: - Skip this test if the dock does not have a HDMI port. - 1. Connect a display (if not already connected) to the HDMI port on the dock - 2. Repeat step 1 for each additional HDMI port, if any -_verification: - Was the desktop displayed correctly on both screens? - ### Suspend, Undocked, Docked, Resume ### id: dock/suspend-dock-resume @@ -1838,6 +2164,27 @@ _description: Did a notification show and was the connection correctly established? plugin: user-interact +id: dock/usb-insert-after-suspend-dock-resume +category_id: suspend-dock-resume +depends: dock/suspend-dock-resume +estimated_duration: 10.0 +_summary: USB drive insertion after resuming +command: removable_storage_watcher insert usb +_purpose: + This test will check that the system correctly detects the insertion of + a USB storage device plugged on the dock after suspend +_steps: + 1. Click "Test" and insert a USB storage device, preferably a HDD, + in one of the dock's port. + Although a USB pen drive may be used it might cause performance + related tests to fail. (Note: this test will time-out after 20 + seconds.) + 2. Do not unplug the device after the test. +_verification: + The verification of this test is automated. Do not change the + automatically selected result. + +plugin: user-interact id: dock/usb3-insert-after-suspend-dock-resume category_id: suspend-dock-resume depends: dock/suspend-dock-resume @@ -1858,6 +2205,17 @@ _verification: automatically selected result. plugin: shell +id: dock/usb-storage-automated-after-suspend-dock-resume +category_id: suspend-dock-resume +depends: dock/usb-insert-after-suspend-dock-resume +user: root +estimated_duration: 45.0 +_summary: USB drive storage test after resuming +command: removable_storage_test -s 268400000 usb +_description: + This test is automated and executes after the dock/usb-insert test is run. + +plugin: shell id: dock/usb3-storage-automated-after-suspend-dock-resume category_id: suspend-dock-resume depends: dock/usb3-insert-after-suspend-dock-resume @@ -1869,6 +2227,23 @@ _description: This test is automated and executes after the dock/usb3_insert test is run. plugin: user-interact +id: dock/usb-remove-after-suspend-dock-resume +category_id: suspend-dock-resume +depends: dock/usb-insert-after-suspend-dock-resume +estimated_duration: 10.0 +_summary: USB drive removal after resuming +command: removable_storage_watcher remove usb +_purpose: + This test will check that the system correctly detects the removal of + a USB storage device after suspend +_steps: + 1. Click "Test" and remove the USB device from the dock. + (Note: this test will time-out after 20 seconds.) +_verification: + The verification of this test is automated. Do not change the + automatically selected result. + +plugin: user-interact id: dock/usb3-remove-after-suspend-dock-resume category_id: suspend-dock-resume depends: dock/usb3-insert-after-suspend-dock-resume @@ -1914,3 +2289,13 @@ _steps: 2. Repeat step 1 for each additional HDMI port, if any _verification: Was the desktop displayed correctly on both screens? + +id: dock/mac-address-passthrough +category_id: dock +plugin: shell +user: root +command: mac-passthrough.py +_description: Check dock MAC address pass-through function, please make sure following items before running this test: + 1. Pass-through mac address is flashed in BIOS + 2. MAC address pass-through is enabled in BIOS settings + 3. Dock is connected diff --git a/units/dock/test-plan.pxu b/units/dock/test-plan.pxu new file mode 100644 index 0000000..4c906ac --- /dev/null +++ b/units/dock/test-plan.pxu @@ -0,0 +1,251 @@ +id: dock-cert-full +unit: test plan +_name: Dock Cert tests +_description: + Cert tests for dock devices. +include: +nested_part: + dock-cold-plug-cert + dock-hot-plug + +id: after-suspend-dock-cert-full +unit: test plan +_name: Dock Cert tests after suspend +_description: + Cert tests for dock devices. +include: +nested_part: + after-suspend-dock-cert + after-suspend-dock-hot-plug + suspend-dock-resume-test + suspend-undock-resume-test + +id: dock-cold-plug-cert +unit: test plan +_name: Dock Cold Plug Cert tests +_description: + Cert tests for dock cold plug. +include: + dock/cold-plug certification-status=blocker + dock/power-button certification-status=blocker + dock/monitor-displayport certification-status=blocker + dock/audio-playback-displayport certification-status=blocker + dock/monitor-dvi certification-status=blocker + dock/monitor-hdmi certification-status=blocker + dock/audio-playback-hdmi certification-status=blocker + dock/monitor-vga certification-status=blocker + dock/monitor_type-c_displayport certification-status=blocker + dock/audio-playback-type-c-displayport certification-status=blocker + dock/monitor_type-c_hdmi certification-status=blocker + dock/audio-playback-type-c-hdmi certification-status=blocker + dock/monitor_type-c_vga certification-status=blocker + dock/keys-video-out certification-status=blocker + dock/monitor-dual-head certification-status=blocker + dock/monitor-triple-head + dock/audio-alsa-info-collect + dock/audio-alsa-info-attachment + dock/audio-list-devices + dock/audio-playback-headphones certification-status=blocker + dock/audio-alsa-record-playback-external certification-status=blocker + dock/audio-external-linein + dock/audio-external-lineout + dock/networking-gateway-ping certification-status=blocker + dock/networking-ntp certification-status=blocker + dock/mac-address-passthrough + dock/usb-HID certification-status=blocker + dock/usb-insert certification-status=blocker + dock/usb-storage-automated certification-status=blocker + dock/usb-remove certification-status=blocker + dock/usb3-insert certification-status=blocker + dock/usb3-storage-automated certification-status=blocker + dock/usb3-remove certification-status=blocker + dock/usb-c/c-to-a-adapter/insert + dock/usb-c/c-to-a-adapter/storage-automated + dock/usb-c/c-to-a-adapter/remove + dock/usb-c/insert + dock/usb-c/storage-automated + dock/usb-c/remove + dock/battery-charging certification-status=blocker + dock/monitor-thunderbolt3 + dock/audio-playback-thunderbolt3 + dock/thunderbolt3-insert + dock/thunderbolt3-storage-test + dock/thunderbolt3-remove + dock/thunderbolt3-daisy-chain + dock/network-before-suspend certification-status=blocker + dock/audio-before-suspend certification-status=blocker + +id: dock-hot-plug +unit: test plan +_name: Dock Hot Plug tests +_description: + Tests for dock cold plug. +include: + dock/hot-plug + dock/hotplug-ethernet + dock/hotplug-usb-insert + dock/hotplug-usb-storage-automated + dock/hotplug-usb-remove + dock/hotplug-usb3-insert + dock/hotplug-usb3-storage-automated + dock/hotplug-usb3-remove + dock/hotplug-playback-displayport + dock/hotplug-playback-hdmi + dock/hotplug-playback-type-c-displayport + dock/hotplug-playback-thumderbolt3 + +id: after-suspend-dock-cert +unit: test plan +_name: After suspend Dock Cert tests +_description: + Cert tests for the dock after suspend. +include: + # Do power on/off to clean up the test environment + power-management/poweroff certification-status=blocker + power-management/poweroff-log-attach + suspend/suspend_advanced certification-status=blocker + dock/power-button-after-suspend certification-status=blocker + dock/networking-gateway-ping-after-suspend certification-status=blocker + dock/usb-HID-after-suspend certification-status=blocker + dock/usb-insert-after-suspend certification-status=blocker + dock/usb-storage-automated-after-suspend certification-status=blocker + dock/usb-remove-after-suspend certification-status=blocker + dock/usb3-insert-after-suspend certification-status=blocker + dock/usb3-storage-automated-after-suspend certification-status=blocker + dock/usb3-remove-after-suspend certification-status=blocker + dock/usb-c/c-to-a-adapter/insert-after-suspend + dock/usb-c/c-to-a-adapter/storage-automated-after-suspend + dock/usb-c/c-to-a-adapter/remove-after-suspend + dock/usb-c/insert-after-suspend + dock/usb-c/storage-automated-after-suspend + dock/usb-c/remove-after-suspend + after-suspend-manual-dock/audio-playback-headphones certification-status=blocker + after-suspend-manual-dock/audio-alsa-record-playback-external certification-status=blocker + dock/monitor-displayport-after-suspend certification-status=blocker + after-suspend-manual-dock/audio-playback-displayport certification-status=blocker + dock/monitor-dvi-after-suspend certification-status=blocker + dock/monitor-hdmi-after-suspend certification-status=blocker + after-suspend-manual-dock/audio-playback-hdmi certification-status=blocker + dock/monitor-vga-after-suspend certification-status=blocker + dock/monitor_type-c_displayport-after-suspend + after-suspend-manual-dock/audio-playback-type-c-displayport + dock/monitor_type-c_hdmi-after-suspend + dock/monitor_type-c_vga-after-suspend + after-suspend-manual-dock/thunderbolt3-insert + after-suspend-manual-dock/thunderbolt3-storage-test + after-suspend-manual-dock/thunderbolt3-remove + after-suspend-manual-dock/thunderbolt3-daisy-chain + after-suspend-manual-dock/audio-playback-thunderbolt3 + dock/monitor-dual-head-after-suspend certification-status=blocker + dock/monitor-triple-head-after-suspend + +id: after-suspend-dock-hot-plug +unit: test plan +_name: After suspend Dock Hot-plug tests +_description: + Tests for after suspend dock hot-plug. +include: + dock/hot-plug-after-suspend + dock/hotplug-ethernet-after-suspend + dock/hotplug-usb-insert-after-suspend + dock/hotplug-usb-storage-automated-after-suspend + dock/hotplug-usb-remove-after-suspend + dock/hotplug-usb3-insert-after-suspend + dock/hotplug-usb3-storage-automated-after-suspend + dock/hotplug-usb3-remove-after-suspend + dock/hotplug-monitor-displayport-after-suspend + dock/hotplug-monitor-hdmi-after-suspend + after-suspend-manual-dock/hotplug-playback-displayport + after-suspend-manual-dock/hotplug-playback-hdmi + after-suspend-manual-dock/hotplug-playback-type-c-displayport + after-suspend-manual-dock/hotplug-playback-thumderbolt3 + +id: suspend-dock-resume-test +unit: test plan +_name: Suspend dock resume tests +_description: + Tests for suspend dock then resume it. +include: + dock/suspend-dock-resume + dock/ethernet-after-suspend-dock-resume + dock/usb-insert-after-suspend-dock-resume + dock/usb-storage-automated-after-suspend-dock-resume + dock/usb-remove-after-suspend-dock-resume + dock/usb3-insert-after-suspend-dock-resume + dock/usb3-storage-automated-after-suspend-dock-resume + dock/usb3-remove-after-suspend-dock-resume + dock/monitor-displayport-after-suspend-dock-resume + dock/monitor-hdmi-after-suspend-dock-resume + +id: suspend-undock-resume-test +unit: test plan +_name: After suspend Dock Hot-plug Cert tests +_description: + Test for after susepnd then undock then resume the system. +include: + dock/suspend-undock-resume + +id: dock-cold-plug-cert-blockers +unit: test plan +_name: Dock Cold Plug Cert blocker tests +_description: + Cert blocker tests for dock cold plug. +include: + dock/cold-plug certification-status=blocker + dock/power-button certification-status=blocker + dock/monitor-displayport certification-status=blocker + dock/audio-playback-displayport certification-status=blocker + dock/monitor-dvi certification-status=blocker + dock/monitor-hdmi certification-status=blocker + dock/audio-playback-hdmi certification-status=blocker + dock/monitor-vga certification-status=blocker + dock/monitor_type-c_displayport certification-status=blocker + dock/dock_playback_type-c_displayport certification-status=blocker + dock/monitor_type-c_hdmi certification-status=blocker + dock/dock_playback_type-c_hdmi certification-status=blocker + dock/monitor_type-c_vga certification-status=blocker + dock/keys-video-out certification-status=blocker + dock/monitor-dual-head certification-status=blocker + dock/audio-playback-headphones certification-status=blocker + dock/audio-alsa-record-playback-external certification-status=blocker + dock/networking-gateway-ping certification-status=blocker + dock/networking-ntp certification-status=blocker + dock/usb-HID certification-status=blocker + dock/usb-insert certification-status=blocker + dock/usb-storage-automated certification-status=blocker + dock/usb-remove certification-status=blocker + dock/usb3-insert certification-status=blocker + dock/usb3-storage-automated certification-status=blocker + dock/usb3-remove certification-status=blocker + dock/battery-charging certification-status=blocker + dock/network-before-suspend certification-status=blocker + dock/audio-before-suspend certification-status=blocker + +id: after-suspend-dock-cert-blockers +unit: test plan +_name: After suspend Dock Cert blocker tests +_description: + Cert blocker tests for the dock after suspend. +include: + # Do power on/off to clean up the test environment + power-management/poweroff certification-status=blocker + power-management/poweroff-log-attach + suspend/suspend_advanced certification-status=blocker + dock/power-button-after-suspend certification-status=blocker + dock/networking-gateway-ping-after-suspend certification-status=blocker + after-suspend-manual-dock/audio-playback-headphones certification-status=blocker + after-suspend-manual-dock/audio-alsa-record-playback-external certification-status=blocker + dock/usb-HID-after-suspend certification-status=blocker + dock/usb-insert-after-suspend certification-status=blocker + dock/usb-storage-automated-after-suspend certification-status=blocker + dock/usb-remove-after-suspend certification-status=blocker + dock/usb3-insert-after-suspend certification-status=blocker + dock/usb3-storage-automated-after-suspend certification-status=blocker + dock/usb3-remove-after-suspend certification-status=blocker + dock/monitor-displayport-after-suspend certification-status=blocker + after-suspend-manual-dock/audio-playback-displayport certification-status=blocker + dock/monitor-dvi-after-suspend certification-status=blocker + dock/monitor-hdmi-after-suspend certification-status=blocker + after-suspend-manual-dock/audio-playback-hdmi certification-status=blocker + dock/monitor-vga-after-suspend certification-status=blocker + dock/monitor-dual-head-after-suspend certification-status=blocker diff --git a/units/gpgpu/jobs.pxu b/units/gpgpu/jobs.pxu index 2c823a0..e65d0be 100644 --- a/units/gpgpu/jobs.pxu +++ b/units/gpgpu/jobs.pxu @@ -5,4 +5,4 @@ estimated_duration: 300 requires: package.name == 'cuda' _summary: GPGPU stress testing -command: cd /opt/gpu-burn/ && ./gpu_burn 1800 +command: cd /opt/gpu-burn/ && ./gpu_burn 14400 | grep -v -e '^[[:space:]]*$' -e "errors:" -e "Summary at" diff --git a/units/graphics/jobs.pxu b/units/graphics/jobs.pxu index 13d9afe..fba1f7e 100644 --- a/units/graphics/jobs.pxu +++ b/units/graphics/jobs.pxu @@ -405,7 +405,7 @@ command: totem --fullscreen $PLAINBOX_PROVIDER_DATA/video/Ogg_Theora_Video.ogv 2>/dev/null & set -o pipefail sleep 15 && camera_test still --device=/dev/external_webcam -f $PLAINBOX_SESSION_SHARE/screenshot_fullscreen_video_{index}.jpg -q 2>&1 | ansi_parser - sleep 5 && totem --quit 2>/dev/null + sleep 5 && pkill totem gsettings set org.gnome.totem repeat false _summary: Test FSV screenshot for {vendor} {product} _description: diff --git a/units/i2c/jobs.pxu b/units/i2c/jobs.pxu index db90509..a928917 100644 --- a/units/i2c/jobs.pxu +++ b/units/i2c/jobs.pxu @@ -16,9 +16,9 @@ _steps: detected there's at least one i2c bus. command: if [ -z ${I2C_BUS_NUMBER+x} ]; then - i2c_driver_test bus + i2c_driver_test.py bus else - i2c_driver_test bus -b $I2C_BUS_NUMBER + i2c_driver_test.py bus -b $I2C_BUS_NUMBER fi user: root plugin: shell @@ -37,7 +37,7 @@ _steps: 1. This task is fully automatic, test will pass if there's at least one i2c device detected on any I2C bus. command: - i2c_driver_test device + i2c_driver_test.py device user: root plugin: shell category_id: i2c diff --git a/units/info/jobs.pxu b/units/info/jobs.pxu index e7ac3ea..e1d56ae 100644 --- a/units/info/jobs.pxu +++ b/units/info/jobs.pxu @@ -103,7 +103,7 @@ category_id: com.canonical.plainbox::info user: root command: if [[ -v SNAP ]]; then - checkbox-support-lsusb -f $SNAP/checkbox-runtime/var/lib/usbutils/usb.ids + checkbox-support-lsusb -f $CHECKBOX_RUNTIME/var/lib/usbutils/usb.ids else lsusb -vv | iconv -t 'utf-8' -c fi @@ -218,7 +218,7 @@ plugin: attachment category_id: com.canonical.plainbox::info requires: package.name == 'dkms' -command: dkms_info --format json +command: dkms_info.py --format json _description: Attaches json dumps of installed dkms package information. _summary: Attaches json dumps of installed dkms package information. @@ -361,7 +361,7 @@ user: root estimated_duration: 0.2 _summary: Check existence of recovery partition _description: Check existence of recovery partition -command: recovery_info +command: recovery_info.py plugin: shell category_id: com.canonical.plainbox::info @@ -372,7 +372,7 @@ user: root estimated_duration: 0.2 _summary: Check the recovery type is dell or not _description: Check the recovery type is dell or not -command: recovery_info checktype DELL +command: recovery_info.py checktype DELL plugin: attachment category_id: com.canonical.plainbox::info @@ -389,7 +389,7 @@ _description: Example: image_version: somerville-trusty-amd64-20140620-0 bto_version: A00_dell-bto-trusty-houston-15-A11-iso-20141203-0.iso -command: recovery_info version +command: recovery_info.py version plugin: attachment category_id: com.canonical.plainbox::info @@ -404,7 +404,7 @@ _description: The information include: - fish packages - dell recovery stage 2 boot log -command: recovery_info file bto.xml +command: recovery_info.py file bto.xml plugin: attachment category_id: com.canonical.plainbox::info @@ -463,7 +463,11 @@ command: network_configs id: parts_meta_info_attachment plugin: attachment category_id: com.canonical.plainbox::info -command: cat $SNAP/parts_meta_info $SNAP/checkbox-runtime/parts_meta_info || echo "no parts meta info available" +command: + # some top-level snaps don't bother with parts + cat $SNAP/parts_meta_info || true + # should always have parts info from content snap + cat $CHECKBOX_RUNTIME/parts_meta_info environ: SNAP estimated_duration: 0.02 _summary: Attaches an information about all parts that constituted this snap diff --git a/units/info/packaging.pxu b/units/info/packaging.pxu index 67f18c2..61e26d2 100644 --- a/units/info/packaging.pxu +++ b/units/info/packaging.pxu @@ -3,11 +3,6 @@ unit: packaging meta-data os-id: debian Depends: python3-debian -# The dkms_info script requires python3-guacamole package -unit: packaging meta-data -os-id: debian -Depends: python3-guacamole (>= 0.9) - # This is for lsblk attachment and disk/detect unit: packaging meta-data os-id: debian diff --git a/units/keys/jobs.pxu b/units/keys/jobs.pxu index 4402b4c..7bc69a2 100644 --- a/units/keys/jobs.pxu +++ b/units/keys/jobs.pxu @@ -277,3 +277,16 @@ _description: VERIFICATION: Did the keyboard overhead light key or switch turns on and off the light? +plugin: manual +category_id: com.canonical.plainbox::keys +id: keys/power-button +estimated_duration: 30.0 +flags: also-after-suspend-manual +_purpose: + This test will test the power button +_steps: + 1. press the power button. + 2. press cancel to quit. +_verification: + Did the power management prompt pop up when press power button? + diff --git a/units/keys/test-plan.pxu b/units/keys/test-plan.pxu index 38386ef..87b7fbc 100644 --- a/units/keys/test-plan.pxu +++ b/units/keys/test-plan.pxu @@ -24,6 +24,7 @@ include: keys/wireless certification-status=blocker keys/keyboard-backlight certification-status=blocker keys/microphone-mute certification-status=blocker + keys/power-button certification-status=blocker id: keys-cert-automated unit: test plan diff --git a/units/led/test-plan.pxu b/units/led/test-plan.pxu index 40f5697..33c5642 100644 --- a/units/led/test-plan.pxu +++ b/units/led/test-plan.pxu @@ -22,9 +22,6 @@ _description: - led/wireless is redundant given that we have led/wlan and led/bluetooth already. (Manual) include: - led/battery-charged - led/battery-charging - led/battery-low camera/led_.* certification-status=blocker led/caps-lock certification-status=blocker led/numeric-keypad certification-status=blocker @@ -52,9 +49,6 @@ unit: test plan _name: LED tests (after suspend) _description: LED tests (after suspend) include: - suspend/led_after_suspend/battery-charged - suspend/led_after_suspend/battery-charging - suspend/led_after_suspend/battery-low after-suspend-manual-camera/led_.* certification-status=blocker suspend/led_after_suspend/caps-lock certification-status=blocker suspend/led_after_suspend/numeric-keypad certification-status=blocker diff --git a/units/miscellanea/jobs.pxu b/units/miscellanea/jobs.pxu index 41c9253..ebc7c4d 100644 --- a/units/miscellanea/jobs.pxu +++ b/units/miscellanea/jobs.pxu @@ -150,12 +150,23 @@ command: boot_mode_test secureboot plugin: shell category_id: com.canonical.plainbox::miscellanea estimated_duration: 0.5 +id: miscellanea/reboot_firmware +requires: + cpuinfo.platform in ("i386", "x86_64", "aarch64") +depends: miscellanea/efi_boot_mode +_summary: Test that system supports booting into firmware setup utility +_description: + Test that the system supports rebooting into the firmware setup utility. +command: boot_mode_test reboot_firmware + +plugin: shell +category_id: com.canonical.plainbox::miscellanea +estimated_duration: 0.5 unit: template template-resource: model_assertion template-unit: job requires: - executable.name == 'dumpimage' - executable.name == 'mokutil' + executable.name == 'dumpimage' or executable.name == 'mokutil' id: miscellanea/secure_boot_mode_{gadget} _summary: Test that {gadget} Ubuntu Core system booted with Secure Boot active _description: @@ -190,6 +201,16 @@ command: check-prerelease plugin: shell category_id: com.canonical.plainbox::miscellanea +estimated_duration: 0.5 +id: miscellanea/kernel_taint_test +_summary: Test that kernel is not tainted +_description: + Test to verify that the kernel is not tainted by out-of-tree + drivers, live patches, proprietary modules, etc. +command: kernel_taint_test + +plugin: shell +category_id: com.canonical.plainbox::miscellanea id: miscellanea/bmc_info requires: package.name == 'ipmitool' or executable.name == 'ipmitool' diff --git a/units/miscellanea/test-plan.pxu b/units/miscellanea/test-plan.pxu index 1b119f1..b202727 100644 --- a/units/miscellanea/test-plan.pxu +++ b/units/miscellanea/test-plan.pxu @@ -32,7 +32,6 @@ include: firmware/fwts_uefirtvariable.* certification-status=blocker miscellanea/oops certification-status=blocker miscellanea/oops_results.log - miscellanea/dmitest_client miscellanea/fan_stress_reaction bootstrap_include: fwts @@ -65,8 +64,10 @@ mandatory_include: miscellanea/get_maas_version certification-status=blocker miscellanea/efi_boot_mode certification-status=blocker miscellanea/secure_boot_mode + miscellanea/reboot_firmware miscellanea/efi_pxeboot miscellanea/check_prerelease + miscellanea/kernel_taint_test miscellanea/cpus_are_not_samples miscellanea/ipmi_test certification-status=blocker miscellanea/bmc_info diff --git a/units/monitor/jobs.pxu b/units/monitor/jobs.pxu index df1bb95..c16471d 100644 --- a/units/monitor/jobs.pxu +++ b/units/monitor/jobs.pxu @@ -403,17 +403,16 @@ plugin: shell category_id: com.canonical.plainbox::monitor _summary: Automated HDMI hotplug test _description: - This test will use edid injector on muxpi to check if system detect HDMI hotplug -environ: HDMI_PORT MUXPI_IP -requires: manifest.has_muxpi_hdmi == 'True' -imports: from com.canonical.plainbox import manifest + Use chamleon board to simulate connection/disconnection of an HDMI monitor. +environ: HDMI_PORT CHAMELEON_IP +command: + chameleon_hdmi_hotplug.py $HDMI_PORT $CHAMELEON_IP + +id: monitor/edid-stress-automated +estimated_duration: 30m +plugin: shell +category_id: com.canonical.plainbox::monitor +_summary: Stress DUT by switching through large EDID set +environ: HDMI_PORT CHAMELEON_IP command: - export SSH_OPTS="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" - timeout 60 ssh $SSH_OPTS ubuntu@$MUXPI_IP "stm -hdmi off" || exit 1 - sleep 3 - timeout 60 ssh $SSH_OPTS ubuntu@$MUXPI_IP "stm -hdmi on" || exit 1 - sleep 3 - if [ "$(cat /sys/class/drm/$HDMI_PORT/status)" != "connected" ] ;then exit 1; fi - timeout 60 ssh $SSH_OPTS ubuntu@$MUXPI_IP "stm -hdmi off" || exit 1 - sleep 3 - if [ "$(cat /sys/class/drm/$HDMI_PORT/status)" != "disconnected" ] ;then exit 1; fi + chameleon_edid_stress.py $HDMI_PORT $CHAMELEON_IP diff --git a/units/monitor/test-plan.pxu b/units/monitor/test-plan.pxu index 5eb1f66..aa99f3f 100644 --- a/units/monitor/test-plan.pxu +++ b/units/monitor/test-plan.pxu @@ -36,9 +36,6 @@ include: monitor/1_dvi_.* certification-status=blocker monitor/1_hdmi_.* certification-status=blocker audio/1_playback_hdmi_.* certification-status=blocker - monitor/1_thunderbolt_.* certification-status=blocker - audio/1_playback_thunderbolt_.* certification-status=blocker - thunderbolt/daisy-chain certification-status=blocker monitor/1_thunderbolt3_.* certification-status=non-blocker audio/1_playback_thunderbolt3_.* certification-status=non-blocker thunderbolt3/daisy-chain certification-status=non-blocker @@ -65,9 +62,6 @@ include: after-suspend-manual-monitor/1_dvi_.* certification-status=blocker after-suspend-manual-monitor/1_hdmi_.* certification-status=blocker after-suspend-manual-audio/1_playback_hdmi_.* certification-status=blocker - after-suspend-manual-monitor/1_thunderbolt_.* certification-status=blocker - after-suspend-manual-audio/1_playback_thunderbolt_.* certification-status=blocker - after-suspend-manual-thunderbolt/daisy-chain certification-status=blocker after-suspend-manual-monitor/1_thunderbolt3_.* certification-status=non-blocker after-suspend-manual-audio/1_playback_thunderbolt3_.* certification-status=non-blocker after-suspend-manual-thunderbolt3/daisy-chain certification-status=non-blocker @@ -116,8 +110,6 @@ include: monitor/2_dvi_.* certification-status=blocker monitor/2_hdmi_.* certification-status=blocker audio/2_playback_hdmi_.* certification-status=blocker - monitor/2_thunderbolt_.* certification-status=blocker - audio/2_playback_thunderbolt_.* certification-status=blocker monitor/2_thunderbolt3_.* certification-status=non-blocker audio/2_playback_thunderbolt3_.* certification-status=non-blocker monitor/2_vga_.* certification-status=blocker @@ -153,9 +145,6 @@ include: after-suspend-manual-monitor/2_dvi_.* certification-status=blocker after-suspend-manual-monitor/2_hdmi_.* certification-status=blocker after-suspend-manual-audio/2_playback_hdmi_.* certification-status=blocker - after-suspend-manual-monitor/2_thunderbolt_.* certification-status=blocker - after-suspend-manual-audio/2_playback_thunderbolt_.* certification-status=blocker - after-suspend-manual-thunderbolt/daisy-chain certification-status=blocker after-suspend-manual-monitor/2_thunderbolt3_.* certification-status=non-blocker after-suspend-manual-audio/2_playback_thunderbolt3_.* certification-status=non-blocker after-suspend-manual-thunderbolt3/daisy-chain certification-status=non-blocker @@ -191,9 +180,6 @@ include: monitor/1_dvi_.* certification-status=blocker monitor/1_hdmi_.* certification-status=blocker audio/1_playback_hdmi_.* certification-status=blocker - monitor/1_thunderbolt_.* certification-status=blocker - audio/1_playback_thunderbolt_.* certification-status=blocker - thunderbolt/daisy-chain certification-status=blocker monitor/1_vga_.* certification-status=blocker monitor/1_multi-head_.* certification-status=blocker bootstrap_include: @@ -216,9 +202,6 @@ include: after-suspend-manual-monitor/1_dvi_.* certification-status=blocker after-suspend-manual-monitor/1_hdmi_.* certification-status=blocker after-suspend-manual-audio/1_playback_hdmi_.* certification-status=blocker - after-suspend-manual-monitor/1_thunderbolt_.* certification-status=blocker - after-suspend-manual-audio/1_playback_thunderbolt_.* certification-status=blocker - after-suspend-manual-thunderbolt/daisy-chain certification-status=blocker after-suspend-manual-monitor/1_vga_.* certification-status=blocker after-suspend-manual-monitor/1_multi-head_.* certification-status=blocker bootstrap_include: @@ -241,8 +224,6 @@ include: monitor/2_dvi_.* certification-status=blocker monitor/2_hdmi_.* certification-status=blocker audio/2_playback_hdmi_.* certification-status=blocker - monitor/2_thunderbolt_.* certification-status=blocker - audio/2_playback_thunderbolt_.* certification-status=blocker monitor/2_vga_.* certification-status=blocker monitor/2_multi-head_.* certification-status=blocker bootstrap_include: @@ -265,9 +246,6 @@ include: after-suspend-manual-monitor/2_dvi_.* certification-status=blocker after-suspend-manual-monitor/2_hdmi_.* certification-status=blocker after-suspend-manual-audio/2_playback_hdmi_.* certification-status=blocker - after-suspend-manual-monitor/2_thunderbolt_.* certification-status=blocker - after-suspend-manual-audio/2_playback_thunderbolt_.* certification-status=blocker - after-suspend-manual-thunderbolt/daisy-chain certification-status=blocker after-suspend-manual-monitor/2_vga_.* certification-status=blocker after-suspend-manual-monitor/2_multi-head_.* certification-status=blocker bootstrap_include: @@ -299,6 +277,7 @@ _name: Automated monitor tests _description: Automated monitor tests for Snappy Ubuntu Core devices include: monitor/hdmi-hotplug-automated + monitor/edid-stress-automated id: after-suspend-monitor-full unit: test plan diff --git a/units/stress/boot.pxu b/units/stress/boot.pxu index dcc0bf5..f4085aa 100644 --- a/units/stress/boot.pxu +++ b/units/stress/boot.pxu @@ -48,7 +48,7 @@ plugin: shell command: lspci -i $SNAP/usr/share/misc/pci.ids > $PLAINBOX_SESSION_SHARE/lspci_original || true nmcli -t -f active,BSSID d w l | grep -oP "(?<=^yes:).*" > $PLAINBOX_SESSION_SHARE/wifi_original || true - checkbox-support-lsusb -f $SNAP/checkbox-runtime/var/lib/usbutils/usb.ids -s | sort > $PLAINBOX_SESSION_SHARE/lsusb_original || true + checkbox-support-lsusb -f $CHECKBOX_RUNTIME/var/lib/usbutils/usb.ids -s | sort > $PLAINBOX_SESSION_SHARE/lsusb_original || true environ: LD_LIBRARY_PATH user: root estimated_duration: 1s @@ -101,7 +101,7 @@ environ: LD_LIBRARY_PATH command: lspci -i $SNAP/usr/share/misc/pci.ids > $PLAINBOX_SESSION_SHARE/lspci_test nmcli -t -f active,BSSID d w l | grep -oP "(?<=^yes:).*" > $PLAINBOX_SESSION_SHARE/wifi_test - checkbox-support-lsusb -f $SNAP/checkbox-runtime/var/lib/usbutils/usb.ids -s | sort > $PLAINBOX_SESSION_SHARE/lsusb_test + checkbox-support-lsusb -f $CHECKBOX_RUNTIME/var/lib/usbutils/usb.ids -s | sort > $PLAINBOX_SESSION_SHARE/lsusb_test diff -u $PLAINBOX_SESSION_SHARE/lspci_original $PLAINBOX_SESSION_SHARE/lspci_test if [ $? -ne 0 ] ; then echo "lspci mismatch during cycle 1" @@ -136,7 +136,7 @@ environ: LD_LIBRARY_PATH command: lspci -i $SNAP/usr/share/misc/pci.ids > $PLAINBOX_SESSION_SHARE/lspci_test nmcli -t -f active,BSSID d w l | grep -oP "(?<=^yes:).*" > $PLAINBOX_SESSION_SHARE/wifi_test - checkbox-support-lsusb -f $SNAP/checkbox-runtime/var/lib/usbutils/usb.ids -s | sort > $PLAINBOX_SESSION_SHARE/lsusb_test + checkbox-support-lsusb -f $CHECKBOX_RUNTIME/var/lib/usbutils/usb.ids -s | sort > $PLAINBOX_SESSION_SHARE/lsusb_test diff -u $PLAINBOX_SESSION_SHARE/lspci_original $PLAINBOX_SESSION_SHARE/lspci_test if [ $? -ne 0 ] ; then echo "lspci mismatch during cycle {reboot_id}" @@ -205,7 +205,7 @@ environ: LD_LIBRARY_PATH command: lspci -i $SNAP/usr/share/misc/pci.ids > $PLAINBOX_SESSION_SHARE/lspci_test nmcli -t -f active,BSSID d w l | grep -oP "(?<=^yes:).*" > $PLAINBOX_SESSION_SHARE/wifi_test - checkbox-support-lsusb -f $SNAP/checkbox-runtime/var/lib/usbutils/usb.ids -s | sort > $PLAINBOX_SESSION_SHARE/lsusb_test + checkbox-support-lsusb -f $CHECKBOX_RUNTIME/var/lib/usbutils/usb.ids -s | sort > $PLAINBOX_SESSION_SHARE/lsusb_test diff -u $PLAINBOX_SESSION_SHARE/lspci_original $PLAINBOX_SESSION_SHARE/lspci_test if [ $? -ne 0 ] ; then echo "lspci mismatch during cycle 1" @@ -240,7 +240,7 @@ environ: LD_LIBRARY_PATH command: lspci -i $SNAP/usr/share/misc/pci.ids > $PLAINBOX_SESSION_SHARE/lspci_test nmcli -t -f active,BSSID d w l | grep -oP "(?<=^yes:).*" > $PLAINBOX_SESSION_SHARE/wifi_test - checkbox-support-lsusb -f $SNAP/checkbox-runtime/var/lib/usbutils/usb.ids -s | sort > $PLAINBOX_SESSION_SHARE/lsusb_test + checkbox-support-lsusb -f $CHECKBOX_RUNTIME/var/lib/usbutils/usb.ids -s | sort > $PLAINBOX_SESSION_SHARE/lsusb_test diff -u $PLAINBOX_SESSION_SHARE/lspci_original $PLAINBOX_SESSION_SHARE/lspci_test if [ $? -ne 0 ] ; then echo "lspci mismatch during cycle {reboot_id}" diff --git a/units/stress/jobs.pxu b/units/stress/jobs.pxu index 719e9e1..cf44c6c 100644 --- a/units/stress/jobs.pxu +++ b/units/stress/jobs.pxu @@ -86,13 +86,14 @@ flags: noreturn user: root environ: PM_TEST_DRY_RUN command: + rm -f $PLAINBOX_SESSION_SHARE/__result pm_test reboot --checkbox-respawn-cmd $PLAINBOX_SESSION_SHARE/__respawn_checkbox --fwts --log-level=debug --log-dir=$PLAINBOX_SESSION_SHARE --suspends-before-reboot=30 -r 3 --silent --check-hardware-list _summary: 30 suspend/resume cycles and 1 reboot, 3 times (automated stress test) _description: This is an automated stress test that will run a sequence of '30 suspend/resume cycles and one reboot' 3 times. _siblings: [ { "id": "power-management/suspend_30_cycles_with_coldboots", - "command": "pm_test poweroff --checkbox-respawn-cmd $PLAINBOX_SESSION_SHARE/__respawn_checkbox --fwts --log-level=debug --log-dir=$PLAINBOX_SESSION_SHARE --suspends-before-reboot=30 -r 3 --silent --check-hardware-list", + "command": "rm -f $PLAINBOX_SESSION_SHARE/__result; pm_test poweroff --checkbox-respawn-cmd $PLAINBOX_SESSION_SHARE/__respawn_checkbox --fwts --log-level=debug --log-dir=$PLAINBOX_SESSION_SHARE --suspends-before-reboot=30 -r 3 --silent --check-hardware-list", "_description": "This is an automated stress test that will run a sequence of '30 suspend/resume cycles and one poweroff' 3 times.", "_summary": "30 suspend/resume cycles and 1 poweroff, 3 times (automated stress test)" } @@ -267,7 +268,9 @@ category_id: com.canonical.plainbox::stress id: stress/reboot estimated_duration: 4500.0 requires: executable.name == 'fwts' -command: pm_test --checkbox-respawn-cmd $PLAINBOX_SESSION_SHARE/__respawn_checkbox -r 100 --silent --log-level=notset reboot --log-dir=$PLAINBOX_SESSION_SHARE +command: + rm -f $PLAINBOX_SESSION_SHARE/__result + pm_test --checkbox-respawn-cmd $PLAINBOX_SESSION_SHARE/__respawn_checkbox -r 100 --silent --log-level=notset reboot --log-dir=$PLAINBOX_SESSION_SHARE flags: noreturn user: root environ: PLAINBOX_SESSION_SHARE PM_TEST_DRY_RUN @@ -288,7 +291,9 @@ category_id: com.canonical.plainbox::stress id: stress/reboot_30 requires: executable.name == 'fwts' executable.name == 'x-terminal-emulator' -command: pm_test --checkbox-respawn-cmd $PLAINBOX_SESSION_SHARE/__respawn_checkbox -r 30 --silent --log-level=notset reboot --log-dir=$PLAINBOX_SESSION_SHARE +command: + rm -f $PLAINBOX_SESSION_SHARE/__result + pm_test --checkbox-respawn-cmd $PLAINBOX_SESSION_SHARE/__respawn_checkbox -r 30 --silent --log-level=notset reboot --log-dir=$PLAINBOX_SESSION_SHARE flags: noreturn estimated_duration: 2700 user: root @@ -311,7 +316,9 @@ estimated_duration: 4500.0 requires: executable.name == 'fwts' executable.name == 'x-terminal-emulator' -command: pm_test --checkbox-respawn-cmd $PLAINBOX_SESSION_SHARE/__respawn_checkbox -r 100 --silent --log-level=notset poweroff --log-dir=$PLAINBOX_SESSION_SHARE +command: + rm -f $PLAINBOX_SESSION_SHARE/__result + pm_test --checkbox-respawn-cmd $PLAINBOX_SESSION_SHARE/__respawn_checkbox -r 100 --silent --log-level=notset poweroff --log-dir=$PLAINBOX_SESSION_SHARE flags: noreturn user: root environ: PLAINBOX_SESSION_SHARE PM_TEST_DRY_RUN @@ -333,7 +340,9 @@ id: stress/poweroff_30 requires: executable.name == 'fwts' executable.name == 'x-terminal-emulator' -command: pm_test --checkbox-respawn-cmd $PLAINBOX_SESSION_SHARE/__respawn_checkbox -r 30 --wakeup 150 --silent --log-level=notset poweroff --log-dir=$PLAINBOX_SESSION_SHARE +command: + rm -f $PLAINBOX_SESSION_SHARE/__result + pm_test --checkbox-respawn-cmd $PLAINBOX_SESSION_SHARE/__respawn_checkbox -r 30 --wakeup 150 --silent --log-level=notset poweroff --log-dir=$PLAINBOX_SESSION_SHARE flags: noreturn estimated_duration: 3600 user: root @@ -495,3 +504,41 @@ _description: 2. Press wifi hotkey at a rate of 1 press per second and slowly increase the speed of the tap, until you are tapping as fast as possible VERIFICATION: Verify the system is not frozen and the wifi and bluetooth applets are still visible and functional + +plugin:shell +category_id: com.canonical.plainbox::stress +id: stress/s2idle_pm-graph_30 +estimated_duration: 10m +requires: executable.name == 'sleepgraph' +user: root +_summary: Resume from idle by using Intel pm-graph +command: + sleepgraph -m freeze -rtcwake 10 -sync -gzip -multi 30 0 -skiphtml -o $PLAINBOX_SESSION_SHARE/s2idle_pm-graph/s2idle-{date}-{time} + +plugin: attachment +category_id: com.canonical.plainbox::stress +id: stress/s2idle_pm-graph_30.tar.xz +estimated_duration: 1 +user: root +_summary: Attach pm-graph logs (s2idle) +command: + tar Jcf $PLAINBOX_SESSION_SHARE/s2idle_pm-graph_30.tar.xz $PLAINBOX_SESSION_SHARE/s2idle_pm-graph && cat $PLAINBOX_SESSION_SHARE/s2idle_pm-graph_30.tar.xz + +plugin:shell +category_id: com.canonical.plainbox::stress +id: stress/s3_pm-graph_30 +estimated_duration: 10m +requires: executable.name == 'sleepgraph' +user: root +_summary: Resume from suspend by using Intel pm-graph +command: + sleepgraph -m mem -rtcwake 10 -sync -gzip -multi 30 0 -skiphtml -o $PLAINBOX_SESSION_SHARE/s3_pm-graph/suspend-{date}-{time} + +plugin: attachment +category_id: com.canonical.plainbox::stress +id: stress/s3_pm-graph_30.tar.xz +estimated_duration: 1 +user: root +_summary: Attach pm-graph logs (s3) +command: + tar Jcf $PLAINBOX_SESSION_SHARE/s3_pm-graph_30.tar.xz $PLAINBOX_SESSION_SHARE/s3_pm-graph && cat $PLAINBOX_SESSION_SHARE/s3_pm-graph_30.tar.xz diff --git a/units/stress/test-plan.pxu b/units/stress/test-plan.pxu index 7967dd5..f162517 100644 --- a/units/stress/test-plan.pxu +++ b/units/stress/test-plan.pxu @@ -245,3 +245,13 @@ include: ethernet/iperf3_.* bootstrap_include: device + +id: stress-pm-graph +unit: test plan +_name: pm-graph stress test +_description: pm-graph stress test +include: + stress/s2idle_pm-graph_30 + stress/s2idle_pm-graph_30.tar.xz + stress/s3_pm-graph_30 + stress/s3_pm-graph_30.tar.xz diff --git a/units/submission/jobs.pxu b/units/submission/jobs.pxu index b433fd3..e0829b6 100644 --- a/units/submission/jobs.pxu +++ b/units/submission/jobs.pxu @@ -2,7 +2,7 @@ id: dkms_info_json plugin: attachment category_id: com.canonical.plainbox::info command: - dkms_info --format json | python3 -m plainbox dev parse dkms-info | \ + dkms_info.py --format json | python3 -m plainbox dev parse dkms-info | \ jq '.dkms_info' _description: Attaches json dumps of installed dkms package information. _summary: Attaches json dumps of installed dkms package information. @@ -62,7 +62,7 @@ _description: The information include: - fish packages - dell recovery stage 2 boot log -command: recovery_info file bto.xml | python3 -m plainbox dev parse bto +command: recovery_info.py file bto.xml | python3 -m plainbox dev parse bto id: recovery_info_attachment_json plugin: attachment @@ -79,7 +79,7 @@ _description: Example: image_version: somerville-trusty-amd64-20140620-0 bto_version: A00_dell-bto-trusty-houston-15-A11-iso-20141203-0.iso -command: recovery_info version | python3 -m plainbox dev parse recovery-info +command: recovery_info.py version | python3 -m plainbox dev parse recovery-info id: system_info_json plugin: attachment diff --git a/units/submission/test-plan.pxu b/units/submission/test-plan.pxu index 3bb4cd3..5ae692f 100644 --- a/units/submission/test-plan.pxu +++ b/units/submission/test-plan.pxu @@ -4,7 +4,6 @@ _name: Submission resources _description: Submission resources include: mandatory_include: - com.canonical.plainbox::collect-manifest miscellanea/device_check # Meta-job to include required resources, don't remove. miscellanea/submission-resources diff --git a/units/suspend/test-plan.pxu b/units/suspend/test-plan.pxu index 1d6e12f..e84d6e9 100644 --- a/units/suspend/test-plan.pxu +++ b/units/suspend/test-plan.pxu @@ -35,8 +35,6 @@ include: suspend/cpu_before_suspend certification-status=blocker suspend/memory_before_suspend certification-status=blocker suspend/bluetooth_obex_send_before_suspend certification-status=blocker - suspend/bluetooth_obex_browse_before_suspend certification-status=blocker - suspend/bluetooth_obex_get_before_suspend certification-status=blocker bluetooth4/beacon_eddystone_url_.* certification-status=blocker bootstrap_include: device @@ -52,8 +50,6 @@ include: suspend/memory_after_suspend certification-status=blocker suspend/bluetooth_detect_after_suspend certification-status=blocker suspend/bluetooth_obex_send_after_suspend certification-status=blocker - suspend/bluetooth_obex_browse_after_suspend certification-status=blocker - suspend/bluetooth_obex_get_after_suspend certification-status=blocker after-suspend-manual-bluetooth4/beacon_eddystone_url_.* certification-status=blocker bootstrap_include: device diff --git a/units/thunderbolt/test-plan.pxu b/units/thunderbolt/test-plan.pxu index 75aa0ac..5015cde 100644 --- a/units/thunderbolt/test-plan.pxu +++ b/units/thunderbolt/test-plan.pxu @@ -23,9 +23,6 @@ _name: Thunderbolt tests (Manual) _description: Thunderbolt tests (Manual) include: - thunderbolt/insert certification-status=blocker - thunderbolt/storage-test certification-status=blocker - thunderbolt/remove certification-status=blocker thunderbolt3/insert certification-status=non-blocker thunderbolt3/storage-test certification-status=non-blocker thunderbolt3/remove certification-status=non-blocker @@ -43,9 +40,6 @@ unit: test plan _name: Thunderbolt tests (certification blockers only) _description: Thunderbolt tests (certification blockers only) include: - thunderbolt/insert certification-status=blocker - thunderbolt/storage-test certification-status=blocker - thunderbolt/remove certification-status=blocker id: after-suspend-thunderbolt-cert-manual unit: test plan @@ -53,9 +47,6 @@ _name: Thunderbolt tests (after suspend Manual) _description: Thunderbolt tests (after suspend Manual) include: - after-suspend-manual-thunderbolt/insert certification-status=blocker - after-suspend-manual-thunderbolt/storage-test certification-status=blocker - after-suspend-manual-thunderbolt/remove certification-status=blocker after-suspend-manual-thunderbolt3/insert certification-status=non-blocker after-suspend-manual-thunderbolt3/storage-test certification-status=non-blocker after-suspend-manual-thunderbolt3/remove certification-status=non-blocker @@ -65,8 +56,5 @@ unit: test plan _name: Thunderbolt tests (after suspend - certification blockers only) _description: Thunderbolt tests (after suspend - certification blockers only) include: - after-suspend-manual-thunderbolt/insert certification-status=blocker - after-suspend-manual-thunderbolt/storage-test certification-status=blocker - after-suspend-manual-thunderbolt/remove certification-status=blocker diff --git a/units/tpm/sysfs.pxu b/units/tpm/sysfs.pxu index fdbf390..5db5fe7 100644 --- a/units/tpm/sysfs.pxu +++ b/units/tpm/sysfs.pxu @@ -11,8 +11,8 @@ plugin: resource _summary: Count the number of visible TPM chips in sysfs _description: This job just counts the number of visible TPM chips in as reported by - tpm-sysfs-resource tool. The only resource attribute is 'count' -command: echo "count: $(tpm-sysfs-resource | grep -F 'x-sysfs-device-name' | wc -l)" + tpm_sysfs_resource.py tool. The only resource attribute is 'count' +command: echo "count: $(tpm_sysfs_resource.py | grep -F 'x-sysfs-device-name' | wc -l)" estimated_duration: 2s flags: preserve-locale @@ -24,7 +24,7 @@ _summary: Collect TPM information from sysfs _description: This job collects all the available TPM information from /sys/class/tpm/*/device/*. -command: tpm-sysfs-resource +command: tpm_sysfs_resource.py estimated_duration: 2s flags: preserve-locale # Tie this resource with the has_tpm_chip manifest entry. This way it will @@ -43,7 +43,7 @@ _description: This job collects all the available TPM information from /sys/class/tpm/*/device/*. Distinct files present there are converted to attributes of resource records. -command: tpm-sysfs-resource +command: tpm_sysfs_resource.py estimated_duration: 2s flags: preserve-locale # See note about manifest on the sysfs_tpm job above. @@ -60,7 +60,7 @@ _description: This job collects all the available TPM information from /sys/class/tpm/*/device/*. Distinct files present there are converted to attributes of resource records. -command: tpm-sysfs-resource +command: tpm_sysfs_resource.py estimated_duration: 2s flags: preserve-locale # See note about manifest on the sysfs_tpm job above. @@ -76,6 +76,6 @@ _description: This job collects all the available DMI information from /sys/class/dmi/id/*. The main purpose of including this job is to allow the provider to include vendor-specific quirks by looking at the sysfs_dmi.bios_vendor attribute. -command: dmi-sysfs-resource +command: dmi_sysfs_resource.py estimated_duration: 1s flags: preserve-locale diff --git a/units/usb/manifest.pxu b/units/usb/manifest.pxu index 914e892..fef1a64 100644 --- a/units/usb/manifest.pxu +++ b/units/usb/manifest.pxu @@ -2,3 +2,8 @@ unit: manifest entry id: has_usb_type_c _name: USB Type-C Support value-type: bool + +unit: manifest entry +id: has_usb_storage +_name: USB Storage Device Connected +value-type: bool diff --git a/units/usb/test-plan.pxu b/units/usb/test-plan.pxu index f177798..02dc6e5 100644 --- a/units/usb/test-plan.pxu +++ b/units/usb/test-plan.pxu @@ -221,6 +221,7 @@ unit: test plan _name: Automated USB tests _description: Automated USB tests for Snappy Ubuntu Core devices include: + usb/storage-detect usb/storage-preinserted-.* bootstrap_include: removable_partition @@ -326,6 +327,7 @@ unit: test plan _name: Automated USB tests _description: Automated USB tests for Snappy Ubuntu Core devices include: + after-suspend-usb/storage-detect after-suspend-usb/storage-preinserted-.* bootstrap_include: removable_partition diff --git a/units/usb/usb.pxu b/units/usb/usb.pxu index 670f649..77dd601 100644 --- a/units/usb/usb.pxu +++ b/units/usb/usb.pxu @@ -7,7 +7,7 @@ estimated_duration: 1.0 command: set -o pipefail if [[ -v SNAP ]]; then - checkbox-support-lsusb -f $SNAP/checkbox-runtime/var/lib/usbutils/usb.ids 2>/dev/null | sed 's/.*\(ID .*\)/\1/' | head -n 4 || echo "No USB devices were detected" >&2 + checkbox-support-lsusb -f $CHECKBOX_RUNTIME/var/lib/usbutils/usb.ids 2>/dev/null | sed 's/.*\(ID .*\)/\1/' | head -n 4 || echo "No USB devices were detected" >&2 else lsusb 2>/dev/null | sort || echo "No USB devices were detected" >&2 fi @@ -18,6 +18,8 @@ plugin: user-interact-verify category_id: com.canonical.plainbox::usb id: usb/disk_detect depends: usb/detect +requires: + package.name == 'udisks2' or snap.name == 'udisks2' estimated_duration: 1.0 command: removable_storage_test -l usb _description: @@ -232,7 +234,7 @@ category_id: com.canonical.plainbox::usb id: usb/storage-preinserted user: root estimated_duration: 45.0 -command: removable_storage_test -l usb && removable_storage_test -s 268400000 usb +command: removable_storage_test -l usb && timeout 300 removable_storage_test -s 268400000 usb flags: also-after-suspend preserve-cwd requires: cpuinfo.platform != 's390x' @@ -256,7 +258,7 @@ requires: package.name == 'udisks2' or snap.name == 'udisks2' package.name == 'udisks2' or (snap.name == 'core' and int(snap.revision) >= 1804) estimated_duration: 45.0 -command: removable_storage_test -l usb && removable_storage_test -s 268400000 -m 500000000 usb --driver xhci_hcd +command: removable_storage_test -l usb && timeout 300 removable_storage_test -s 268400000 -m 500000000 usb --driver xhci_hcd _summary: Test USB 3.0 or 3.1 ports _description: @@ -291,6 +293,18 @@ _description: This test will check that your USB 2.0 port transfers data at a minimum expected speed. +id: usb/storage-detect +category_id: com.canonical.plainbox::usb +plugin: shell +_summary: Detect storage partitions on a device on the USB bus +command: + udev_resource -f PARTITION | grep "bus: usb" +estimated_duration: 1.0 +flags: also-after-suspend +imports: from com.canonical.plainbox import manifest +requires: + manifest.has_usb_storage == 'True' + unit: template template-resource: removable_partition template-filter: "usb" in removable_partition.bus @@ -308,6 +322,7 @@ _description: Tests USB 2.0 or 1.1 ports on a system by doing write/read/compare tests on randomly created data. It requires that a USB stick is plugged into an available USB port before running the certification suite. +depends: usb/storage-detect id: usb/hid _summary: USB keyboard works diff --git a/units/wireless/wifi-ap.pxu b/units/wireless/wifi-ap.pxu index 335d505..493a2ca 100644 --- a/units/wireless/wifi-ap.pxu +++ b/units/wireless/wifi-ap.pxu @@ -585,7 +585,6 @@ plugin: user-interact-verify requires: wifi_interface_mode.{interface}_AP == 'supported' snap.name == 'wifi-ap' -depends: wireless/wireless_connection_open_bg_nm_{interface} estimated_duration: 120.0 environ: LD_LIBRARY_PATH OPEN_BG_SSID command: @@ -624,7 +623,6 @@ plugin: shell requires: wifi_interface_mode.{interface}_AP == 'supported' snap.name == 'wifi-ap' -depends: wireless/wireless_connection_open_bg_nm_{interface} estimated_duration: 120.0 environ: LD_LIBRARY_PATH OPEN_BG_SSID WIFI_AP_SETUPTIME command: @@ -673,7 +671,6 @@ plugin: user-interact-verify requires: wifi_interface_mode.{interface}_AP == 'supported' snap.name == 'wifi-ap' -depends: wireless/wireless_connection_open_bg_nm_{interface} estimated_duration: 120.0 environ: LD_LIBRARY_PATH $OPEN_BG_SSID command: @@ -712,7 +709,6 @@ plugin: shell requires: wifi_interface_mode.{interface}_AP == 'supported' snap.name == 'wifi-ap' -depends: wireless/wireless_connection_open_bg_nm_{interface} estimated_duration: 120.0 environ: LD_LIBRARY_PATH OPEN_BG_SSID WIFI_AP_SETUPTIME command: @@ -761,7 +757,6 @@ plugin: user-interact-verify requires: wifi_interface_mode.{interface}_AP == 'supported' snap.name == 'wifi-ap' -depends: wireless/wireless_connection_open_bg_nm_{interface} estimated_duration: 120.0 environ: LD_LIBRARY_PATH OPEN_BG_SSID command: @@ -800,7 +795,6 @@ plugin: shell requires: wifi_interface_mode.{interface}_AP == 'supported' snap.name == 'wifi-ap' -depends: wireless/wireless_connection_open_bg_nm_{interface} estimated_duration: 120.0 environ: LD_LIBRARY_PATH OPEN_BG_SSID WIFI_AP_SETUPTIME command: @@ -849,7 +843,6 @@ plugin: user-interact-verify requires: wifi_interface_mode.{interface}_AP == 'supported' snap.name == 'wifi-ap' -depends: wireless/wireless_connection_open_bg_nm_{interface} estimated_duration: 120.0 environ: LD_LIBRARY_PATH OPEN_BG_SSID command: @@ -888,7 +881,6 @@ plugin: shell requires: wifi_interface_mode.{interface}_AP == 'supported' snap.name == 'wifi-ap' -depends: wireless/wireless_connection_open_bg_nm_{interface} estimated_duration: 120.0 environ: LD_LIBRARY_PATH OPEN_BG_SSID WIFI_AP_SETUPTIME command: diff --git a/units/wwan/jobs.pxu b/units/wwan/jobs.pxu index d18cdc3..b031385 100644 --- a/units/wwan/jobs.pxu +++ b/units/wwan/jobs.pxu @@ -15,7 +15,7 @@ _purpose: plugin: shell user: root command: - COUNT=$(wwan_tests count) + COUNT=$(wwan_tests.py count) if [ $COUNT -eq 0 ]; then exit 1 fi @@ -38,7 +38,7 @@ _description: plugin: shell command: BEGIN_CONNECTION_TEST_TS=`date '+%Y-%m-%d %H:%M:%S'` - wwan_tests 3gpp-connection $WWAN_CONTROL_IF $WWAN_NET_IF $WWAN_APN ${{WWAN_SETUPTIME:-30}} + wwan_tests.py 3gpp-connection $WWAN_CONTROL_IF $WWAN_NET_IF $WWAN_APN ${{WWAN_SETUPTIME:-30}} RETVAL=$? if [ $RETVAL -ne 0 ]; then echo "==== Service units logs ====" @@ -64,7 +64,7 @@ _summary: Check if a SIM card is present in a slot connected to the modem _description: Check if a SIM card is present in a slot connected to the modem plugin: shell -command: wwan_tests sim-present {hw_id} +command: wwan_tests.py sim-present {hw_id} environ: LD_LIBRARY_PATH user: root estimated_duration: 10.0 @@ -90,7 +90,7 @@ _steps: 1. Start the test to automatically retrieve information from the SIM card _verification: Check the output, if as expected then mark the test as passed. -command: wwan_tests sim-info {hw_id} +command: wwan_tests.py sim-info {hw_id} environ: LD_LIBRARY_PATH user: root estimated_duration: 5s diff --git a/units/wwan/resource.pxu b/units/wwan/resource.pxu index a0e82ff..0197f15 100644 --- a/units/wwan/resource.pxu +++ b/units/wwan/resource.pxu @@ -11,7 +11,7 @@ category_id: wwan plugin: resource _summary: Gather device info about WWAN modems _description: Gather device info about WWAN modems -command: wwan_tests --use-cli resources +command: wwan_tests.py --use-cli resources user: root estimated_duration: 3s flags: preserve-locale |