diff options
Diffstat (limited to 'units')
87 files changed, 12547 insertions, 156 deletions
diff --git a/units/audio/jobs.pxu b/units/audio/jobs.pxu new file mode 100644 index 0000000..fa75447 --- /dev/null +++ b/units/audio/jobs.pxu @@ -0,0 +1,580 @@ +plugin: shell +category_id: com.canonical.plainbox::audio +id: audio/list_devices +estimated_duration: 1.0 +requires: + device.category == 'AUDIO' + package.name == 'alsa-base' +command: cat /proc/asound/cards +_description: Test to detect audio devices + +plugin: user-interact-verify +category_id: com.canonical.plainbox::audio +id: audio/playback_auto +estimated_duration: 5.0 +depends: audio/list_devices +requires: + device.category == 'AUDIO' + package.name == 'alsa-base' + package.name == 'gir1.2-gst-plugins-base-0.10' or package.name == 'gir1.2-gst-plugins-base-1.0' + package.name == 'pulseaudio-utils' +command: + audio_settings store --file=$PLAINBOX_SESSION_SHARE/pulseaudio_settings + audio_settings set --device=pci --volume=50 + gst_pipeline_test -t 2 'audiotestsrc wave=sine freq=512 ! audioconvert ! audioresample ! autoaudiosink' + EXIT_CODE=$? + audio_settings restore --file=$PLAINBOX_SESSION_SHARE/pulseaudio_settings + exit $EXIT_CODE +_description: + PURPOSE: + This test will check that internal speakers work correctly + STEPS: + 1. Make sure that no external speakers or headphones are connected + When testing a desktop, you can skip this test if there is no + internal speaker, we will test the external output later + 2. Commence the test to play a brief tone on your audio device + VERIFICATION: + Did you hear a tone? + +unit: template +template-resource: graphics_card +template-filter: graphics_card.prime_gpu_offload == 'Off' +plugin: user-interact-verify +category_id: com.canonical.plainbox::audio +id: audio/{index}_playback_hdmi_{product_slug} +estimated_duration: 30.0 +requires: + display.hdmi == 'supported' + device.category == 'AUDIO' + package.name == 'alsa-base' + package.name == 'gir1.2-gst-plugins-base-0.10' or package.name == 'gir1.2-gst-plugins-base-1.0' + package.name == 'pulseaudio-utils' +command: + audio_settings store --verbose --file=$PLAINBOX_SESSION_SHARE/pulseaudio_settings + audio_settings set --verbose --device=hdmi --volume=50 + gst_pipeline_test -t 2 --device hdmi 'audiotestsrc wave=sine freq=512 ! audioconvert ! audioresample ! autoaudiosink' + EXIT_CODE=$? + audio_settings restore --verbose --file=$PLAINBOX_SESSION_SHARE/pulseaudio_settings + exit $EXIT_CODE +_purpose: + HDMI audio interface verification +_steps: + 1. Plug an external HDMI device with sound (Use only one HDMI/DisplayPort/Thunderbolt interface at a time for this test) + 2. Commence the test +_verification: + Did you hear the sound from the HDMI device? + +unit: template +template-resource: graphics_card +template-filter: graphics_card.prime_gpu_offload == 'Off' +plugin: user-interact-verify +category_id: com.canonical.plainbox::audio +id: audio/{index}_playback_displayport_{product_slug} +estimated_duration: 30.0 +requires: + display.dp == 'supported' + device.category == 'AUDIO' + package.name == 'alsa-base' + package.name == 'gir1.2-gst-plugins-base-0.10' or package.name == 'gir1.2-gst-plugins-base-1.0' + package.name == 'pulseaudio-utils' +command: + audio_settings store --verbose --file=$PLAINBOX_SESSION_SHARE/pulseaudio_settings + audio_settings set --verbose --device=hdmi --volume=50 + gst_pipeline_test -t 2 --device hdmi 'audiotestsrc wave=sine freq=512 ! audioconvert ! audioresample ! autoaudiosink' + EXIT_CODE=$? + audio_settings restore --verbose --file=$PLAINBOX_SESSION_SHARE/pulseaudio_settings + exit $EXIT_CODE +_purpose: + DisplayPort audio interface verification +_steps: + 1. Plug an external DisplayPort device with sound (Use only one HDMI/DisplayPort/Thunderbolt interface at a time for this test) + 2. Commence the test +_verification: + Did you hear the sound from the DisplayPort device? + +unit: template +template-resource: graphics_card +template-filter: graphics_card.prime_gpu_offload == 'Off' +plugin: user-interact-verify +category_id: com.canonical.plainbox::audio +id: audio/{index}_playback_thunderbolt_{product_slug} +imports: from com.canonical.plainbox import manifest +estimated_duration: 30.0 +requires: + manifest.has_thunderbolt == 'True' + device.category == 'AUDIO' + package.name == 'alsa-base' + package.name == 'gir1.2-gst-plugins-base-0.10' or package.name == 'gir1.2-gst-plugins-base-1.0' + package.name == 'pulseaudio-utils' +command: + audio_settings store --verbose --file=$PLAINBOX_SESSION_SHARE/pulseaudio_settings + audio_settings set --verbose --device=hdmi --volume=50 + gst_pipeline_test -t 2 --device hdmi 'audiotestsrc wave=sine freq=512 ! audioconvert ! audioresample ! autoaudiosink' + EXIT_CODE=$? + audio_settings restore --verbose --file=$PLAINBOX_SESSION_SHARE/pulseaudio_settings + exit $EXIT_CODE +_purpose: + Thunderbolt audio interface verification +_steps: + 1. Plug an external Thunderbolt device with sound (Use only one HDMI/DisplayPort/Thunderbolt interface at a time for this test) + 2. Commence the test +_verification: + Did you hear the sound from the Thunderbolt device? + +unit: template +template-resource: graphics_card +template-filter: graphics_card.prime_gpu_offload == 'Off' +plugin: user-interact-verify +category_id: com.canonical.plainbox::audio +id: audio/{index}_playback_thunderbolt3_{product_slug} +imports: from com.canonical.plainbox import manifest +estimated_duration: 30.0 +requires: + manifest.has_thunderbolt3 == 'True' + device.category == 'AUDIO' + package.name == 'alsa-base' + package.name == 'gir1.2-gst-plugins-base-0.10' or package.name == 'gir1.2-gst-plugins-base-1.0' + package.name == 'pulseaudio-utils' +command: + audio_settings store --verbose --file=$PLAINBOX_SESSION_SHARE/pulseaudio_settings + audio_settings set --verbose --device=hdmi --volume=50 + gst_pipeline_test -t 2 --device hdmi 'audiotestsrc wave=sine freq=512 ! audioconvert ! audioresample ! autoaudiosink' + EXIT_CODE=$? + audio_settings restore --verbose --file=$PLAINBOX_SESSION_SHARE/pulseaudio_settings + exit $EXIT_CODE +_purpose: + Thunderbolt audio interface verification +_steps: + 1. Plug an external Thunderbolt device with sound (Use only one HDMI/DisplayPort/Thunderbolt interface at a time for this test) + 2. Commence the test +_verification: + Did you hear the sound from the Thunderbolt device? + +unit: template +template-resource: graphics_card +template-filter: graphics_card.prime_gpu_offload == 'Off' +plugin: user-interact-verify +category_id: com.canonical.plainbox::audio +id: audio/{index}_playback_type-c_displayport_{product_slug} +imports: from com.canonical.plainbox import manifest +estimated_duration: 30.0 +requires: + manifest.has_usb_type_c == 'True' + display.dp == 'supported' + device.category == 'AUDIO' + package.name == 'alsa-base' + package.name == 'gir1.2-gst-plugins-base-0.10' or package.name == 'gir1.2-gst-plugins-base-1.0' + package.name == 'pulseaudio-utils' +command: + audio_settings store --verbose --file=$PLAINBOX_SESSION_SHARE/pulseaudio_settings + audio_settings set --verbose --device=hdmi --volume=50 + gst_pipeline_test -t 2 --device hdmi 'audiotestsrc wave=sine freq=512 ! audioconvert ! audioresample ! autoaudiosink' + EXIT_CODE=$? + audio_settings restore --verbose --file=$PLAINBOX_SESSION_SHARE/pulseaudio_settings + exit $EXIT_CODE +_purpose: + DisplayPort audio via USB Type-C port interface verification +_steps: + 1. Plug an external DisplayPort device with sound on a USB Type-C port using a "USB Typce-C to DisplayPort" adapter (Use only one HDMI/DisplayPort/Thunderbolt interface at a time for this test) + 2. Commence the test +_verification: + Did you hear the sound from the DisplayPort device? + +plugin: user-interact-verify +category_id: com.canonical.plainbox::audio +id: audio/playback_headphones +estimated_duration: 20.0 +depends: audio/list_devices +requires: + device.category == 'AUDIO' + package.name == 'alsa-base' + package.name == 'gir1.2-gst-plugins-base-0.10' or package.name == 'gir1.2-gst-plugins-base-1.0' + package.name == 'pulseaudio-utils' +command: + audio_settings store --file=$PLAINBOX_SESSION_SHARE/pulseaudio_settings + audio_settings set --device=pci --volume=50 + gst_pipeline_test -t 2 'audiotestsrc wave=sine freq=512 ! audioconvert ! audioresample ! autoaudiosink' + EXIT_CODE=$? + audio_settings restore --file=$PLAINBOX_SESSION_SHARE/pulseaudio_settings + exit $EXIT_CODE +_description: + PURPOSE: + This test will check that headphones connector works correctly + STEPS: + 1. Connect a pair of headphones to your audio device + 2. Commence the test to play a sound to your audio device + VERIFICATION: + Did you hear a sound through the headphones and did the sound play without any distortion, clicks or other strange noises from your headphones? + +plugin: user-interact-verify +category_id: com.canonical.plainbox::audio +id: audio/alsa_record_playback_internal +estimated_duration: 20.0 +depends: audio/playback_auto +requires: + device.category == 'AUDIO' + package.name == 'alsa-base' + package.name == 'pulseaudio-utils' + package.name == 'gstreamer1.0-plugins-good' or package.name == 'gstreamer0.10-plugins-good' +command: + audio_settings store --file=$PLAINBOX_SESSION_SHARE/pulseaudio_settings + audio_settings set --device=pci --volume=50 + alsa_record_playback + EXIT_CODE=$? + audio_settings restore --file=$PLAINBOX_SESSION_SHARE/pulseaudio_settings + exit $EXIT_CODE +_description: + PURPOSE: + This test will check that recording sound using the onboard microphone works correctly + STEPS: + 1. Disconnect any external microphones that you have plugged in + 2. Click "Test", then speak into your internal microphone + 3. After a few seconds, your speech will be played back to you. + VERIFICATION: + Did you hear your speech played back? + +plugin: user-interact-verify +category_id: com.canonical.plainbox::audio +id: audio/alsa_record_playback_external +estimated_duration: 20.0 +depends: audio/playback_headphones +requires: + device.category == 'AUDIO' + package.name == 'alsa-base' + package.name == 'pulseaudio-utils' + package.name == 'gstreamer1.0-plugins-good' or package.name == 'gstreamer0.10-plugins-good' +command: + audio_settings store --file=$PLAINBOX_SESSION_SHARE/pulseaudio_settings + audio_settings set --device=pci --volume=50 + alsa_record_playback + EXIT_CODE=$? + audio_settings restore --file=$PLAINBOX_SESSION_SHARE/pulseaudio_settings + exit $EXIT_CODE +_description: + PURPOSE: + This test will check that recording sound using an external microphone works correctly + STEPS: + 1. Connect a microphone to your microphone port + 2. Click "Test", then speak into the external microphone + 3. After a few seconds, your speech will be played back to you + VERIFICATION: + Did you hear your speech played back? + +plugin: user-interact-verify +category_id: com.canonical.plainbox::audio +id: audio/alsa_record_playback_usb +estimated_duration: 120.0 +requires: + device.category == 'AUDIO' + package.name == 'alsa-base' + package.name == 'pulseaudio-utils' + package.name == 'gstreamer1.0-plugins-good' or package.name == 'gstreamer0.10-plugins-good' +command: + audio_settings store --file=$PLAINBOX_SESSION_SHARE/pulseaudio_settings + audio_settings set --device=usb --volume=50 + alsa_record_playback + EXIT_CODE=$? + audio_settings restore --file=$PLAINBOX_SESSION_SHARE/pulseaudio_settings + exit $EXIT_CODE +_description: + PURPOSE: + This test will check that a USB audio device works correctly + STEPS: + 1. Connect a USB audio device to your system + 2. Click "Test", then speak into the microphone + 3. After a few seconds, your speech will be played back to you + VERIFICATION: + Did you hear your speech played back through the USB headphones? + +plugin: shell +category_id: com.canonical.plainbox::audio +id: audio/alsa_record_playback_automated +estimated_duration: 10.0 +requires: + package.name == 'python3-gi' + package.name == 'gir1.2-gstreamer-1.0' + package.name == 'libgstreamer1.0-0' + package.name == 'gstreamer1.0-plugins-good' + package.name == 'gstreamer1.0-pulseaudio' + package.name == 'alsa-base' + device.category == 'AUDIO' +command: audio_test +_description: + Play back a sound on the default output and listen for it on the + default input. + +plugin: shell +category_id: com.canonical.plainbox::audio +id: audio/alsa_info_collect +estimated_duration: 2.0 +command: alsa_info --no-dialog --no-upload --output ${PLAINBOX_SESSION_SHARE}/alsa_info.log +_description: + Collect audio-related system information. This data can be used to + simulate this computer's audio subsystem and perform more detailed tests + under a controlled environment. + +plugin: attachment +category_id: com.canonical.plainbox::audio +id: audio/alsa_info_attachment +depends: audio/alsa_info_collect +estimated_duration: 1.0 +command: [ -e ${PLAINBOX_SESSION_SHARE}/alsa_info.log ] && cat ${PLAINBOX_SESSION_SHARE}/alsa_info.log +_description: + Attaches the audio hardware data collection log to the results. + +plugin: user-interact-verify +category_id: com.canonical.plainbox::audio +id: audio/channels +estimated_duration: 20.0 +command: speaker-test -c 2 -l 1 -t wav +_description: + PURPOSE: + Check that the various audio channels are working properly + STEPS: + 1. Commence the test + VERIFICATION: + You should clearly hear a voice from the different audio channels + +plugin: shell +category_id: com.canonical.plainbox::audio +id: audio/check_volume +estimated_duration: 1.0 +requires: + package.name == 'pulseaudio-utils' + device.category == 'AUDIO' +command: volume_test --minvol 1 --maxvol 100 +_description: + This test will verify that the volume levels are at an acceptable level on + your local system. The test will validate that the volume is greater than + or equal to minvol and less than or equal to maxvol for all sources (inputs) + and sinks (outputs) recognized by PulseAudio. It will also validate that the + active source and sink are not muted. You should not manually adjust the + volume or mute before running this test. + +plugin: manual +category_id: com.canonical.plainbox::audio +id: audio/external-lineout +estimated_duration: 30.0 +_description: + PURPOSE: + Check that external line out connection works correctly + STEPS: + 1. Insert cable to speakers (with built-in amplifiers) on the line out port + 2. Open system sound preferences, 'Output' tab, select 'Line-out' on the connector list. Commence the test + 3. On the system sound preferences, select 'Internal Audio' on the device list and click 'Test Speakers' to check left and right channel + VERIFICATION: + 1. Do you hear a sound in the speakers? The internal speakers should *not* be muted automatically + 2. Do you hear the sound coming out on the corresponding channel? + +plugin: user-interact-verify +category_id: com.canonical.plainbox::audio +id: audio/external-linein +estimated_duration: 120.0 +requires: + device.category == 'AUDIO' + package.name == 'alsa-base' + package.name == 'pulseaudio-utils' + package.name == 'gstreamer1.0-plugins-good' or package.name == 'gstreamer0.10-plugins-good' +command: + audio_settings store --file=$PLAINBOX_SESSION_SHARE/pulseaudio_settings + audio_settings set --device=pci --volume=50 + alsa_record_playback + EXIT_CODE=$? + audio_settings restore --file=$PLAINBOX_SESSION_SHARE/pulseaudio_settings + exit $EXIT_CODE +_description: + PURPOSE: + Check that external line in connection works correctly + STEPS: + 1. Use a cable to connect the line in port to an external line out source. + 2. Open system sound preferences, 'Input' tab, select 'Line-in' on the connector list. Commence the test + 3. After a few seconds, your recording will be played back to you. + VERIFICATION: + Did you hear your recording? + +plugin: user-interact +category_id: com.canonical.plainbox::audio +id: audio/speaker-headphone-plug-detection +estimated_duration: 60.0 +requires: + device.category == 'AUDIO' + package.name == 'pulseaudio-utils' +command: pulse-active-port-change sinks +_description: + PURPOSE: + Check that system detects speakers or headphones being plugged in + STEPS: + 1. Prepare a pair of headphones or speakers with a standard 3.5mm jack + 2. Locate the speaker / headphone jack on the device under test + 3. Run the test (you have 30 seconds from now on) + 4. Plug headphones or speakers into the appropriate jack + 5. Unplug the device for subsequent tests. + VERIFICATION: + Verification is automatic, no action is required. + The test times out after 30 seconds (and fails in that case). + +plugin: user-interact +category_id: com.canonical.plainbox::audio +id: audio/microphone-plug-detection +estimated_duration: 60.0 +requires: + device.category == 'AUDIO' + package.name == 'pulseaudio-utils' +command: pulse-active-port-change sources +_description: + PURPOSE: + Check that system detects a microphone being plugged in + STEPS: + 1. Prepare a microphone with a standard 3.5mm jack + 2. Locate the microphone jack on the device under test. + Keep in mind that it may be shared with the headphone jack. + 3. Run the test (you have 30 seconds from now on) + 4. Plug the microphone into the appropriate jack + 5. Unplug the device for subsequent tests. + VERIFICATION: + Verification is automatic, no action is required. + The test times out after 30 seconds (and fails in that case). + +plugin: user-interact-verify +category_id: com.canonical.plainbox::audio +id: audio/balance_internal_speaker +estimated_duration: 20.0 +depends: audio/playback_auto +requires: + device.category == 'AUDIO' + package.name == 'alsa-base' + package.name == 'gir1.2-gst-plugins-base-0.10' or package.name == 'gir1.2-gst-plugins-base-1.0' + package.name == 'pulseaudio-utils' +command: + audio_settings store --file=$PLAINBOX_SESSION_SHARE/pulseaudio_settings + audio_settings set --device=pci --volume=50 + gst_pipeline_test -t 10 'audiotestsrc wave=sine freq=512 ! audioconvert ! audioresample ! autoaudiosink' + EXIT_CODE=$? + audio_settings restore --file=$PLAINBOX_SESSION_SHARE/pulseaudio_settings + exit $EXIT_CODE +_description: + PURPOSE: + Check that balance control works correctly on internal speakers + STEPS: + 1. Check that moving the balance slider from left to right works smoothly + 2. Commence the test to play an audio tone for 10 seconds. + 3. Move the balance slider from left to right and back. + 4. Check that actual speaker audio balance follows your setting. + VERIFICATION: + Does the slider move smoothly, as well as being followed by the setting by the actual audio output? + +plugin: user-interact-verify +category_id: com.canonical.plainbox::audio +id: audio/balance_headphones +depends: audio/playback_headphones +estimated_duration: 30.0 +requires: + device.category == 'AUDIO' + package.name == 'alsa-base' + package.name == 'gir1.2-gst-plugins-base-0.10' or package.name == 'gir1.2-gst-plugins-base-1.0' + package.name == 'pulseaudio-utils' +command: + audio_settings store --file=$PLAINBOX_SESSION_SHARE/pulseaudio_settings + audio_settings set --device=pci --volume=50 + gst_pipeline_test -t 10 'audiotestsrc wave=sine freq=512 ! audioconvert ! audioresample ! autoaudiosink' + EXIT_CODE=$? + audio_settings restore --file=$PLAINBOX_SESSION_SHARE/pulseaudio_settings + exit $EXIT_CODE +_description: + PURPOSE: + Check that balance control works correctly on external headphone + STEPS: + 1. Check that moving the balance slider from left to right works smoothly + 2. Commence the test to play an audio tone for 10 seconds. + 3. Move the balance slider from left to right and back. + 4. Check that actual headphone audio balance follows your setting. + VERIFICATION: + Does the slider move smoothly, as well as being followed by the setting by the actual audio output? + +plugin: shell +category_id: com.canonical.plainbox::audio +id: audio/list_devices_after_suspend_30_cycles +estimated_duration: 1.0 +depends: power-management/suspend_30_cycles +requires: + device.category == 'AUDIO' + package.name == 'alsa-base' +command: cat /proc/asound/cards +_description: Test to detect audio devices after suspending 30 times. + +plugin: user-interact-verify +category_id: com.canonical.plainbox::audio +id: audio/playback_auto_after_suspend_30_cycles +estimated_duration: 5.0 +depends: audio/list_devices power-management/suspend_30_cycles +requires: + device.category == 'AUDIO' + package.name == 'alsa-base' + package.name == 'gir1.2-gst-plugins-base-0.10' or package.name == 'gir1.2-gst-plugins-base-1.0' + package.name == 'pulseaudio-utils' +command: + audio_settings store --file=$PLAINBOX_SESSION_SHARE/pulseaudio_settings + audio_settings set --device=pci --volume=50 + gst_pipeline_test -t 2 'audiotestsrc wave=sine freq=512 ! audioconvert ! audioresample ! autoaudiosink' + EXIT_CODE=$? + audio_settings restore --file=$PLAINBOX_SESSION_SHARE/pulseaudio_settings + exit $EXIT_CODE +_description: + PURPOSE: + This test will check that internal speakers work correctly after suspending 30 times. + STEPS: + 1. Make sure that no external speakers or headphones are connected + When testing a desktop, you can skip this test if there is no + internal speaker, we will test the external output later + 2. Commence the test to play a brief tone on your audio device + VERIFICATION: + Did you hear a tone? + +plugin: shell +category_id: com.canonical.plainbox::audio +id: audio/alsa_record_playback_automated_after_suspend_30_cycles +estimated_duration: 10.0 +depends: power-management/suspend_30_cycles +requires: + package.name == 'python3-gi' + package.name == 'gir1.2-gstreamer-1.0' + package.name == 'libgstreamer1.0-0' + package.name == 'gstreamer1.0-plugins-good' + package.name == 'gstreamer1.0-pulseaudio' + package.name == 'alsa-base' + device.category == 'AUDIO' +command: audio_test +_description: + Play back a sound on the default output and listen for it on the + default input, after suspending 30 times. + +plugin: shell +category_id: com.canonical.plainbox::audio +id: audio/check_volume_after_suspend_30_cycles +estimated_duration: 1.0 +depends: power-management/suspend_30_cycles +requires: + package.name == 'pulseaudio-utils' + device.category == 'AUDIO' +command: volume_test --minvol 1 --maxvol 100 +_description: + This test will verify that the volume levels are at an acceptable level on + your local system. The test will validate that the volume is greater than + or equal to minvol and less than or equal to maxvol for all sources (inputs) + and sinks (outputs) recognized by PulseAudio. It will also validate that the + active source and sink are not muted. You should not manually adjust the + volume or mute before running this test. + +plugin: shell +category_id: com.canonical.plainbox::audio +id: audio/audio_after_suspend_30_cycles +estimated_duration: 1.0 +depends: power-management/suspend_30_cycles +requires: + device.category == 'AUDIO' + package.name == 'alsa-base' +_description: Record mixer settings after suspending 30 times. +command: + audio_settings store --file=$PLAINBOX_SESSION_SHARE/audio_settings_after_suspend_30_cycles + diff $PLAINBOX_SESSION_SHARE/audio_settings_before_suspend $PLAINBOX_SESSION_SHARE/audio_settings_after_suspend_30_cycles + diff --git a/units/audio/test-plan.pxu b/units/audio/test-plan.pxu new file mode 100644 index 0000000..b425c57 --- /dev/null +++ b/units/audio/test-plan.pxu @@ -0,0 +1,66 @@ +id: audio-cert-full +unit: test plan +_name: Audio tests +_description: + Audio tests + See Monitor / Graphic test plans for hybrid-graphic monitor audio tests +include: + audio/alsa_info_collect + audio/alsa_info_attachment + audio/list_devices + audio/speaker-headphone-plug-detection certification-status=blocker + audio/microphone-plug-detection certification-status=blocker + audio/playback_headphones certification-status=blocker + audio/alsa_record_playback_external certification-status=blocker + audio/playback_auto certification-status=blocker + audio/alsa_record_playback_internal certification-status=blocker + audio/channels + audio/external-linein + audio/external-lineout + +id: after-suspend-audio-cert-full +unit: test plan +_name: Audio tests (after suspend) +_description: Audio tests (after suspend) +include: + suspend/speaker-headphone-plug-detection-after-suspend certification-status=blocker + suspend/microphone-plug-detection-after-suspend certification-status=blocker + suspend/playback_headphones-after-suspend certification-status=blocker + suspend/alsa_record_playback_external-after-suspend certification-status=blocker + +id: audio-cert-automated +unit: test plan +_name: Audio tests (automated) +_description: + Audio tests (automated) +include: + audio/alsa_info_collect + audio/alsa_info_attachment + audio/list_devices + +id: audio-cert-blockers +unit: test plan +_name: Audio tests (certification blockers only) +_description: + Audio tests (certification blockers only) + See Monitor / Graphic test plans for hybrid-graphic monitor audio tests +include: + audio/alsa_info_collect + audio/alsa_info_attachment + audio/list_devices + audio/speaker-headphone-plug-detection certification-status=blocker + audio/microphone-plug-detection certification-status=blocker + audio/playback_headphones certification-status=blocker + audio/alsa_record_playback_external certification-status=blocker + audio/playback_auto certification-status=blocker + audio/alsa_record_playback_internal certification-status=blocker + +id: after-suspend-audio-cert-blockers +unit: test plan +_name: Audio tests (after suspend, certification blockers only) +_description: Audio tests (after suspend, certification blockers only) +include: + suspend/speaker-headphone-plug-detection-after-suspend certification-status=blocker + suspend/microphone-plug-detection-after-suspend certification-status=blocker + suspend/playback_headphones-after-suspend certification-status=blocker + suspend/alsa_record_playback_external-after-suspend certification-status=blocker diff --git a/units/autotesting.pxu b/units/autotesting.pxu deleted file mode 100644 index 3fa75e7..0000000 --- a/units/autotesting.pxu +++ /dev/null @@ -1,104 +0,0 @@ -id: autotesting -_name: Automated Tests -unit: test plan -_description: - This test plan is used for automated testing of the test cases themselves. - The resources jobs are needed for a valid submission, so they are included - explicitly, even if they are usually run implicitly by jobs requiring them. -include: - IEEE_80211 - cpuinfo - dpkg - lsb - package - uname - acpi_sleep_attachment - codecs_attachment - cpuinfo_attachment - dmesg_attachment - dmi_attachment - dmidecode_attachment - efi_attachment - lspci_attachment - lspci_standard_config_attachment - meminfo_attachment - modinfo_attachment - modprobe_attachment - modules_attachment - sysctl_attachment - sysfs_attachment - udev_attachment - recovery_info_attachment - dell_bto_xml_attachment - lsmod_attachment - dkms_info_attachment - audio/alsa_record_playback_automated - bluetooth/detect-output - camera/detect - camera/multiple-resolution-images - cpu/scaling_test - cpu/scaling_test-log-attach - cpu/offlining_test - cpu/topology - disk/read_performance_.* - graphics/xorg-version - graphics/1_compiz_check_.* - graphics/xorg-failsafe - graphics/xorg-process - graphics/1_screenshot_.* - graphics/1_screenshot_fullscreen_video_.* - install/apt-get-gets-updates - mediacard/sd-preinserted - memory/info - memory/check - miscellanea/submission-resources - miscellanea/fwts_test - miscellanea/fwts_results.log - ethernet/detect - networking/http - networking/gateway_ping - power-management/tickless_idle - power-management/rtc - power-management/fwts_wakealarm - power-management/fwts_wakealarm-log-attach - usb/detect - usb/storage-preinserted - wireless/wireless_scanning - wireless/wireless_connection_wpa_bg - wireless/wireless_connection_open_bg - wireless/wireless_connection_wpa_n - wireless/wireless_connection_open_n - wireless/wireless_connection_wpa_ac - wireless/wireless_connection_open_ac - wireless/monitor_wireless_connection_udp - suspend/audio_before_suspend - suspend/bluetooth_obex_send_before_suspend - suspend/bluetooth_obex_browse_before_suspend - suspend/bluetooth_obex_get_before_suspend - suspend/cpu_before_suspend - suspend/network_before_suspend - suspend/memory_before_suspend - suspend/suspend_advanced_auto - suspend/suspend-single-log-check - suspend/audio_after_suspend_auto - suspend/network_resume_time_auto - suspend/wifi_resume_time_auto - suspend/usb_storage_preinserted_after_suspend - suspend/record_playback_after_suspend_auto - suspend/bluetooth_obex_send_after_suspend_auto - suspend/bluetooth_obex_browse_after_suspend_auto - suspend/bluetooth_obex_get_after_suspend_auto - suspend/cpu_after_suspend_auto - suspend/memory_after_suspend_auto - suspend/wireless_connection_after_suspend_wpa_bg_auto - suspend/wireless_connection_after_suspend_open_bg_auto - suspend/wireless_connection_after_suspend_wpa_n_auto - suspend/wireless_connection_after_suspend_open_n_auto - suspend/wireless_connection_after_suspend_wpa_ac_auto - suspend/wireless_connection_after_suspend_open_ac_auto - suspend/gpu_lockup_after_suspend - suspend/screenshot_after_suspend - screenshot_after_suspend.jpg -bootstrap_include: - device - graphics_card diff --git a/units/benchmarks/jobs.pxu b/units/benchmarks/jobs.pxu new file mode 100644 index 0000000..504401b --- /dev/null +++ b/units/benchmarks/jobs.pxu @@ -0,0 +1,336 @@ +unit: template +template-resource: device +template-filter: device.category == 'DISK' +plugin: shell +category_id: com.canonical.plainbox::benchmarks +id: benchmarks/disk/hdparm-read_{name} +estimated_duration: 15.0 +user: root +command: hdparm -t /dev/{name} +_summary: Raw read timing benchmark of {name} using hdparm +_description: This test runs hdparm timing of device reads as a benchmark for {name} + +unit: template +template-resource: device +template-filter: device.category == 'DISK' +plugin: shell +category_id: com.canonical.plainbox::benchmarks +id: benchmarks/disk/hdparm-cache-read_{name} +estimated_duration: 10.0 +user: root +command: hdparm -T /dev/{name} +_summary: Cached read timing benchmark of {name} using hdparm +_description: This test runs hdparm timing of cache reads as a benchmark for {name} + +plugin: shell +category_id: com.canonical.plainbox::benchmarks +id: benchmarks/graphics/gtkperf +depends: graphics/xorg-version +requires: package.name == 'gtkperf' +command: python3 -c 'import re,sys,subprocess; (s, o) = subprocess.getstatusoutput("gtkperf -a"); [sys.exit(1) for i in [s] if s]; m = re.search("Total time:\s+(.*)\n", o); [print(i.group(1)+" Seconds") for i in [m] if m]' +estimated_duration: 30.000 +_description: + Run gtkperf to make sure that GTK based test cases work + +plugin: shell +category_id: com.canonical.plainbox::benchmarks +id: benchmarks/graphics/render-bench +requires: package.name == 'render-bench' +command: /usr/bin/time -f "%e Seconds" render_bench 2>&1 >/dev/null +estimated_duration: 52.000 +_description: Run Render-Bench XRender/Imlib2 benchmark + +plugin: shell +category_id: com.canonical.plainbox::benchmarks +id: benchmarks/graphics/qgears2-Xrender-gearsfancy +requires: package.name == 'phoronix-test-suite' +command: PRESET_OPTIONS="qgears2.render-backend=1; qgears2.test-mode=0" pts_run qgears2 +estimated_duration: 180.000 +_description: Run Qgears2 XRender Extension gearsfancy benchmark + +plugin: shell +category_id: com.canonical.plainbox::benchmarks +id: benchmarks/graphics/qgears2-Xrender-compo +requires: package.name == 'phoronix-test-suite' +command: PRESET_OPTIONS="qgears2.render-backend=1; qgears2.test-mode=2" pts_run qgears2 +estimated_duration: 31.500 +_description: Run Qgears2 XRender Extension image scaling benchmark + +plugin: shell +category_id: com.canonical.plainbox::benchmarks +id: benchmarks/graphics/qgears2-gl-gearsfancy +requires: package.name == 'phoronix-test-suite' +command: PRESET_OPTIONS="qgears2.render-backend=2; qgears2.test-mode=0" pts_run qgears2 +estimated_duration: 52.000 +_description: Run Qgears2 OpenGL gearsfancy benchmark + +plugin: shell +category_id: com.canonical.plainbox::benchmarks +id: benchmarks/graphics/qgears2-gl-compo +requires: package.name == 'phoronix-test-suite' +command: PRESET_OPTIONS="qgears2.render-backend=2; qgears2.test-mode=2" pts_run qgears2 +estimated_duration: 23.000 +_description: Run Qgears2 OpenGL image scaling benchmark + +plugin: shell +category_id: com.canonical.plainbox::benchmarks +id: benchmarks/graphics/glmark2-es2 +requires: + package.name == 'glmark2-es2' + 'arm' in cpuinfo.type +command: + glmark2-es2 2>&1 | sed -e :a -e '$!N;s/\n/ /;ta' | sed -E 's/.*(Score:\s+[0-9]+).*/\1/' +_description: Run GLmark2-ES2 benchmark + +plugin: shell +category_id: com.canonical.plainbox::benchmarks +id: benchmarks/graphics/glmark2 +requires: + package.name == 'glmark2' + cpuinfo.platform in ("i386", "x86_64") +command: + glmark2 2>&1 | sed -e :a -e '$!N;s/\n/ /;ta' | sed -E 's/.*(Score:\s+[0-9]+).*/\1/' +estimated_duration: 306.000 +_description: Run GLmark2 benchmark + +plugin: shell +category_id: com.canonical.plainbox::benchmarks +id: benchmarks/graphics/unigine-sanctuary +requires: package.name == 'phoronix-test-suite' +command: pts_run unigine-sanctuary +_description: Run Unigine Santuary benchmark + +plugin: shell +category_id: com.canonical.plainbox::benchmarks +id: benchmarks/graphics/unigine-tropics +requires: package.name == 'phoronix-test-suite' +command: pts_run unigine-tropics +_description: Run Unigine Tropics benchmark + +plugin: shell +category_id: com.canonical.plainbox::benchmarks +id: benchmarks/graphics/unigine-heaven +requires: package.name == 'phoronix-test-suite' +command: pts_run unigine-heaven +_description: Run Unigine Heaven benchmark + +plugin: shell +category_id: com.canonical.plainbox::benchmarks +id: benchmarks/graphics/lightsmark +requires: package.name == 'phoronix-test-suite' +command: pts_run lightsmark +_description: Run Lightsmark benchmark + +plugin: shell +category_id: com.canonical.plainbox::benchmarks +id: benchmarks/memory/cachebench-read +estimated_duration: 160.0 +requires: package.name == 'phoronix-test-suite' +command: PRESET_OPTIONS="cachebench.test=0" pts_run cachebench +_description: Run Cachebench Read benchmark + +plugin: shell +category_id: com.canonical.plainbox::benchmarks +id: benchmarks/memory/cachebench-write +estimated_duration: 130.0 +requires: package.name == 'phoronix-test-suite' +command: PRESET_OPTIONS="cachebench.test=1" pts_run cachebench +_description: Run Cachebench Write benchmark + +plugin: shell +category_id: com.canonical.plainbox::benchmarks +id: benchmarks/memory/cachebench-read-modify-write +estimated_duration: 140.0 +requires: package.name == 'phoronix-test-suite' +command: PRESET_OPTIONS="cachebench.test=2" pts_run cachebench +_description: Run Cachebench Read / Modify / Write benchmark + +plugin: shell +category_id: com.canonical.plainbox::benchmarks +id: benchmarks/memory/stream-copy +estimated_duration: 6.0 +requires: package.name == 'phoronix-test-suite' +command: PRESET_OPTIONS="stream.run-type=0" pts_run stream +_description: Run Stream Copy benchmark + +plugin: shell +category_id: com.canonical.plainbox::benchmarks +id: benchmarks/memory/stream-scale +estimated_duration: 6.0 +requires: package.name == 'phoronix-test-suite' +command: PRESET_OPTIONS="stream.run-type=1" pts_run stream +_description: Run Stream Scale benchmark + +plugin: shell +category_id: com.canonical.plainbox::benchmarks +id: benchmarks/memory/stream-add +estimated_duration: 5.0 +requires: package.name == 'phoronix-test-suite' +command: PRESET_OPTIONS="stream.run-type=2" pts_run stream +_description: Run Stream Add benchmark + +plugin: shell +category_id: com.canonical.plainbox::benchmarks +id: benchmarks/memory/stream-triad +estimated_duration: 4.0 +requires: package.name == 'phoronix-test-suite' +command: PRESET_OPTIONS="stream.run-type=3" pts_run stream +_description: Run Stream Triad benchmark + +plugin: shell +category_id: com.canonical.plainbox::benchmarks +id: benchmarks/network/network-loopback +requires: package.name == 'phoronix-test-suite' +command: pts_run network-loopback +estimated_duration: 85.0 +_description: Run Network Loopback benchmark + +plugin: shell +category_id: com.canonical.plainbox::benchmarks +id: benchmarks/network/wifi_time_to_reconnect +requires: device.category == 'WIRELESS' +command: wifi_time2reconnect +_description: Check the time needed to reconnect to a WIFI access point + +plugin: shell +category_id: com.canonical.plainbox::benchmarks +id: benchmarks/processor/encode-mp3 +estimated_duration: 140.0 +requires: package.name == 'phoronix-test-suite' +command: pts_run encode-mp3 +_description: Run Encode MP3 benchmark + +plugin: shell +category_id: com.canonical.plainbox::benchmarks +id: benchmarks/processor/x264 +estimated_duration: 400.0 +requires: package.name == 'phoronix-test-suite' +command: pts_run x264 +_description: Run x264 H.264/AVC encoder benchmark + +plugin: shell +category_id: com.canonical.plainbox::benchmarks +id: benchmarks/processor/gnupg +requires: package.name == 'phoronix-test-suite' +command: pts_run gnupg +_description: Run GnuPG benchmark + +plugin: shell +category_id: com.canonical.plainbox::benchmarks +id: benchmarks/processor/compress-pbzip2 +estimated_duration: 90.0 +requires: package.name == 'phoronix-test-suite' +command: pts_run compress-pbzip2 +_description: Run Compress PBZIP2 benchmark + +plugin: shell +category_id: com.canonical.plainbox::benchmarks +id: benchmarks/processor/compress-7zip +estimated_duration: 1400 +requires: package.name == 'phoronix-test-suite' +command: pts_run compress-7zip +_description: Run Compress 7ZIP benchmark + +plugin: shell +category_id: com.canonical.plainbox::benchmarks +id: benchmarks/processor/n-queens +estimated_duration: 330.0 +requires: package.name == 'phoronix-test-suite' +command: pts_run n-queens +_description: Run N-Queens benchmark + +plugin: shell +category_id: com.canonical.plainbox::benchmarks +id: benchmarks/processor/himeno +estimated_duration: 75.0 +requires: package.name == 'phoronix-test-suite' +command: pts_run himeno +_description: Run Himeno benchmark + +plugin: shell +category_id: com.canonical.plainbox::benchmarks +id: benchmarks/system/cpu_on_idle +estimated_duration: 10.0 +requires: package.name == 'sysstat' +command: iostat -x -m 1 10 | python3 -c 'import sys, re; lines="".join(sys.stdin.readlines()); l=[float(n) for n in (re.findall("idle\n.*?(\S+)\n", lines))]; print(sum(l)/len(l),"%")' +_description: CPU utilization on an idle system. + +plugin: shell +category_id: com.canonical.plainbox::benchmarks +id: benchmarks/system/disk_on_idle +estimated_duration: 10.0 +requires: package.name == 'sysstat' +command: iostat -x -m 1 10 | python3 -c 'import sys, re; lines="".join(sys.stdin.readlines()); l=[float(n) for n in (re.findall("util\n.*?(\S+)\n", lines))]; print(sum(l)/len(l),"%")' +_description: Disk utilization on an idle system. + +plugin: shell +category_id: com.canonical.plainbox::benchmarks +id: benchmarks/graphics/gputest_furmark_fullscreen_1920x1080 +requires: + package.name == 'gputest' + cpuinfo.platform == 'x86_64' +command: gputest_benchmark fur --width 1920 --height 1080 -f +estimated_duration: 75.000 +_description: + Run a stress test based on FurMark (OpenGL 2.1 or 3.2) + Fullscreen 1920x1080 no antialiasing + +plugin: shell +category_id: com.canonical.plainbox::benchmarks +id: benchmarks/graphics/gputest_furmark_windowed_1024x640 +requires: + package.name == 'gputest' + cpuinfo.platform == 'x86_64' +command: gputest_benchmark fur +estimated_duration: 75.000 +_description: + Run a stress test based on FurMark (OpenGL 2.1 or 3.2) + Windowed 1024x640 no antialiasing + +plugin: shell +category_id: com.canonical.plainbox::benchmarks +id: benchmarks/graphics/gputest_gimark_fullscreen_1920x1080 +requires: + package.name == 'gputest' + cpuinfo.platform == 'x86_64' +command: gputest_benchmark gi --width 1920 --height 1080 -f +estimated_duration: 75.00 +_description: + Run GiMark, a geometry instancing test (OpenGL 3.3) + Fullscreen 1920x1080 no antialiasing + +plugin: shell +category_id: com.canonical.plainbox::benchmarks +id: benchmarks/graphics/gputest_gimark_windowed_1024x640 +requires: + package.name == 'gputest' + cpuinfo.platform == 'x86_64' +command: gputest_benchmark gi +estimated_duration: 75.500 +_description: + Run GiMark, a geometry instancing test (OpenGL 3.3) + Windowed 1024x640 no antialiasing + +plugin: shell +category_id: com.canonical.plainbox::benchmarks +id: benchmarks/graphics/gputest_tessmark_fullscreen_1920x1080 +requires: + package.name == 'gputest' + cpuinfo.platform == 'x86_64' +command: gputest_benchmark tess --width 1920 --height 1080 -f +estimated_duration: 75.000 +_description: + Run a tessellation test based on TessMark (OpenGL 4.0) + Fullscreen 1920x1080 no antialiasing + +plugin: shell +category_id: com.canonical.plainbox::benchmarks +id: benchmarks/graphics/gputest_tessmark_windowed_1024x640 +requires: + package.name == 'gputest' + cpuinfo.platform == 'x86_64' +command: gputest_benchmark tess +_description: + Run a tessellation test based on TessMark (OpenGL 4.0) + Windowed 1024x640 no antialiasing + diff --git a/units/bluetooth/jobs.pxu b/units/bluetooth/jobs.pxu new file mode 100644 index 0000000..0a246df --- /dev/null +++ b/units/bluetooth/jobs.pxu @@ -0,0 +1,196 @@ +plugin: shell +category_id: com.canonical.plainbox::bluetooth +id: bluetooth/detect-output +estimated_duration: 1.2 +requires: + package.name == 'bluez' or snap.name == 'bluez' + device.category == 'BLUETOOTH' +command: + if rfkill list bluetooth | grep -q 'Hard blocked: yes'; then + echo "rfkill shows BT is hard blocked" + fi + if rfkill list bluetooth | grep -q 'Soft blocked: yes'; then + echo "rfkill shows BT is soft blocked, removing before testing" + rfkill unblock bluetooth + sleep 3 + fi + output=$(hcitool dev | tail -n+2 | awk '{print $2}' | tee $PLAINBOX_SESSION_SHARE/bluetooth_address) + echo "$output" + if [ -z "$output" ]; then + echo "BT hardware not available" + exit 1 + fi +_description: + Automated test to store bluetooth device information in checkbox report + +plugin: manual +category_id: com.canonical.plainbox::bluetooth +id: bluetooth/browse-files +depends: bluetooth/detect-output +estimated_duration: 120.0 +_description: + PURPOSE: + This test will check that bluetooth connection works correctly + STEPS: + 1. Enable bluetooth on any mobile device (PDA, smartphone, etc.) + 2. Click on the bluetooth icon in the menu bar + 3. Select 'Setup new device' + 4. Look for the device in the list and select it + 5. In the device write the PIN code automatically chosen by the wizard + 6. The device should pair with the computer + 7. Right-click on the bluetooth icon and select browse files + 8. Authorize the computer to browse the files in the device if needed + 9. You should be able to browse the files + VERIFICATION: + Did all the steps work? + +plugin: manual +category_id: com.canonical.plainbox::bluetooth +id: bluetooth/file-transfer +depends: bluetooth/detect-output +estimated_duration: 120.0 +_description: + PURPOSE: + This test will check that you can transfer information through a bluetooth connection + STEPS: + 1. Make sure that you're able to browse the files in your mobile device + 2. Copy a file from the computer to the mobile device + 3. Copy a file from the mobile device to the computer + VERIFICATION: + Were all files copied correctly? + +plugin: user-interact-verify +category_id: com.canonical.plainbox::bluetooth +id: bluetooth/audio-a2dp +depends: bluetooth/detect-output +estimated_duration: 120.0 +command: + audio_settings store --file=$PLAINBOX_SESSION_SHARE/pulseaudio_settings + idx=`pactl list cards short | awk '/bluez/{print $1}'` + bt_sink=`pactl list sinks short | awk '/bluez/{print $2}'` + pactl set-card-profile $idx a2dp + pactl set-default-sink $bt_sink + gst_pipeline_test -t 2 'audiotestsrc wave=sine freq=512 ! audioconvert ! audioresample ! autoaudiosink' + EXIT_CODE=$? + audio_settings restore --file=$PLAINBOX_SESSION_SHARE/pulseaudio_settings + exit $EXIT_CODE +_description: + PURPOSE: + This test will check the High Fidelity Playback (A2DP) capability of your Bluetooth device, + to see if you can hear audio from it. + STEPS: + 1. Enable and pair the bluetooth headset + 2. Click "Test" to play a brief tone on your Bluetooth device, if it failed to set the Mode to A2DP, + please select the device and change it manually in the "Sound Settings" + VERIFICATION: + Did you hear the tone? + +plugin: user-interact-verify +category_id: com.canonical.plainbox::bluetooth +id: bluetooth/audio +depends: bluetooth/detect-output +estimated_duration: 120.0 +command: audio_bluetooth_loopback_test +_description: + PURPOSE: + This test will check the Telephony Duplex capability of your bluetooth device, + to see if you can record and hear audio from it simultaneously. + STEPS: + 1. Enable the bluetooth headset + 2. Click on the bluetooth icon in the menu bar + 3. Select "Setup new device" + 4. Look for the device in the list and pair it + 5. Click the sound icon + 6. Click "Sound Settings" + 7. Select device and ensure Mode is set to "Telephony Duplex (HSP/HFP)" + 8. Click "Test" to record and play for five seconds in the bluetooth device + VERIFICATION: + Did you hear the sound you recorded in the bluetooth with a slight intended delay? + +plugin: user-interact-verify +category_id: com.canonical.plainbox::bluetooth +id: bluetooth/HID +depends: bluetooth/detect-output +estimated_duration: 120.0 +command: keyboard_test +_description: + PURPOSE: + This test will check that you can use a BlueTooth HID device + STEPS: + 1. Enable either a BT mouse or keyboard + 2. Click on the bluetooth icon in the menu bar + 3. Select 'Setup new device' + 4. Look for the device in the list and select it + 5. For mice, perform actions such as moving the pointer, right and left button clicks and double clicks + 6. For keyboards, commence the test to launch a small tool. Enter some text into the tool and close it. + VERIFICATION: + Did the device work as expected? + +plugin: shell +category_id: com.canonical.plainbox::bluetooth +id: bluetooth/file_transfer_stress +estimated_duration: 300.0 +requires: + package.name == 'bluez' + package.name == 'obexftp' + device.category == 'BLUETOOTH' +command: + if [ -z "$BTDEVADDR" ] + then + echo "btdevaddr option not set to device address of Bluetooth target in plainbox.conf" + exit 1 + fi + if rfkill list bluetooth | grep -q 'Hard blocked: yes' + then + echo "rfkill list shows BT is hard blocked" + fi + if rfkill list bluetooth | grep -q 'Soft blocked: yes' + then + echo "rfkill list shows BT is soft blocked, removing before testing" + rfkill unblock bluetooth + sleep 3 + fi + bluetooth_transfer_stress $BTDEVADDR +_description: + This is an automated Bluetooth file transfer test. It sends a large file + to the device specified by the BTDEVADDR environment variable, then + retrieves it again using Bluetooth and verifies the checksum to ensure the + transfer didn't corrupt the file. + +plugin: user-interact-verify +category_id: com.canonical.plainbox::bluetooth +id: bluetooth4/HOGP-mouse +depends: bluetooth/detect-output +imports: from com.canonical.plainbox import manifest +requires: + manifest.has_bt_smart == 'True' + package.name == 'bluez' and package.version >= '5.37' +estimated_duration: 30.0 +command: bt_connect --mouse +_purpose: + This test will check that you can use a HID Over GATT Profile (HOGP) with your Bluetooth Smart mouse. +_steps: + 1. Enable a Bluetooth smart mouse, and put it into paring mode. + 2. Commence the test to do the auto-pairing, you will be asked to select targeting mouse from the list. + 3. After it's paired and connected, perform actions such as moving the pointer, right and left button clicks and double clicks. +_verification: + Did the Bluetooth Smart mouse work as expected? + +plugin: user-interact-verify +category_id: com.canonical.plainbox::bluetooth +id: bluetooth4/HOGP-keyboard +depends: bluetooth/detect-output +imports: from com.canonical.plainbox import manifest +requires: + manifest.has_bt_smart == 'True' + package.name == 'bluez' and package.version >= '5.37' +estimated_duration: 30.0 +command: bt_connect --keyboard +_purpose: + This test will check that you can use a HID Over GATT Profile (HOGP) with your Bluetooth Smart keyboard. +_steps: + 1. Enable a Bluetooth Smart keyboard, and put it into paring mode. + 2. Commence the test to do the auto-pairing, you will be asked to select targeting keyboard from the list. + 3. After it's paired and connected, enter some text with your keyboard and close the small input test tool. +_verification: + Did the Bluetooth Smart keyboard work as expected? diff --git a/units/bluetooth/manifest.pxu b/units/bluetooth/manifest.pxu new file mode 100644 index 0000000..ebf5874 --- /dev/null +++ b/units/bluetooth/manifest.pxu @@ -0,0 +1,4 @@ +unit: manifest entry +id: has_bt_smart +_name: Bluetooth Smart (4.0 or later) Support +value-type: bool diff --git a/units/bluetooth/test-plan.pxu b/units/bluetooth/test-plan.pxu new file mode 100644 index 0000000..e6d64e9 --- /dev/null +++ b/units/bluetooth/test-plan.pxu @@ -0,0 +1,27 @@ +id: bluetooth-cert-full +unit: test plan +_name: Bluetooth tests +_description: Bluetooth tests +include: + bluetooth/detect-output + bluetooth/audio-a2dp + bluetooth/browse-files + bluetooth/file-transfer + bluetooth/HID + bluetooth4/HOGP-mouse certification-status=blocker + bluetooth4/HOGP-keyboard certification-status=blocker + +id: bluetooth-cert-automated +unit: test plan +_name: Bluetooth tests (automated) +_description: Bluetooth tests (automated) +include: + bluetooth/detect-output + +id: bluetooth-cert-blockers +unit: test plan +_name: Bluetooth tests (certification blockers only) +_description: Bluetooth tests (certification blockers only) +include: + bluetooth4/HOGP-mouse certification-status=blocker + bluetooth4/HOGP-keyboard certification-status=blocker diff --git a/units/camera/jobs.pxu b/units/camera/jobs.pxu new file mode 100644 index 0000000..4db5222 --- /dev/null +++ b/units/camera/jobs.pxu @@ -0,0 +1,57 @@ +plugin: shell +category_id: com.canonical.plainbox::camera +id: camera/detect +estimated_duration: 1.2 +requires: + device.category == 'CAPTURE' +command: if [ "`lsb_release -c | awk {'print $2'}`" == "precise" ]; then camera_test_legacy detect; else camera_test detect; fi +_description: This Automated test attempts to detect a camera. + +plugin: user-interact-verify +category_id: com.canonical.plainbox::camera +id: camera/display +estimated_duration: 120.0 +depends: camera/detect +requires: + device.category == 'CAPTURE' +command: if [ "`lsb_release -c | awk {'print $2'}`" == "precise" ]; then camera_test_legacy display; else camera_test display; fi +_description: + PURPOSE: + This test will check that the built-in camera works + STEPS: + 1. Click on Test to display a video capture from the camera for ten seconds. + VERIFICATION: + Did you see the video capture? + +plugin: user-interact-verify +category_id: com.canonical.plainbox::camera +id: camera/still +estimated_duration: 120.0 +depends: camera/detect +requires: + package.name == 'gir1.2-gst-plugins-base-0.10' or package.name == 'gir1.2-gst-plugins-base-1.0' + package.name == 'eog' + package.name == 'fswebcam' or package.name == 'gir1.2-gst-plugins-base-0.10' or package.name == 'gir1.2-gst-plugins-base-1.0' + device.category == 'CAPTURE' +command: if [ "`lsb_release -c | awk {'print $2'}`" == "precise" ]; then camera_test_legacy still; else camera_test still; fi +_description: + PURPOSE: + This test will check that the built-in camera works + STEPS: + 1. Click on Test to display a still image from the camera for ten seconds. + VERIFICATION: + Did you see the image? + +plugin: shell +category_id: com.canonical.plainbox::camera +id: camera/multiple-resolution-images +estimated_duration: 1.2 +depends: camera/detect +requires: + package.name == 'fswebcam' or package.name == 'gir1.2-gst-plugins-base-0.10' or package.name == 'gir1.2-gst-plugins-base-1.0' + device.category == 'CAPTURE' +command: if [ "`lsb_release -c | awk {'print $2'}`" == "precise" ]; then camera_test_legacy resolutions; else camera_test resolutions; fi +_description: + Takes multiple pictures based on the resolutions supported by the camera and + validates their size and that they are of a valid format. + diff --git a/units/camera/packaging.pxu b/units/camera/packaging.pxu new file mode 100644 index 0000000..bb12d64 --- /dev/null +++ b/units/camera/packaging.pxu @@ -0,0 +1,3 @@ +unit: packaging meta-data +os-id: debian +Depends: gir1.2-cheese-3.0 diff --git a/units/camera/test-plan.pxu b/units/camera/test-plan.pxu new file mode 100644 index 0000000..2cf5550 --- /dev/null +++ b/units/camera/test-plan.pxu @@ -0,0 +1,43 @@ +id: camera-cert-full +unit: test plan +_name: Camera tests +_description: Camera tests +include: + camera/detect certification-status=blocker + camera/still certification-status=blocker + camera/display certification-status=blocker + camera/multiple-resolution-images certification-status=blocker + +id: after-suspend-camera-cert-full +unit: test plan +_name: Camera tests (after suspend) +_description: Camera tests (after suspend) +include: + camera/display_after_suspend certification-status=blocker + camera/still_after_suspend certification-status=blocker + +id: camera-cert-automated +unit: test plan +_name: Camera tests (automated) +_description: Camera tests (automated) +include: + camera/detect certification-status=blocker + camera/multiple-resolution-images certification-status=blocker + +id: camera-cert-blockers +unit: test plan +_name: Camera tests (certification blockers only) +_description: Camera tests (certification blockers only) +include: + camera/detect certification-status=blocker + camera/still certification-status=blocker + camera/display certification-status=blocker + camera/multiple-resolution-images certification-status=blocker + +id: after-suspend-camera-cert-blockers +unit: test plan +_name: Camera tests (after suspend, certification blockers only) +_description: Camera tests (after suspend, certification blockers only) +include: + camera/display_after_suspend certification-status=blocker + camera/still_after_suspend certification-status=blocker diff --git a/units/codecs/jobs.pxu b/units/codecs/jobs.pxu new file mode 100644 index 0000000..7ad2d4b --- /dev/null +++ b/units/codecs/jobs.pxu @@ -0,0 +1,34 @@ +plugin: user-interact-verify +category_id: com.canonical.plainbox::codecs +id: codecs/ogg-vorbis +requires: + package.name == 'gstreamer0.10-plugins-base' or package.name == 'gstreamer1.0-plugins-base' + package.name == 'totem' + package.name == 'ubuntu-sounds' +command: totem /usr/share/sounds/ubuntu/stereo/system-ready.ogg +_description: + PURPOSE: + This test will verify your system's ability to play Ogg Vorbis audio files. + STEPS: + 1. Click Test to play an Ogg Vorbis file (.ogg) + 2. Please close the player to proceed. + VERIFICATION: + Did the sample play correctly? + +plugin: user-interact-verify +category_id: com.canonical.plainbox::codecs +id: codecs/wav +requires: + package.name == 'gstreamer0.10-plugins-good' or package.name == 'gstreamer1.0-plugins-good' + package.name == 'totem' + package.name == 'alsa-utils' +command: totem /usr/share/sounds/alsa/Noise.wav +_description: + PURPOSE: + This test will verify your system's ability to play Wave Audio files. + STEPS: + 1. Select Test to play a Wave Audio format file (.wav) + 2. Please close the player to proceed. + VERIFICATION: + Did the sample play correctly? + diff --git a/units/cpu/jobs.pxu b/units/cpu/jobs.pxu new file mode 100644 index 0000000..bdea273 --- /dev/null +++ b/units/cpu/jobs.pxu @@ -0,0 +1,165 @@ +plugin: shell +category_id: com.canonical.plainbox::cpu +id: cpu/scaling_test +estimated_duration: 150.0 +requires: + package.name == 'fwts' or executable.name == 'fwts' +user: root +environ: PLAINBOX_SESSION_SHARE LD_LIBRARY_PATH SNAP +command: + if [[ -v SNAP ]]; then + export LD_LIBRARY_PATH=$SNAP/usr/lib/fwts:$LD_LIBRARY_PATH + fi + checkbox-support-fwts_test -t cpufreq -l ${PLAINBOX_SESSION_SHARE}/scaling_test.log +_summary: + Test the CPU scaling capabilities +_description: + Use Firmware Test Suite (fwts cpufreq) to test the scaling capabilities of the + CPU. + +plugin: attachment +category_id: com.canonical.plainbox::cpu +id: cpu/scaling_test-log-attach +estimated_duration: 1.0 +depends: cpu/scaling_test +command: [[ -e ${PLAINBOX_SESSION_SHARE}/scaling_test.log ]] && cat ${PLAINBOX_SESSION_SHARE}/scaling_test.log +_summary: + Attach CPU scaling capabilities log +_description: + Attaches the log generated by cpu/scaling_test to the results submission. + +plugin: shell +category_id: com.canonical.plainbox::cpu +id: cpu/maxfreq_test +estimated_duration: 0.6 +requires: + package.name == 'fwts' or executable.name == 'fwts' + cpuinfo.platform in ("i386", "x86_64") +user: root +environ: LD_LIBRARY_PATH SNAP +command: + if [[ -v SNAP ]]; then + export LD_LIBRARY_PATH=$SNAP/usr/lib/fwts:$LD_LIBRARY_PATH + fi + checkbox-support-fwts_test -t maxfreq -l $PLAINBOX_SESSION_SHARE/maxfreq_test.log +_summary: + Test that the CPU can run at its max frequency +_description: + Use the Firmware Test Suite (fwts cpufreq) to ensure that the CPU can run at + its maximum frequency. + +plugin: attachment +category_id: com.canonical.plainbox::cpu +id: cpu/maxfreq_test-log-attach +estimated_duration: 1.0 +depends: cpu/maxfreq_test +command: [ -e $PLAINBOX_SESSION_SHARE/maxfreq_test.log ] && cat $PLAINBOX_SESSION_SHARE/maxfreq_test.log +_summary: + Attach CPU max frequency log +_description: + Attaches the log generated by cpu/maxfreq_test to the results submission. + +plugin: shell +category_id: com.canonical.plainbox::cpu +id: cpu/clocktest +estimated_duration: 300.0 +command: clocktest +_summary: + Tests the CPU for clock jitter +_description: + Runs a test for clock jitter on SMP machines. + +plugin: shell +category_id: com.canonical.plainbox::cpu +id: cpu/offlining_test +estimated_duration: 128.0 +user: root +command: cpu_offlining +_summary: + Test offlining of each CPU core +_description: + Attempts to offline each core in a multicore system. + +plugin: shell +category_id: com.canonical.plainbox::cpu +id: cpu/topology +estimated_duration: 1.0 +requires: int(cpuinfo.count) > 1 and (cpuinfo.platform == 'i386' or cpuinfo.platform == 'x86_64' or cpuinfo.platform == 's390x') +command: cpu_topology +_summary: + Check CPU topology for accuracy between proc and sysfs +_description: + Parses information about CPU topology provided by proc and sysfs and checks + that they are consistent. + +plugin: shell +category_id: com.canonical.plainbox::cpu +id: cpu/frequency_governors +estimated_duration: 22.0 +user: root +command: nice -n -20 frequency_governors_test --debug +_description: + This test checks that CPU frequency governors are obeyed when set. + +plugin: shell +category_id: com.canonical.plainbox::cpu +id: cpu/arm_vfp_support +estimated_duration: 1.0 +requires: + cpuinfo.platform in ("armv7l", "aarch64") +user: root +command: grep VFP /var/log/syslog +_summary: + Validate that the Vector Floating Point Unit is running on ARM device +_description: + Validate that the Vector Floating Point Unit is running on ARM device. + +plugin:shell +id: cpu/cstates +estimated_duration: 10.0 +requires: + package.name == 'fwts' or executable.name == 'fwts' +user: root +_summary: + Run C-States tests +_description: + Uses the Firmware Test Suite (fwts) to test the power saving states of the CPU. +environ: PLAINBOX_SESSION_SHARE LD_LIBRARY_PATH SNAP +command: + if [[ -v SNAP ]]; then + export LD_LIBRARY_PATH=$SNAP/usr/lib/fwts:$LD_LIBRARY_PATH + fi + checkbox-support-fwts_test -l $PLAINBOX_SESSION_SHARE/fwts_cstates_results.log -t cstates + +plugin: attachment +category_id: com.canonical.plainbox::cpu +estimated_duration: 0.5 +id: cpu/cstates_results.log +command: + [ -e ${PLAINBOX_SESSION_SHARE}/fwts_cstates_results.log ] && cat ${PLAINBOX_SESSION_SHARE}/fwts_cstates_results.log +_summary: + Attach C-States test log +_description: + Attaches the FWTS desktop diagnosis results log to the submission. + +plugin: shell +category_id: com.canonical.plainbox::cpu +id: cpu/cpuinfo_before_suspend +estimated_duration: 1.2 +_summary: Verify that all the CPUs are online before suspending +command: cpuinfo_resource > $PLAINBOX_SESSION_SHARE/cpuinfo_before_suspend +_siblings: [ + { "id": "after-suspend-cpu/cpuinfo_after_suspend", + "_summary": "Verify that all the CPUs are online after suspending", + "command": "cpuinfo_resource | diff $PLAINBOX_SESSION_SHARE/cpuinfo_before_suspend -", + "depends": "com.canonical.certification::suspend/suspend_advanced_auto"} + ] + +plugin: shell +category_id: com.canonical.plainbox::cpu +estimated_duration: 0.5 +id: cpu/purley_id +command: + egrep -q avx512 /proc/cpuinfo && echo "This is a Purley CPU" || echo "CPU is NOT Purley" +_description: Identifies Purley Platform CPUs + diff --git a/units/cpu/test-plan.pxu b/units/cpu/test-plan.pxu new file mode 100644 index 0000000..2678c31 --- /dev/null +++ b/units/cpu/test-plan.pxu @@ -0,0 +1,36 @@ +id: cpu-cert-full +unit: test plan +_name: CPU tests +_description: CPU tests +include: +nested_part: + cpu-cert-automated + +id: cpu-cert-automated +unit: test plan +_name: CPU tests (automated) +_description: CPU tests (automated) +include: + cpu/cstates certification-status=blocker + cpu/cstates_results.log + cpu/scaling_test certification-status=blocker + cpu/scaling_test-log-attach + cpu/maxfreq_test certification-status=blocker + cpu/maxfreq_test-log-attach + cpu/offlining_test certification-status=blocker + cpu/topology certification-status=blocker + cpu/clocktest + +id: cpu-cert-blockers +unit: test plan +_name: CPU tests (certification blockers only) +_description: CPU tests (certification blockers only) +include: + cpu/cstates certification-status=blocker + cpu/cstates_results.log + cpu/scaling_test certification-status=blocker + cpu/scaling_test-log-attach + cpu/maxfreq_test certification-status=blocker + cpu/maxfreq_test-log-attach + cpu/offlining_test certification-status=blocker + cpu/topology certification-status=blocker diff --git a/units/disk/jobs.pxu b/units/disk/jobs.pxu new file mode 100644 index 0000000..072ac52 --- /dev/null +++ b/units/disk/jobs.pxu @@ -0,0 +1,150 @@ +plugin: shell +category_id: com.canonical.plainbox::disk +id: disk/detect +requires: + package.name == 'util-linux' or executable.name == 'lsblk' +_summary: Gathers information about each disk detected +_description: + Uses lsblk to gather information about each disk detected on the system + under test. +command: disk_info +estimated_duration: 0.25 + +unit: template +template-resource: device +template-filter: device.category == 'DISK' and device.name != '' +template-unit: job +plugin: shell +category_id: com.canonical.plainbox::disk +id: disk/stats_{name} +flags: deprecated +user: root +estimated_duration: 10.0 +command: disk_stats_test {name} +_summary: Disk statistics for {product} +_description: + This test checks disk stats, generates some activity and rechecks stats to + verify they've changed. It also verifies that disks appear in the various + files they're supposed to. + . + This test will inspect the following disk: + . + product name: {product} + sysfs path: {path} + device node path: /dev/{name} + +unit: template +template-resource: device +template-filter: device.category == 'DISK' +plugin: shell +category_id: com.canonical.plainbox::disk +id: disk/smart_{name} +estimated_duration: 180.0 +requires: + package.name == 'smartmontools' or executable.name == 'smartctl' + block_device.smart == 'True' and block_device.name == '{name}' +_summary: + Test SMART capabilities for {product} +_description: + This tests the SMART capabilities for {product} (Note that this test may not work against hardware RAID) +user: root +command: disk_smart -b /dev/{name} -s 130 -t 530 + +unit: template +template-resource: device +template-filter: device.category == 'DISK' +plugin: shell +category_id: com.canonical.plainbox::disk +id: disk/read_performance_{name} +estimated_duration: 65.0 +requires: +_summary: Disk performance test for {product} +_description: Verify that disk storage performs at or above baseline performance +user: root +command: disk_read_performance_test {name} + +unit: template +template-resource: device +template-filter: device.category == 'DISK' +plugin: shell +category_id: com.canonical.plainbox::disk +id: disk/storage_device_{name} +estimated_duration: 375.0 +user: root +requires: +_summary: Disk I/O stress test for {product} +_description: Take the path of the storage device and test is it a block device +command: storage_test {name} + +unit: template +template-resource: device +template-filter: device.category == 'DISK' +plugin: shell +category_id: com.canonical.plainbox::disk +id: disk/disk_stress_ng_{name} +estimated_duration: 4560.0 +user: root +requires: + package.name == 'stress-ng' or executable.name == 'stress-ng' + package.name == 'uuid-runtime' or executable.name == 'uuidgen' +_summary: Disk stress_ng test for {product} +_description: Disk stress_ng test for {product} +command: disk_stress_ng {name} --base-time 240 --really-run + +unit: template +template-resource: device +template-filter: device.category == 'DISK' +plugin: shell +category_id: com.canonical.plainbox::disk +id: disk/disk_cpu_load_{name} +estimated_duration: 15 +user: root +requires: + block_device.state != 'removable' and block_device.name == '{name}' + package.name == 'bc' or executable.name == 'bc' +_summary: Check of CPU load imposed by {product} +_description: Check to ensure CPU load of {product} is not too high +command: disk_cpu_load {name} + +plugin: shell +category_id: com.canonical.plainbox::disk +id: disk/spindown +estimated_duration: 875.0 +requires: + device.category == 'DISK' + package.name == 'smartmontools' +user: root +command: spindown +_description: + Some new hard drives include a feature that parks the drive heads after a short period of inactivity. This is a power-saving feature, but it can have a bad interaction with the operating system that results in the drive constantly parked then activated. This produces excess wear on the drive, potentially leading to early failures. + +plugin: user-interact +category_id: com.canonical.plainbox::disk +id: disk/hdd-parking +estimated_duration: 60.0 +requires: + device.category == 'DISK' + package.name == 'hdapsd' +depends: input/accelerometer +user: root +command: hdd_parking +_description: + PURPOSE: + This test checks that a systems drive protection mechanism works properly. + STEPS: + 1. Click on Test + 2. Move the system under test around, ensuring it is raised and lowered at some point. + VERIFICATION: + The verification of this test is automated. Do not change the + automatically selected result. + +unit: template +template-resource: device +template-filter: device.driver == 'nvme' and device.category == 'OTHER' +plugin: shell +category_id: com.canonical.plainbox::disk +id: disk/apste_support_on_{name} +estimated_duration: 1.0 +user: root +command: nvme get-feature -f 0x0c -H /dev/{name} | grep '(APSTE).*Enabled' && test -e /sys/class/nvme/{name}/power/pm_qos_latency_tolerance_us +_summary: Check support for Autonomous Power State Transition on {name} diff --git a/units/disk/packaging.pxu b/units/disk/packaging.pxu new file mode 100644 index 0000000..515d8e1 --- /dev/null +++ b/units/disk/packaging.pxu @@ -0,0 +1,9 @@ +# This is for disk/apste_support_on_* +unit: packaging meta-data +os-id: debian +Depends: nvme-cli + +# The stress/disk_stress_ng_test job requires uuid-runtime package +unit: packaging meta-data +os-id: debian +Depends: uuid-runtime diff --git a/units/disk/test-plan.pxu b/units/disk/test-plan.pxu new file mode 100644 index 0000000..08974cb --- /dev/null +++ b/units/disk/test-plan.pxu @@ -0,0 +1,29 @@ +id: disk-cert-manual +unit: test plan +_name: Disk tests (manual) +_description: Disk tests (manual) +include: + disk/detect certification-status=blocker + disk/hdd-parking certification-status=non-blocker + +id: disk-cert-automated +unit: test plan +_name: Disk tests (automated) +_description: Disk tests (automated) +include: + disk/detect certification-status=blocker + disk/stats_.* + disk/read_performance_.* certification-status=blocker + disk/storage_device_.* certification-status=blocker + benchmarks/disk/hdparm-read_.* + benchmarks/disk/hdparm-cache-read_.* + disk/apste_support_on_.* + +id: disk-cert-blockers +unit: test plan +_name: Disk tests (certification blockers only) +_description: Disk tests (certification blockers only) +include: + disk/detect certification-status=blocker + disk/read_performance_.* certification-status=blocker + disk/storage_device_.* certification-status=blocker diff --git a/units/dock/category.pxu b/units/dock/category.pxu new file mode 100644 index 0000000..5483c39 --- /dev/null +++ b/units/dock/category.pxu @@ -0,0 +1,31 @@ +unit: category +id: dock +_name: Docking station tests + +unit: category +id: dock-display +_name: Docking station display tests + +unit: category +id: dock-audio +_name: Docking station audio tests + +unit: category +id: dock-network +_name: Docking station network tests + +unit: category +id: dock-usb +_name: Docking station usb tests + +unit: category +id: dock-hotplug +_name: Docking station hotplug tests + +unit: category +id: suspend-undock-resume +_name: Docking station functional tests after suspend, undocked, resume. + +unit: category +id: suspend-dock-resume +_name: Docking station functional tests after suspend, undocked, docked, resume. diff --git a/units/dock.pxu b/units/dock/jobs.pxu index 7e66a60..34decb6 100644 --- a/units/dock.pxu +++ b/units/dock/jobs.pxu @@ -1,35 +1,3 @@ -unit: category -id: dock -_name: Docking station tests - -unit: category -id: dock-display -_name: Docking station display tests - -unit: category -id: dock-audio -_name: Docking station audio tests - -unit: category -id: dock-network -_name: Docking station network tests - -unit: category -id: dock-usb -_name: Docking station usb tests - -unit: category -id: dock-hotplug -_name: Docking station hotplug tests - -unit: category -id: suspend-undock-resume -_name: Docking station functional tests after suspend, undocked, resume. - -unit: category -id: suspend-dock-resume -_name: Docking station functional tests after suspend, undocked, docked, resume. - id: dock/docking category_id: dock plugin: manual diff --git a/units/esata/jobs.pxu b/units/esata/jobs.pxu new file mode 100644 index 0000000..65778eb --- /dev/null +++ b/units/esata/jobs.pxu @@ -0,0 +1,41 @@ +plugin: user-interact +category_id: com.canonical.plainbox::esata +id: esata/insert +command: removable_storage_watcher insert ata_serial_esata +_description: + PURPOSE: + This test will check the system can detect the insertion of an eSATA HDD + STEPS: + 1. Click 'Test' to begin the test. This test will + timeout and fail if the insertion has not been detected within 20 seconds. + 2. Plug an eSATA HDD into an available eSATA port. + VERIFICATION: + The verification of this test is automated. Do not change the automatically + selected result + +plugin: shell +category_id: com.canonical.plainbox::esata +id: esata/storage-test +user: root +depends: esata/insert +command: removable_storage_test -s 268400000 ata_serial_esata +_description: + This is an automated test which performs read/write operations on an attached + eSATA HDD + +plugin: user-interact +category_id: com.canonical.plainbox::esata +id: esata/remove +depends: esata/insert +command: removable_storage_watcher remove ata_serial_esata +_description: + PURPOSE: + This test will check the system can detect the removal of an eSATA HDD + STEPS: + 1. Click 'Test' to begin the test. This test will timeout and fail if + the removal has not been detected within 20 seconds. + 2. Remove the previously attached eSATA HDD from the eSATA port. + VERIFICATION: + The verification of this test is automated. Do not change the automatically + selected result + diff --git a/units/esata/test-plan.pxu b/units/esata/test-plan.pxu new file mode 100644 index 0000000..b7e59cd --- /dev/null +++ b/units/esata/test-plan.pxu @@ -0,0 +1,17 @@ +id: esata-cert-full +unit: test plan +_name: eSATA tests +_description: eSATA tests +include: + esata/insert certification-status=blocker + esata/storage-test certification-status=blocker + esata/remove certification-status=blocker + +id: esata-cert-blockers +unit: test plan +_name: eSATA tests (certification blockers only) +_description: eSATA tests (certification blockers only) +include: + esata/insert certification-status=blocker + esata/storage-test certification-status=blocker + esata/remove certification-status=blocker diff --git a/units/ethernet/jobs.pxu b/units/ethernet/jobs.pxu new file mode 100644 index 0000000..90e0edf --- /dev/null +++ b/units/ethernet/jobs.pxu @@ -0,0 +1,131 @@ +plugin: shell +category_id: com.canonical.plainbox::ethernet +id: ethernet/detect +flags: also-after-suspend +command: network_device_info +estimated_duration: 2.0 +_summary: + Report info about available network devices +_description: + Test to detect and return information about available network controllers on + the system under test. + +plugin: shell +category_id: com.canonical.plainbox::ethernet +id: ethernet/info_automated +requires: + executable.name == 'ip' + device.category == 'NETWORK' +command: udev_resource -f NETWORK | awk "/interface: / { print \$2 }" | xargs -n 1 network_info +estimated_duration: 30.0 +_summary: + Gather info on current state of network devices +_description: + Gathers some info on the current state of your network devices. If no devices + are found, the test will exit with an error. + +plugin: user-interact-verify +category_id: com.canonical.plainbox::ethernet +id: ethernet/wired_connection +command: network_check +estimated_duration: 1.2 +_description: + PURPOSE: + This test will check your wired connection + STEPS: + 1. Click on the Network icon in the top panel + 2. Select a network below the "Wired network" section + 3. Click "Test" to verify that it's possible to establish a HTTP connection + VERIFICATION: + Did a notification show and was the connection correctly established? + +unit: template +template-resource: device +template-filter: device.category == 'NETWORK' +plugin: shell +category_id: com.canonical.plainbox::ethernet +id: ethernet/multi_nic_device{__index__}_{interface} +_summary: Multi-NIC Iperf stress testing for NIC {interface} +estimated_duration: 7400.0 +requires: + package.name == 'iperf' or executable.name == 'iperf' + package.name == 'ethtool' or executable.name == 'ethtool' + package.name == 'nmap' or executable.name == 'nmap' +user: root +environ: TEST_TARGET_IPERF +command: network test -i {interface} -t iperf --scan-timeout 3600 --fail-threshold 80 --runtime 900 --num_runs 4 +_description: + This test uses iperf to ensure network devices pass data at an acceptable + minimum percentage of advertized speed. + +unit: template +template-resource: device +template-filter: device.category == 'NETWORK' and device.interface != 'UNKNOWN' +plugin: shell +category_id: com.canonical.plainbox::ethernet +id: ethernet/multi_iperf3_nic_device{__index__}_{interface} +_summary: Multi-NIC Iperf3 stress testing for NIC {interface} +estimated_duration: 7400.0 +requires: + package.name == 'iperf3' or executable.name == 'iperf3' + package.name == 'ethtool' or executable.name == 'ethtool' + package.name == 'nmap' or executable.name == 'nmap' +user: root +environ: TEST_TARGET_IPERF +command: network test -i {interface} -t iperf --iperf3 --scan-timeout 3600 --fail-threshold 80 --cpu-load-fail-threshold 90 --runtime 900 --num_runs 4 +_description: + This test uses iperf3 to ensure network devices pass data at an acceptable + minimum percentage of advertized speed. + +unit: template +template-resource: device +template-filter: device.category == 'NETWORK' and device.interface != 'UNKNOWN' +plugin: shell +category_id: com.canonical.plainbox::ethernet +id: ethernet/ethertool_check_device{__index__}_{interface} +_summary: ethtool check for NIC {interface} +estimated_duration: 330.0 +requires: + package.name == 'ethtool' or executable.name == 'ethtool' +user: root +command: ethtool {interface} +_description: + This test executes ethtool requests against ethernet device {__index__} ({interface}). + +unit: template +template-resource: device +template-filter: device.category == 'NETWORK' and device.interface != 'UNKNOWN' +plugin: user-interact-verify +category_id: com.canonical.plainbox::ethernet +id: ethernet/maximum_bandwidth_device{__index__}_{interface} +_summary: Maximum bandwidth test of device {__index__} ({interface}) +estimated_duration: 330.0 +requires: + package.name == 'zenity' + package.name == 'iperf' +user: root +environ: TEST_TARGET_IPERF +command: network test -i {interface} -t iperf 2>&1 | cat - <(echo; echo "Verify the result and click OK to decide on the outcome") | zenity --text-info --title 'ethernet max bw {interface}' +_purpose: + User verification of whether the observed transfer throughput is acceptable + for the type and maximum speed of each ethernet interface. +_steps: + 1. Click "Test". + 2. Read the network test summary and confirm that the throughput is acceptable. + 3. If needed, click "Test" again to repeat the transfer test. +_verification: + Was the reported throughput acceptable for the type and maximum speed of this interface? + +unit: template +template-resource: device +template-filter: device.category == 'NETWORK' and device.interface != 'UNKNOWN' +plugin: shell +category_id: com.canonical.plainbox::ethernet +id: ethernet/stress_performance_device{__index__}_{interface} +_summary: Stress and performance test of ethernet device {__index__} ({interface}) +estimated_duration: 330.0 +requires: + package.name == 'ethtool' +command: network test -i {interface} -t stress +_description: + Automated test that tests performance of ethernet device {__index__} ({interface}). diff --git a/units/ethernet/packaging.pxu b/units/ethernet/packaging.pxu new file mode 100644 index 0000000..7d4a812 --- /dev/null +++ b/units/ethernet/packaging.pxu @@ -0,0 +1,3 @@ +unit: packaging meta-data +os-id: debian +Depends: kmod diff --git a/units/expresscard/jobs.pxu b/units/expresscard/jobs.pxu new file mode 100644 index 0000000..a1245b9 --- /dev/null +++ b/units/expresscard/jobs.pxu @@ -0,0 +1,12 @@ +plugin: manual +category_id: com.canonical.plainbox::expresscard +id: expresscard/verification +_description: + PURPOSE: + This will verify that an ExpressCard slot can detect inserted devices. + STEPS: + Skip this test if you do not have an ExpressCard slot. + 1. Plug an ExpressCard device into the ExpressCard slot + VERIFICATION: + Was the device correctly detected? + diff --git a/units/fingerprint/jobs.pxu b/units/fingerprint/jobs.pxu new file mode 100644 index 0000000..c7951b0 --- /dev/null +++ b/units/fingerprint/jobs.pxu @@ -0,0 +1,32 @@ +plugin: manual +category_id: com.canonical.plainbox::fingerprint +id: fingerprint/login +_description: + PURPOSE: + This test will verify that a fingerprint reader will work properly for logging into your system. This test case assumes that there's a testing account from which test cases are run and a personal account that the tester uses to verify the fingerprint reader + STEPS: + 1. Click on the User indicator on the left side of the panel (your user name). + 2. Select "Switch User Account" + 3. On the LightDM screen select your username. + 4. Use the fingerprint reader to login. + 5. Click on the user switcher applet. + 6. Select the testing account to continue running tests. + VERIFICATION: + Did the authentication procedure work correctly? + +plugin: manual +category_id: com.canonical.plainbox::fingerprint +id: fingerprint/unlock +_description: + PURPOSE: + This test will verify that a fingerprint reader can be used to unlock a locked system. + STEPS: + 1. Click on the Session indicator (Cog icon on the Left side of the panel) . + 2. Select 'Lock screen'. + 3. Press any key or move the mouse. + 4. A window should appear that provides the ability to unlock either typing your password or using fingerprint authentication. + 5. Use the fingerprint reader to unlock. + 6. Your screen should be unlocked. + VERIFICATION: + Did the authentication procedure work correctly? + diff --git a/units/firewire/jobs.pxu b/units/firewire/jobs.pxu new file mode 100644 index 0000000..9a0df3f --- /dev/null +++ b/units/firewire/jobs.pxu @@ -0,0 +1,44 @@ +plugin: user-interact +category_id: com.canonical.plainbox::firewire +id: firewire/insert +require: device.category == 'FIREWIRE' +command: removable_storage_watcher insert firewire +_description: + PURPOSE: + This test will check the system can detect the insertion of a FireWire HDD + STEPS: + 1. Click 'Test' to begin the test. This test will + timeout and fail if the insertion has not been detected within 20 seconds. + 2. Plug a FireWire HDD into an available FireWire port. + VERIFICATION: + The verification of this test is automated. Do not change the automatically + selected result + +plugin: shell +category_id: com.canonical.plainbox::firewire +id: firewire/storage-test +user: root +require: device.category == 'FIREWIRE' +depends: firewire/insert +command: removable_storage_test -s 268400000 firewire +_description: + This is an automated test which performs read/write operations on an attached + FireWire HDD + +plugin: user-interact +category_id: com.canonical.plainbox::firewire +id: firewire/remove +depends: firewire/insert +require: device.category == 'FIREWIRE' +command: removable_storage_watcher remove firewire +_description: + PURPOSE: + This test will check the system can detect the removal of a FireWire HDD + STEPS: + 1. Click 'Test' to begin the test. This test will timeout and fail if + the removal has not been detected within 20 seconds. + 2. Remove the previously attached FireWire HDD from the FireWire port. + VERIFICATION: + The verification of this test is automated. Do not change the automatically + selected result + diff --git a/units/firewire/test-plan.pxu b/units/firewire/test-plan.pxu new file mode 100644 index 0000000..ecb6084 --- /dev/null +++ b/units/firewire/test-plan.pxu @@ -0,0 +1,17 @@ +id: firewire-cert-full +unit: test plan +_name: Firewire tests +_description: Firewire tests +include: + firewire/insert certification-status=blocker + firewire/storage-test certification-status=blocker + firewire/remove certification-status=blocker + +id: firewire-cert-blockers +unit: test plan +_name: Firewire tests (certification blockers only) +_description: Firewire tests (certification blockers only) +include: + firewire/insert certification-status=blocker + firewire/storage-test certification-status=blocker + firewire/remove certification-status=blocker diff --git a/units/firmware/jobs.pxu b/units/firmware/jobs.pxu new file mode 100644 index 0000000..7c777a2 --- /dev/null +++ b/units/firmware/jobs.pxu @@ -0,0 +1,78 @@ +unit: template +template-resource: fwts +plugin: shell +category_id: com.canonical.plainbox::firmware +id: firmware/fwts_{name} +estimated_duration: 1.2 +requires: package.name == 'fwts' +user: root +command: checkbox-support-fwts_test -t {name} -l $PLAINBOX_SESSION_SHARE/fwts_{name}.log +_description: Run {name} test from Firmware Test Suite. +_summary: Run {name} test from Firmware Test Suite. + +unit: template +template-resource: fwts +plugin: attachment +category_id: com.canonical.plainbox::firmware +id: firmware/fwts_{name}.log +estimated_duration: 1.2 +requires: package.name == 'fwts' +user: root +command: [[ -e $PLAINBOX_SESSION_SHARE/fwts_{name}.log ]] && xz -c $PLAINBOX_SESSION_SHARE/fwts_{name}.log | base64 +_description: Attach log for FWTS {name} test. +_summary: Attach log for FWTS {name} test. + +plugin:shell +category_id: com.canonical.plainbox::firmware +id: firmware/fwts_desktop_diagnosis +estimated_duration: 10.0 +requires: + package.name == 'fwts' +user: root +_description: + Run Firmware Test Suite (fwts) QA-concerned desktop-specific diagnosis tests. +_summary: Run FWTS QA-concerned desktop-specific diagnosis tests. +environ: PLAINBOX_SESSION_SHARE +command: + checkbox-support-fwts_test --qa -l $PLAINBOX_SESSION_SHARE/fwts_desktop_diagnosis_results.log + +plugin:shell +category_id: com.canonical.plainbox::firmware +id: firmware/fwts_desktop_diagnosis_hwe +estimated_duration: 5.0 +requires: + package.name == 'fwts' +user: root +_description: + Run Firmware Test Suite (fwts) HWE-concerned desktop-specific diagnosis tests. +_summary: Run FWTS HWE-concerned desktop-specific diagnosis tests. +environ: PLAINBOX_SESSION_SHARE +command: + checkbox-support-fwts_test --hwe -l $PLAINBOX_SESSION_SHARE/fwts_desktop_diagnosis_results_hwe.log + +plugin: attachment +category_id: com.canonical.plainbox::firmware +estimated_duration: 0.5 +id: firmware/fwts_desktop_diagnosis_results.log.gz +command: + [ -f $PLAINBOX_SESSION_SHARE/fwts_desktop_diagnosis_results.log ] && gzip -c $PLAINBOX_SESSION_SHARE/fwts_desktop_diagnosis_results.log | base64 +_description: Attaches the FWTS desktop diagnosis results log to the submission +_summary: Attach FWTS desktop diagnosis log to submission + +plugin: attachment +category_id: com.canonical.plainbox::firmware +estimated_duration: 0.5 +id: firmware/fwts_desktop_diagnosis_results_hwe.log.gz +command: + [ -f $PLAINBOX_SESSION_SHARE/fwts_desktop_diagnosis_results_hwe.log ] && gzip -c $PLAINBOX_SESSION_SHARE/fwts_desktop_diagnosis_results_hwe.log | base64 +_description: Attaches the FWTS desktop diagnosis results log to the submission (to HWE) +_summary: Attach FWTS desktop diagnosis log to submission (to HWE) + +plugin: shell +category_id: com.canonical.plainbox::firmware +id: firmware/no_ACPI_REV_interface +_summary: No _REV interface in ACPI [DS]SDT tables +user: root +estimated_duration: 0.5 +command: ! sudo grep -r "_REV" /sys/firmware/acpi/tables/* | grep [DS]SDT +_description: This Automated test checks misuse of the _REV interface in ACPI DSDT and SSDT tables diff --git a/units/floppy/jobs.pxu b/units/floppy/jobs.pxu new file mode 100644 index 0000000..ff4018d --- /dev/null +++ b/units/floppy/jobs.pxu @@ -0,0 +1,9 @@ +unit: template +template-resource: device +template-filter: device.driver == 'floppy' +plugin: shell +category_id: com.canonical.plainbox::floppy +id: floppy/check_{name} +_description: Floppy test for {product} +user: root +command: floppy_test /dev/{name} diff --git a/units/graphics/jobs.pxu b/units/graphics/jobs.pxu new file mode 100644 index 0000000..a7af65a --- /dev/null +++ b/units/graphics/jobs.pxu @@ -0,0 +1,505 @@ +unit: template +template-resource: graphics_card +template-filter: graphics_card.prime_gpu_offload == 'Off' +plugin: shell +category_id: com.canonical.plainbox::graphics +id: graphics/{index}_driver_version_{product_slug} +command: + source graphics_env {driver} {index} + graphics_driver +estimated_duration: 0.5 +_description: Parses Xorg.0.Log and discovers the running X driver and version for the {vendor} {product} graphics card +_summary: Test X driver/version for {vendor} {product} + +plugin: shell +category_id: com.canonical.plainbox::graphics +id: graphics/xorg-version +requires: package.name == "x11-utils" +command: set -o pipefail; xdpyinfo | grep "^X.Org version" | cut -d ':' -f 2 | tr -d ' ' +estimated_duration: 0.018 +_description: Test to output the Xorg version +_summary: Test Xorg version + +unit: template +template-resource: graphics_card +plugin: manual +category_id: com.canonical.plainbox::graphics +id: graphics/{index}_resolution-change_{product_slug} +depends: graphics/xorg-version +_summary: Test changing resolution for {vendor} {product} +_description: + PURPOSE: + This test will verify that the GUI is usable after manually changing resolution on the {vendor} {product} graphics card. + STEPS: + 1. Select the {vendor} {product} graphics card (a reboot may be necessary) + 2. Open the Displays application + 3. Select a new resolution from the dropdown list + 4. Click on Apply + 5. Select the original resolution from the dropdown list + 6. Click on Apply + VERIFICATION: + Did the resolution on the {vendor} {product} graphics card change as expected? + +plugin: shell +category_id: com.canonical.plainbox::graphics +id: graphics/xorg-process +requires: + package.name == 'xorg' + package.name == 'procps' +command: pgrep -f '/usr/bin/X' >/dev/null +estimated_duration: 0.100 +_description: Test that the X process is running. +_summary: Test that the X process is running. + +plugin: shell +category_id: com.canonical.plainbox::graphics +id: graphics/xorg-failsafe +requires: package.name == 'xorg' +command: ! test -e /var/log/Xorg.failsafe.log +estimated_duration: 0.030 +_description: Test that the X is not running in failsafe mode. +_summary: Test that X is not in failsafe mode. + +unit: template +template-resource: graphics_card +plugin: user-interact-verify +category_id: com.canonical.plainbox::graphics +id: graphics/{index}_resolution_{product_slug} +requires: + device.category == 'VIDEO' + package.name == 'zenity' +command: resolution_test | zenity --text-info +estimated_duration: 10.00 +_summary: Test default resolution for {vendor} {product} +_description: + PURPOSE: + This test will verify the default display resolution on the {vendor} {product} graphics card. + STEPS: + 1. Click "Test". + 2. Confirm that the resolution shown in the text window is acceptable for your display. + VERIFICATION: + Is the resolution shown acceptable for your display on {vendor} {product} graphics card? + +unit: template +template-resource: graphics_card +plugin: user-interact-verify +category_id: com.canonical.plainbox::graphics +id: graphics/{index}_screen_resolution_{product_slug} +requires: + device.category == 'VIDEO' + package.name == 'qmlscene' +command: qmlscene -qt5 --transparent --fullscreen $PLAINBOX_PROVIDER_DATA/resolution_test.qml +estimated_duration: 10.0 +_summary: Test default resolution for {vendor} {product} +_description: + PURPOSE: + This test will verify the default display resolution on the {vendor} {product} graphics card + STEPS: + 1. Click on Test to display the screen resolution overlay for 5 seconds. + VERIFICATION: + Is this acceptable for your display on the {vendor} {product} graphics card? + +unit: template +template-resource: graphics_card +template-filter: graphics_card.prime_gpu_offload == 'Off' +plugin: shell +category_id: com.canonical.plainbox::graphics +id: graphics/{index}_minimum_resolution_{product_slug} +requires: + device.category == 'VIDEO' +command: + source graphics_env {driver} {index} + resolution_test --horizontal 800 --vertical 600 +estimated_duration: 0.331 +_summary: Test that {vendor} {product} meets minimum resolution requirement +_description: + Ensure the current resolution meets or exceeds the recommended minimum + resolution (800x600) on the {vendor} {product} graphics card. See here for details: + https://help.ubuntu.com/community/Installation/SystemRequirements + +unit: template +template-resource: graphics_card +template-filter: graphics_card.prime_gpu_offload == 'Off' +id: graphics/{index}_maximum_resolution_{product_slug} +plugin: user-interact-verify +category_id: com.canonical.plainbox::graphics +requires: + device.category == 'VIDEO' + package.name == 'zenity' +command: + source graphics_env {driver} {index} + maxi="$(xrandr -q |grep -A 1 "connected\( primary\)* [0-9]" |tail -1 |awk '{{print $1}}')" + now="$(python3 -c 'from gi.repository import Gdk; screen=Gdk.Screen.get_default(); geo = screen.get_monitor_geometry(screen.get_primary_monitor()); print(geo.width, "x", geo.height, sep="")')" + test "$maxi" != "$now" && notify="\nPlease switch to the maximum resolution \nfor every graphic tests" + zenity --info --text "Maximum resolution: $maxi\nCurrent resolution: $now $notify" +estimated_duration: 10.0 +_summary: Test maximum supported resolution for {vendor} {product} +_description: + PURPOSE: + This test will verify the maximum supported resolution on the {vendor} {product} graphics card. + STEPS: + 1. Select the {vendor} {product} graphics card (a reboot may be necessary) + 2. Consult the system's specifications and locate the screen's maximum supported resolution. + 3. Click on Test to display the maximum resolution that can be used by Ubuntu on the current display. + VERIFICATION: + Is this the maximum resolution for the display connected to the {vendor} {product} graphics card? + +unit: template +template-resource: graphics_card +id: graphics/{index}_modes_{product_slug} +plugin: shell +category_id: com.canonical.plainbox::graphics +command: graphics_modes_info +estimated_duration: 0.250 +_description: Collect info on graphics modes (screen resolution and refresh rate) for {vendor} {product} +_summary: Test graphic modes info for {vendor} {product} + +unit: template +template-resource: graphics_card +id: graphics/{index}_color_depth_{product_slug} +plugin: shell +category_id: com.canonical.plainbox::graphics +command: color_depth_info +estimated_duration: 0.150 +_description: Collect info on color depth and pixel format for {vendor} {product} +_summary: Test color depth info for {vendor} {product} + +unit: template +template-resource: graphics_card +id: graphics/{index}_fresh_rate_{product_slug} +plugin: shell +category_id: com.canonical.plainbox::graphics +command: fresh_rate_info +_description: Collect info on fresh rate for {vendor} {product}. +_summary: Test refresh rate for {vendor} {product} + +unit: template +template-resource: graphics_card +id: graphics/{index}_graphic_memory_{product_slug} +plugin: shell +category_id: com.canonical.plainbox::graphics +command: graphic_memory_info +_description: Collect info on graphic memory for {vendor} {product}. +_summary: Test graphic memory info for {vendor} {product}. + +unit: template +template-resource: graphics_card +id: graphics/{index}_video_{product_slug} +plugin: user-interact-verify +category_id: com.canonical.plainbox::graphics +requires: + package.name == 'xorg' + package.name == 'gir1.2-gst-plugins-base-0.10' or package.name == 'gir1.2-gst-plugins-base-1.0' +command: + source graphics_env {driver} {index} + gst_pipeline_test -t 2 'videotestsrc ! videoconvert ! autovideosink' || gst_pipeline_test -t 2 'videotestsrc ! ffmpegcolorspace ! autovideosink' +_summary: Test that video can be displayed with {vendor} {product} +_description: + PURPOSE: + This test will test the default display with a sample video + STEPS: + 1. Click "Test" to display a video test. + VERIFICATION: + Do you see color bars and static? + +plugin: shell +category_id: com.canonical.plainbox::graphics +id: graphics/VESA_drivers_not_in_use +command: cat /var/log/Xorg.0.log | perl -e '$a=0;while(<>){$a++ if /Loading.*vesa_drv\.so/;$a-- if /Unloading.*vesa/&&$a}exit 1 if $a' +estimated_duration: 0.011 +_description: Check that VESA drivers are not in use +_summary: Test that VESA drivers are not in use + +unit: template +template-resource: graphics_card +template-filter: graphics_card.prime_gpu_offload == 'Off' +plugin: user-interact-verify +category_id: com.canonical.plainbox::graphics +id: graphics/{index}_cycle_resolution_{product_slug} +requires: package.name == 'xorg' +depends: graphics/VESA_drivers_not_in_use +command: + source graphics_env {driver} {index} + xrandr_cycle --screenshot-dir $PLAINBOX_SESSION_SHARE +estimated_duration: 250.000 +_summary: Test resolution cycling for {vendor} {product} +_description: + PURPOSE: + This test cycles through the detected video modes for the {vendor} {product} graphics card + STEPS: + 1. Click "Test" to start cycling through the video modes + VERIFICATION: + Did the screen appear to be working for each mode? + +unit: template +template-resource: graphics_card +template-filter: graphics_card.prime_gpu_offload == 'Off' +plugin: user-interact-verify +category_id: com.canonical.plainbox::graphics +id: graphics/{index}_rotation_{product_slug} +depends: graphics/xorg-version +command: + source graphics_env {driver} {index} + rotation_test_using_dbus +estimated_duration: 20.000 +_summary: Test rotation for {vendor} {product} +_description: + PURPOSE: + This test will test display rotation on the {vendor} {product} graphics card + STEPS: + 1. Click "Test" to test display rotation. The display will be rotated every 4 seconds. + 2. Check if all rotations (normal right inverted left) took place without permanent screen corruption + VERIFICATION: + Did the display rotation take place without without permanent screen corruption? + +unit: template +template-resource: graphics_card +template-filter: graphics_card.prime_gpu_offload == 'Off' +plugin: shell +category_id: com.canonical.plainbox::graphics +id: graphics/{index}_compiz_check_{product_slug} +requires: package.name == 'nux-tools' +command: + source graphics_env {driver} {index} + ! /usr/lib/nux/unity_support_test -c -p 2>&1 | ansi_parser | grep -e ":\(\s\+\)no$" -ie "error" +estimated_duration: 0.130 +_description: Check that {vendor} {product} hardware is able to run compiz +_summary: Test Compiz support for {vendor} {product} + +unit: template +template-resource: graphics_card +plugin: shell +category_id: com.canonical.plainbox::graphics +id: graphics/{index}_unity_support_{product_slug} +requires: package.name == 'nux-tools' +command: ! /usr/lib/nux/unity_support_test -p 2>&1 | ansi_parser | grep -e ":\(\s\+\)no$" -ie "error" +estimated_duration: 0.131 +_description: Check that {vendor} {product} hardware is able to run Unity 3D +_summary: Test Unity 3D support for {vendor} {product} + +unit: template +template-resource: graphics_card +template-filter: graphics_card.driver in ['amdgpu', 'amdgpu-pro'] +plugin: shell +category_id: com.canonical.plainbox::graphics +id: graphics/{index}_valid_opengl_renderer_{product_slug} +requires: package.name == 'mesa-utils' +command: + renderer=`DRI_PRIME=1 glxinfo | grep "OpenGL re"` + echo $renderer + if grep -qi 'Intel' <<<$renderer; then + echo 'ERROR: renderer is Intel when DRI_PRIME=1' + exit 1 + fi +_summary: Check the OpenGL renderer (AMD GPU and DRI_PRIME=1) + +unit: template +template-resource: graphics_card +plugin: user-interact-verify +category_id: com.canonical.plainbox::graphics +id: graphics/{index}_glxgears_{product_slug} +requires: package.name == 'mesa-utils' +command: + source graphics_env {driver} {index} + glxgears + true +_summary: Test that glxgears works for {vendor} {product} +_description: + PURPOSE: + This test tests the basic 3D capabilities of your {vendor} {product} video card + STEPS: + 1. Click "Test" to execute an OpenGL demo. Press ESC at any time to close. + 2. Verify that the animation is not jerky or slow. + VERIFICATION: + 1. Did the 3d animation appear? + 2. Was the animation free from slowness/jerkiness? + +unit: template +template-resource: graphics_card +plugin: shell +category_id: com.canonical.plainbox::graphics +id: graphics/{index}_3d_window_open_close_{product_slug} +requires: package.name == 'mesa-utils' +command: window_test -t open-close -i 10 +estimated_duration: 60.525 +_description: Open and close a 3D window multiple times on the {vendor} {product} video card +_summary: Test 3D window open/close for {vendor} {product} + +unit: template +template-resource: graphics_card +plugin: shell +category_id: com.canonical.plainbox::graphics +id: graphics/{index}_3d_window_suspend_resume_{product_slug} +requires: package.name == 'mesa-utils' +command: window_test -t suspend-resume -i 10 +estimated_duration: 121.00 +_description: Open, suspend resume and close a 3D window multiple times on the {vendor} {product} video card +_summary: Test a 3D window with suspend/resume for {vendor} {product} + +unit: template +template-resource: graphics_card +plugin: shell +category_id: com.canonical.plainbox::graphics +id: graphics/{index}_multi_3d_windows_open_close_{product_slug} +requires: package.name == 'mesa-utils' +command: window_test -t open-close-multi -i 10 -w 4 +estimated_duration: 60.000 +_description: Open and close 4 3D windows multiple times on the {vendor} {product} video card +_summary: Test Multi 3D window open/close for {vendor} {product} + +unit: template +template-resource: graphics_card +plugin: shell +category_id: com.canonical.plainbox::graphics +id: graphics/{index}_3d_window_move_{product_slug} +requires: + package.name == 'mesa-utils' + package.name == 'wmctrl' +command: window_test -t move +estimated_duration: 50.000 +_description: Move a 3D window around the screen on the {vendor} {product} video card +_summary: Test 3D window movement for {vendor} {product} + +unit: template +template-resource: graphics_card +plugin: shell +category_id: com.canonical.plainbox::graphics +id: graphics/{index}_screenshot_{product_slug} +requires: package.name == 'fswebcam' +command: set -o pipefail; camera_test still --device=/dev/external_webcam -f $PLAINBOX_SESSION_SHARE/screenshot_{index}.jpg -q 2>&1 | ansi_parser +_summary: Test grabbing a screenshot for {vendor} {product} +_description: + PURPOSE: + Take a screengrab of the current screen (logged on Unity desktop) as produced by the {vendor} {product} graphics card + STEPS: + 1. Take picture using USB webcam + VERIFICATION: + Review attachment manually later + +unit: template +template-resource: graphics_card +plugin: attachment +category_id: com.canonical.plainbox::graphics +id: graphics/{index}_screenshot_{product_slug}.jpg +depends: graphics/{index}_screenshot_{product_slug} +command: base64 $PLAINBOX_SESSION_SHARE/screenshot_{index}.jpg +_description: Attaches the screenshot captured in graphics/screenshot for the {vendor} {product} graphics card. +_summary: Attach results of screenshot test for {vendor} {product} + +unit: template +template-resource: graphics_card +plugin: shell +category_id: com.canonical.plainbox::graphics +id: graphics/{index}_screenshot_fullscreen_video_{product_slug} +requires: package.name == 'fswebcam' +command: + [ -f $PLAINBOX_PROVIDER_DATA/video/Ogg_Theora_Video.ogv ] || {{ echo "Video file not found"; exit 1; }} + gsettings set org.gnome.totem repeat true + totem --fullscreen $PLAINBOX_PROVIDER_DATA/video/Ogg_Theora_Video.ogv 2>/dev/null & + set -o pipefail + sleep 15 && camera_test still --device=/dev/external_webcam -f $PLAINBOX_SESSION_SHARE/screenshot_fullscreen_video_{index}.jpg -q 2>&1 | ansi_parser + sleep 5 && totem --quit 2>/dev/null + gsettings set org.gnome.totem repeat false +_summary: Test FSV screenshot for {vendor} {product} +_description: + PURPOSE: + Take a screengrab of the current screen during fullscreen video playback using the {vendor} {product} graphics card + STEPS: + 1. Start a fullscreen video playback + 2. Take picture using USB webcam after a few seconds + VERIFICATION: + Review attachment manually later + +unit: template +template-resource: graphics_card +plugin: manual +category_id: com.canonical.plainbox::graphics +id: graphics/{index}_switch_card_{product_slug} +_summary: Test GPU switching for {vendor} {product} +_description: +_purpose: + Manually switch to the {vendor} {product} card. +_steps: + Note: If your system is already running with the {vendor} {product} card, then please mark this test as "passed" and proceed. + . + 1. Using the appropriate tool (either NVidia settings or AMD Control Center), switch your system to use the {vendor} {product} graphics card. This will require restarting your session. + 2. Once the session restarts, please restart this testing program and select "continue" when prompted for a resume action. + 3. Don't answer the verification question until the system has restarted with the {vendor} {product} card enabled. +_verification: + Is the system using the {vendor} {product} card now? + +unit: template +template-resource: graphics_card +template-filter: graphics_card.prime_gpu_offload == 'Off' +plugin: manual +category_id: com.canonical.plainbox::graphics +id: graphics/{index}_switch_card_{product_slug}_xenial +requires: + graphics_card.driver in ['nouveau', 'nvidia', 'amdgpu-pro'] + lsb.codename == 'xenial' +_summary: Test GPU switching for {vendor} {product} +_description: +_purpose: + Manually switch to the {vendor} {product} card. +_steps: + Note: If your system is already running with the {vendor} {product} card, then please mark this test as "passed" and proceed. + . + 1. Using the appropriate tool (e.g. NVidia settings, amdgpu-pro-px), switch your system to use the {vendor} {product} graphics card. This will require restarting your session. + If using amdgpu-pro drivers, use the following command to switch to: + - integrated GPU: sudo /opt/amdgpu-pro/bin/amdgpu-pro-px --mode powersaving + - discrete GPU: sudo /opt/amdgpu-pro/bin/amdgpu-pro-px --mode performance + 2. Once the session restarts, please restart this testing program and select "continue" when prompted for a resume action. + 3. Don't answer the verification question until the system has restarted with the {vendor} {product} card enabled. +_verification: + Is the system using the {vendor} {product} card now? + +unit: template +template-resource: graphics_card +plugin: attachment +category_id: com.canonical.plainbox::graphics +id: graphics/{index}_screenshot_fullscreen_video_{product_slug}.jpg +depends: graphics/{index}_screenshot_fullscreen_video_{product_slug} +command: base64 $PLAINBOX_SESSION_SHARE/screenshot_fullscreen_video_{index}.jpg +_summary: Attach results of FSV screenshot test for {vendor} {product} +_description: Attaches the screenshot captured in graphics/screenshot_fullscreen_video for the {vendor} {product} graphics card + +plugin: shell +category_id: com.canonical.plainbox::graphics +id: graphics/screen-capture-internal +_summary: Obtains a simple screen capture +estimated_duration: 1.0 +requires: package.name == 'gnome-screenshot' +command: gnome-screenshot --file ${PLAINBOX_SESSION_SHARE}/screen-capture.png +_description: Obtains a simple screen capture using gnome-screenshot if present + +plugin: attachment +category_id: com.canonical.plainbox::graphics +id: screen-capture-internal.png +_summary: Attaches a simple screen capture +estimated_duration: 1.0 +depends: graphics/screen-capture-internal +command: [ -f ${PLAINBOX_SESSION_SHARE}/screen-capture.png ] && base64 ${PLAINBOX_SESSION_SHARE}/screen-capture.png +_description: Attaches the simple screen capture file + +unit: template +template-resource: graphics_card +plugin: shell +category_id: com.canonical.plainbox::graphics +id: graphics/{index}_screen-capture-internal_{product_slug} +_summary: Obtains a simple screen capture of {vendor} {product} +estimated_duration: 1.0 +requires: package.name == 'gnome-screenshot' +command: gnome-screenshot --file $PLAINBOX_SESSION_SHARE/screen-capture-{index}.png +_description: Obtains a simple screen capture of {vendor} {product} using gnome-screenshot if present + +unit: template +template-resource: graphics_card +plugin: attachment +category_id: com.canonical.plainbox::graphics +id: graphics/{index}_screen-capture-internal_{product_slug}.png +depends: graphics/{index}_screen-capture-internal_{product_slug} +_summary: Attaches a simple screen capture of {vendor} {product} +estimated_duration: 1.0 +command: [ -f $PLAINBOX_SESSION_SHARE/screen-capture-{index}.png ] && base64 $PLAINBOX_SESSION_SHARE/screen-capture-{index}.png +_description: Attaches the simple screen capture file of {vendor} {product} + diff --git a/units/graphics/legacy.pxu b/units/graphics/legacy.pxu new file mode 100644 index 0000000..91eb888 --- /dev/null +++ b/units/graphics/legacy.pxu @@ -0,0 +1,276 @@ +plugin: shell +category_id: com.canonical.plainbox::graphics +id: graphics/driver_version +command: graphics_driver +estimated_duration: 0.500 +_description: Parses Xorg.0.Log and discovers the running X driver and version +_summary: Test X driver/version + +plugin: manual +category_id: com.canonical.plainbox::graphics +id: graphics/resolution-change +depends: graphics/xorg-version +_summary: Test changing resolution +_description: + PURPOSE: + This test will verify that the GUI is usable after manually changing resolution + STEPS: + 1. Open the Displays application + 2. Select a new resolution from the dropdown list + 3. Click on Apply + 4. Select the original resolution from the dropdown list + 5. Click on Apply + VERIFICATION: + Did the resolution change as expected? + +plugin: user-interact-verify +category_id: com.canonical.plainbox::graphics +id: graphics/resolution +requires: + device.category == 'VIDEO' +flags: deprecated +command: resolution_test +estimated_duration: 0.750 +_summary: Test default resolution (for old checkbox) +_description: + PURPOSE: + This test will verify the default display resolution + STEPS: + 1. This display is using the following resolution: + INFO: + $output + VERIFICATION: + Is this acceptable for your display? + +plugin: user-interact-verify +category_id: com.canonical.plainbox::graphics +id: graphics/screen-resolution +requires: + device.category == 'VIDEO' + package.name == 'qmlscene' +command: qmlscene -qt5 --transparent --fullscreen $PLAINBOX_PROVIDER_DATA/resolution_test.qml +estimated_duration: 10.0 +_summary: Test default resolution +_description: + PURPOSE: + This test will verify the default display resolution + STEPS: + 1. Click on Test to display the screen resolution overlay for 5 seconds. + VERIFICATION: + Is this acceptable for your display? + +plugin: shell +category_id: com.canonical.plainbox::graphics +id: graphics/minimum_resolution +requires: + device.category == 'VIDEO' +command: resolution_test --horizontal 800 --vertical 600 +estimated_duration: 0.331 +_summary: Test that system meets minimum resolution requirement +_description: + Ensure the current resolution meets or exceeds the recommended minimum + resolution (800x600). See here for details: + . + https://help.ubuntu.com/community/Installation/SystemRequirements + +plugin: user-interact-verify +category_id: com.canonical.plainbox::graphics +id: graphics/maximum_resolution +requires: + device.category == 'VIDEO' + package.name == 'zenity' +command: zenity --info --text "Maximum resolution: $(xrandr -q |grep -A 1 "connected\( primary\)* [0-9]" |tail -1 |awk '{print $1}')" +estimated_duration: 10 +_summary: Test maximum supported resolution +_description: + PURPOSE: + This test will verify the display is operating at its maximum supported resolution + STEPS: + 1. Consult the system's specifications and locate the screen's maximum supported resolution. + 2. Click on Test to display the maximum resolution that can be used by Ubuntu on the current display. + VERIFICATION: + Is this the display's maximum resolution? + +id: graphics/modes +plugin: shell +category_id: com.canonical.plainbox::graphics +command: graphics_modes_info +estimated_duration: 0.250 +_description: Collect info on graphics modes (screen resolution and refresh rate) +_summary: Collect info on graphics modes + +id: graphics/color_depth +plugin: shell +category_id: com.canonical.plainbox::graphics +command: color_depth_info +estimated_duration: 0.150 +_description: Collect info on color depth and pixel format. +_summary: Collect info on color depth and pixel format. + +id: graphics/fresh_rate +plugin: shell +category_id: com.canonical.plainbox::graphics +command: fresh_rate_info +_description: Collect info on fresh rate. +_summary: Collect info on fresh rate. + +id: graphics/graphic_memory +plugin: shell +category_id: com.canonical.plainbox::graphics +command: graphic_memory_info +_description: Collect info on graphic memory. +_summary: Collect info on graphic memory. + +plugin: user-interact-verify +category_id: com.canonical.plainbox::graphics +id: graphics/cycle_resolution +requires: package.name == 'xorg' +depends: graphics/VESA_drivers_not_in_use +command: xrandr_cycle --screenshot-dir $PLAINBOX_SESSION_SHARE +estimated_duration: 250.000 +_summary: Test resolution cycling +_description: + PURPOSE: + This test cycles through the detected video modes + STEPS: + 1. Click "Test" to start cycling through the video modes + VERIFICATION: + Did the screen appear to be working for each mode? + +plugin: user-interact-verify +category_id: com.canonical.plainbox::graphics +id: graphics/rotation +depends: graphics/xorg-version +command: rotation_test +estimated_duration: 20.000 +_summary: Test rotation +_description: + PURPOSE: + This test will test display rotation + STEPS: + 1. Click "Test" to test display rotation. The display will be rotated every 4 seconds. + 2. Check if all rotations (normal right inverted left) took place without permanent screen corruption + VERIFICATION: + Did the display rotation take place without without permanent screen corruption? + +plugin: shell +category_id: com.canonical.plainbox::graphics +id: graphics/compiz_check +requires: package.name == 'nux-tools' +command: ! /usr/lib/nux/unity_support_test -c -p 2>&1 | ansi_parser | grep -e ":\(\s\+\)no$" -ie "error" +estimated_duration: 0.130 +_summary: Test Compiz support +_description: Check that hardware is able to run compiz + +plugin: shell +category_id: com.canonical.plainbox::graphics +id: graphics/unity-support +requires: package.name == 'nux-tools' +command: ! /usr/lib/nux/unity_support_test -p 2>&1 | ansi_parser | grep -e ":\(\s\+\)no$" -ie "error" +estimated_duration: 0.131 +_description: Check that hardware is able to run Unity 3D +_summary: Test Unity 3D support + +plugin: user-interact-verify +category_id: com.canonical.plainbox::graphics +id: graphics/glxgears +requires: package.name == 'mesa-utils' +command: glxgears; true +_summary: Test that glxgears works +_description: + PURPOSE: + This test tests the basic 3D capabilities of your video card + STEPS: + 1. Click "Test" to execute an OpenGL demo. Press ESC at any time to close. + 2. Verify that the animation is not jerky or slow. + VERIFICATION: + 1. Did the 3d animation appear? + 2. Was the animation free from slowness/jerkiness? + +plugin: shell +category_id: com.canonical.plainbox::graphics +id: graphics/3d_window_open_close +requires: package.name == 'mesa-utils' +command: window_test -t open-close -i 10 +estimated_duration: 60.525 +_summary: Test 3D window open/close +_description: Open and close a 3D window multiple times + +plugin: shell +category_id: com.canonical.plainbox::graphics +id: graphics/3d_window_suspend_resume +requires: package.name == 'mesa-utils' +command: window_test -t suspend-resume -i 10 +estimated_duration: 121.00 +_description: Open, suspend resume and close a 3D window multiple times +_summary: Test a 3D window with suspend/resume + +plugin: shell +category_id: com.canonical.plainbox::graphics +id: graphics/multi_3d_windows_open_close +requires: package.name == 'mesa-utils' +command: window_test -t open-close-multi -i 10 -w 4 +estimated_duration: 60.000 +_description: Open and close 4 3D windows multiple times +_summary: Test Multi 3D window open/close + +plugin: shell +category_id: com.canonical.plainbox::graphics +id: graphics/3d_window_move +requires: package.name == 'mesa-utils' +command: window_test -t move +estimated_duration: 50.000 +_description: Move a 3D window around the screen +_summary: Test 3D window movement + +plugin: shell +category_id: com.canonical.plainbox::graphics +id: graphics/screenshot +requires: package.name == 'fswebcam' +command: set -o pipefail; camera_test still --device=/dev/external_webcam -f ${PLAINBOX_SESSION_SHARE}/screenshot.jpg -q 2>&1 | ansi_parser +_summary: Test grabbing a screenshot +_description: + PURPOSE: + Take a screengrab of the current screen (logged on Unity desktop) + STEPS: + 1. Take picture using USB webcam + VERIFICATION: + Review attachment manually later + +plugin: attachment +category_id: com.canonical.plainbox::graphics +id: screenshot.jpg +depends: graphics/screenshot +command: base64 ${PLAINBOX_SESSION_SHARE}/screenshot.jpg +_description: Attaches the screenshot captured in graphics/screenshot. +_summary: Attach results of screenshot test + +plugin: shell +category_id: com.canonical.plainbox::graphics +id: graphics/screenshot_fullscreen_video +requires: package.name == 'fswebcam' +command: + [ -f ${PLAINBOX_PROVIDER_DATA}/video/Ogg_Theora_Video.ogv ] || { echo "Video file not found"; exit 1; } + gsettings set org.gnome.totem repeat true + totem --fullscreen ${PLAINBOX_PROVIDER_DATA}/video/Ogg_Theora_Video.ogv 2>/dev/null & + set -o pipefail + sleep 15 && camera_test still --device=/dev/external_webcam -f ${PLAINBOX_SESSION_SHARE}/screenshot_fullscreen_video.jpg -q 2>&1 | ansi_parser + sleep 5 && totem --quit 2>/dev/null + gsettings set org.gnome.totem repeat false +_summary: Test FSV screenshot +_description: + PURPOSE: + Take a screengrab of the current screen during fullscreen video playback + STEPS: + 1. Start a fullscreen video playback + 2. Take picture using USB webcam after a few seconds + VERIFICATION: + Review attachment manually later + +plugin: attachment +category_id: com.canonical.plainbox::graphics +id: screenshot_fullscreen_video.jpg +depends: graphics/screenshot_fullscreen_video +command: base64 ${PLAINBOX_SESSION_SHARE}/screenshot_fullscreen_video.jpg +_description: Attaches the screenshot captured in graphics/screenshot_fullscreen_video. +_summary: Attach results of FSV screenshot test diff --git a/units/graphics/test-plan.pxu b/units/graphics/test-plan.pxu new file mode 100644 index 0000000..a85b194 --- /dev/null +++ b/units/graphics/test-plan.pxu @@ -0,0 +1,144 @@ +id: graphics-integrated-gpu-cert-full +unit: test plan +_name: Graphics tests (integrated GPU) +_description: Graphics tests (integrated GPU) +include: + miscellanea/chvt + graphics/xorg-version certification-status=blocker + graphics/xorg-failsafe certification-status=blocker + graphics/xorg-process certification-status=blocker + graphics/VESA_drivers_not_in_use certification-status=blocker + graphics/1_maximum_resolution_.* certification-status=blocker + graphics/1_glxgears_.* certification-status=blocker + graphics/1_driver_version_.* certification-status=blocker + graphics/1_compiz_check_.* certification-status=blocker + graphics/1_rotation_.* certification-status=blocker + graphics/1_video_.* certification-status=blocker + graphics/1_minimum_resolution_.* + graphics/1_cycle_resolution_.* certification-status=non-blocker +bootstrap_include: + graphics_card + +id: graphics-discrete-gpu-cert-full +unit: test plan +_name: Graphics tests (discrete GPU) +_description: Graphics tests (discrete GPU) +include: + graphics/2_switch_card_.*_xenial certification-status=blocker + graphics/2_maximum_resolution_.* certification-status=blocker + graphics/2_valid_opengl_renderer_.* certification-status=blocker + graphics/2_glxgears_.* certification-status=blocker + graphics/2_driver_version_.* certification-status=blocker + graphics/2_compiz_check_.* certification-status=blocker + graphics/2_rotation_.* certification-status=blocker + graphics/2_video_.* certification-status=blocker + graphics/2_minimum_resolution_.* + graphics/2_cycle_resolution_.* certification-status=non-blocker +bootstrap_include: + graphics_card + +id: after-suspend-graphics-integrated-gpu-cert-full +unit: test plan +_name: After suspend tests (integrated GPU) +_description: After suspend tests (integrated GPU) +include: + graphics/1_switch_card_.*_xenial certification-status=blocker + suspend/1_resolution_before_suspend_.*_xenial certification-status=blocker + suspend/suspend_advanced certification-status=blocker + suspend/1_suspend-time-check_.*_xenial certification-status=non-blocker + suspend/1_suspend-single-log-attach_.*_xenial + power-management/lid certification-status=blocker + power-management/lid_close certification-status=blocker + power-management/lid_open certification-status=blocker + suspend/1_compiz_check_after_suspend_.*_xenial certification-status=blocker + suspend/1_driver_version_after_suspend_.*_xenial certification-status=blocker + suspend/1_resolution_after_suspend_.*_xenial certification-status=blocker + suspend/1_display_after_suspend_.*_xenial certification-status=blocker + suspend/1_glxgears_after_suspend_.*_xenial certification-status=blocker + suspend/1_video_after_suspend_.*_xenial certification-status=blocker + suspend/1_cycle_resolutions_after_suspend_.*_xenial certification-status=non-blocker + suspend/1_xrandr_screens_after_suspend.tar.gz_xenial + +id: after-suspend-graphics-discrete-gpu-cert-full +unit: test plan +_name: After suspend tests (discrete GPU) +_description: After suspend tests (discrete GPU) +include: + suspend/2_resolution_before_suspend_.*_xenial certification-status=blocker + suspend/2_suspend_after_switch_to_card_.*_xenial certification-status=blocker + suspend/2_suspend-time-check_.*_xenial certification-status=non-blocker + suspend/2_suspend-single-log-attach_.*_xenial + suspend/2_compiz_check_after_suspend_.*_xenial certification-status=blocker + suspend/2_driver_version_after_suspend_.*_xenial certification-status=blocker + suspend/2_resolution_after_suspend_.*_xenial certification-status=blocker + suspend/2_display_after_suspend_.*_xenial certification-status=blocker + suspend/2_glxgears_after_suspend_.*_xenial certification-status=blocker + suspend/2_video_after_suspend_.*_xenial certification-status=blocker + suspend/2_cycle_resolutions_after_suspend_.*_xenial certification-status=non-blocker + suspend/2_xrandr_screens_after_suspend_.*.tar.gz_xenial + +id: graphics-integrated-gpu-cert-blockers +unit: test plan +_name: Graphics tests (integrated GPU, certification blockers only) +_description: Graphics tests (integrated GPU, certification blockers only) +include: + graphics/xorg-version certification-status=blocker + graphics/xorg-failsafe certification-status=blocker + graphics/xorg-process certification-status=blocker + graphics/VESA_drivers_not_in_use certification-status=blocker + graphics/1_maximum_resolution_.* certification-status=blocker + graphics/1_glxgears_.* certification-status=blocker + graphics/1_driver_version_.* certification-status=blocker + graphics/1_compiz_check_.* certification-status=blocker + graphics/1_rotation_.* certification-status=blocker + graphics/1_video_.* certification-status=blocker +bootstrap_include: + graphics_card + +id: graphics-discrete-gpu-cert-blockers +unit: test plan +_name: Graphics tests (discrete GPU, certification blockers only) +_description: Graphics tests (discrete GPU, certification blockers only) +include: + graphics/2_switch_card_.*_xenial certification-status=blocker + graphics/2_maximum_resolution_.* certification-status=blocker + graphics/2_valid_opengl_renderer_.* certification-status=blocker + graphics/2_glxgears_.* certification-status=blocker + graphics/2_driver_version_.* certification-status=blocker + graphics/2_compiz_check_.* certification-status=blocker + graphics/2_rotation_.* certification-status=blocker + graphics/2_video_.* certification-status=blocker +bootstrap_include: + graphics_card + +id: after-suspend-graphics-integrated-gpu-cert-blockers +unit: test plan +_name: After suspend tests (integrated GPU, certification blockers only) +_description: After suspend tests (integrated GPU, certification blockers only) +include: + graphics/1_switch_card_.*_xenial certification-status=blocker + suspend/1_resolution_before_suspend_.*_xenial certification-status=blocker + suspend/suspend_advanced certification-status=blocker + power-management/lid certification-status=blocker + power-management/lid_close certification-status=blocker + power-management/lid_open certification-status=blocker + suspend/1_compiz_check_after_suspend_.*_xenial certification-status=blocker + suspend/1_driver_version_after_suspend_.*_xenial certification-status=blocker + suspend/1_resolution_after_suspend_.*_xenial certification-status=blocker + suspend/1_display_after_suspend_.*_xenial certification-status=blocker + suspend/1_glxgears_after_suspend_.*_xenial certification-status=blocker + suspend/1_video_after_suspend_.*_xenial certification-status=blocker + +id: after-suspend-graphics-discrete-gpu-cert-blockers +unit: test plan +_name: After suspend tests (discrete GPU, certification blockers only) +_description: After suspend tests (discrete GPU, certification blockers only) +include: + suspend/2_resolution_before_suspend_.*_xenial certification-status=blocker + suspend/2_suspend_after_switch_to_card_.*_xenial certification-status=blocker + suspend/2_compiz_check_after_suspend_.*_xenial certification-status=blocker + suspend/2_driver_version_after_suspend_.*_xenial certification-status=blocker + suspend/2_resolution_after_suspend_.*_xenial certification-status=blocker + suspend/2_display_after_suspend_.*_xenial certification-status=blocker + suspend/2_glxgears_after_suspend_.*_xenial certification-status=blocker + suspend/2_video_after_suspend_.*_xenial certification-status=blocker diff --git a/units/hibernate/jobs.pxu b/units/hibernate/jobs.pxu new file mode 100644 index 0000000..75f233f --- /dev/null +++ b/units/hibernate/jobs.pxu @@ -0,0 +1,97 @@ +plugin: user-interact-verify +category_id: com.canonical.plainbox::hibernate +id: power-management/hibernate_advanced +requires: + sleep.disk == 'supported' + rtc.state == 'supported' +environ: PLAINBOX_SESSION_SHARE +user: root +command: + if type -P fwts >/dev/null; then + echo "Calling fwts" + checkbox-support-fwts_test -l $PLAINBOX_SESSION_SHARE/hibernate-single -f none -s s4 --s4-device-check --s4-device-check-delay=45 --s4-sleep-delay=120 + else + echo "Calling sleep_test" + sleep_test -s disk -w 120 + fi +estimated_duration: 300.00 +_description: + PURPOSE: + This test will check to make sure your system can successfully hibernate (if supported) + STEPS: + 1. Click on Test + 2. The system will hibernate and should wake itself within 5 minutes + 3. If your system does not wake itself after 5 minutes, please press the power button to wake the system manually + 4. If the system fails to resume from hibernate, please restart System Testing and mark this test as Failed + VERIFICATION: + Did the system successfully hibernate and did it work properly after waking up? + +plugin: shell +category_id: com.canonical.plainbox::hibernate +id: power-management/hibernate_advanced_auto +requires: + sleep.disk == 'supported' + rtc.state == 'supported' +environ: PLAINBOX_SESSION_SHARE +user: root +command: checkbox-support-fwts_test -l $PLAINBOX_SESSION_SHARE/hibernate-single -f none -s s4 --s4-device-check --s4-device-check-delay=45 --s4-sleep-delay=120 +estimated_duration: 300.00 +_description: + This is the automated version of suspend/hibernate_advanced. + +unit: template +template-resource: graphics_card +template-filter: graphics_card.prime_gpu_offload == 'Off' +requires: + sleep.disk == 'supported' + rtc.state == 'supported' +plugin: user-interact-verify +category_id: com.canonical.plainbox::hibernate +id: power-management/{index}_hibernate_advanced_{vendor_slug}_{product_slug} +user: root +environ: PLAINBOX_SESSION_SHARE +estimated_duration: 300.00 +command: + if type -P fwts >/dev/null; then + echo "Calling fwts" + checkbox-support-fwts_test -l $PLAINBOX_SESSION_SHARE/{index}_hibernate-single -f none -s s4 --s4-device-check --s4-device-check-delay=45 --s4-sleep-delay=120 + else + echo "Calling sleep_test" + sleep_test -s disk -w 120 + fi +_purpose: + This test will check to make sure your system can successfully hibernate (if supported) +_steps: + 1. Click on Test + 2. The system will hibernate and should wake itself within 5 minutes + 3. If your system does not wake itself after 5 minutes, please press the power button to wake the system manually + 4. If the system fails to resume from hibernate, please restart System Testing and mark this test as Failed +_verification: + Did the system successfully hibernate and did it work properly after waking up? + +plugin: shell +category_id: com.canonical.plainbox::hibernate +id: power-management/hibernate-single-log-check +command: [ -e $PLAINBOX_SESSION_SHARE/hibernate-single.log ] && sleep_test_log_check -v s4 $PLAINBOX_SESSION_SHARE/hibernate-single.log +_description: + Automated check of the hibernate log for errors discovered by fwts + +plugin: attachment +category_id: com.canonical.plainbox::hibernate +id: power-management/hibernate-single-log-attach +depends: power-management/hibernate_advanced +command: [ -e $PLAINBOX_SESSION_SHARE/hibernate-single.log ] && cat $PLAINBOX_SESSION_SHARE/hibernate-single.log +estimated_duration: 0.5 +_description: attaches log from single hibernate/resume test to results + +unit: template +template-resource: graphics_card +template-filter: graphics_card.prime_gpu_offload == 'Off' +plugin: attachment +category_id: com.canonical.plainbox::hibernate +id: power-management/{index}_hibernate-single-log-attach_{product_slug} +depends: power-management/{index}_hibernate_advanced_{vendor_slug}_{product_slug} +estimated_duration: 0.5 +command: [ -e $PLAINBOX_SESSION_SHARE/{index}_hibernate-single.log ] && cat $PLAINBOX_SESSION_SHARE/{index}_hibernate-single.log +_description: attaches log from single hibernate/resume test to results + diff --git a/units/hibernate/test-plan.pxu b/units/hibernate/test-plan.pxu new file mode 100644 index 0000000..8514f12 --- /dev/null +++ b/units/hibernate/test-plan.pxu @@ -0,0 +1,16 @@ +id: hibernate-integrated-gpu-cert-full +unit: test plan +_name: Hibernate tests (integrated GPU) +_description: Hibernate tests (integrated GPU) +include: + power-management/1_hibernate_advanced_.* certification-status=non-blocker + power-management/1_hibernate-single-log-attach_.* + keys/hibernate certification-status=non-blocker + +id: hibernate-discrete-gpu-cert-full +unit: test plan +_name: Hibernate tests (discrete GPU) +_description: Hibernate tests (discrete GPU) +include: + power-management/2_hibernate_advanced_.* certification-status=non-blocker + power-management/2_hibernate-single-log-attach_.* diff --git a/units/info/jobs.pxu b/units/info/jobs.pxu new file mode 100644 index 0000000..0588e26 --- /dev/null +++ b/units/info/jobs.pxu @@ -0,0 +1,417 @@ +id: codecs_attachment +plugin: attachment +category_id: com.canonical.plainbox::info +command: cat /proc/asound/card*/codec#* +estimated_duration: 0.023 +_description: Attaches a report of installed codecs for Intel HDA + +id: cpuinfo_attachment +plugin: attachment +category_id: com.canonical.plainbox::info +command: cat /proc/cpuinfo +estimated_duration: 0.006 +_description: Attaches a report of CPU information +_summary: Attach a copy of /proc/cpuinfo + +id: dmesg_attachment +plugin: attachment +category_id: com.canonical.plainbox::info +user: root +command: + if [ -e /var/log/dmesg ]; then + cat /var/log/dmesg | ansi_parser + else + dmesg | ansi_parser + fi +estimated_duration: 0.640 +_description: Attaches a copy of /var/log/dmesg or the current dmesg buffer to the test results +_summary: Attach a copy of dmesg + +id: dmi_attachment +plugin: attachment +category_id: com.canonical.plainbox::info +command: [ -d /sys/class/dmi/id/ ] && (grep -r . /sys/class/dmi/id/ 2>/dev/null || true) || false +estimated_duration: 0.044 +_description: Attaches info on DMI +_summary: Attach a copy of /sys/class/dmi/id/* +requires: + dmi_present.state == 'supported' + +id: dmidecode_attachment +plugin: attachment +category_id: com.canonical.plainbox::info +requires: + package.name == 'dmidecode' or executable.name == 'dmidecode' + dmi_present.state == 'supported' +user: root +command: dmidecode | iconv -t 'utf-8' -c +estimated_duration: 0.030 +_description: Attaches dmidecode output +_summary: Attach output of dmidecode + +id: lshw_attachment +plugin: attachment +category_id: com.canonical.plainbox::info +requires: package.name == 'lshw' or executable.name == 'lshw' +user: root +command: lshw | iconv -t 'utf-8' -c +_summary: Attach lshw output +_description: Attaches lshw output + +id: efi_attachment +plugin: attachment +category_id: com.canonical.plainbox::info +user: root +command: [ -d /sys/firmware/efi ] && grep -m 1 -o --color=never 'EFI v.*' /var/log/kern.log* || true +estimated_duration: 0.5 +_summary: Attaches firmware version info +_description: Attaches the firmware version + +id: lspci_attachment +plugin: attachment +category_id: com.canonical.plainbox::info +command: + if [[ -v SNAP ]]; then + lspci -i $SNAP/usr/share/misc/pci.ids -vvnn + else + lspci -vvnn | iconv -t 'utf-8' -c + fi +estimated_duration: 0.042 +_description: Attaches very verbose lspci output. +_summary: Attach a list of PCI devices + +id: lspci_network_attachment +plugin: attachment +category_id: com.canonical.plainbox::info +command: lspci -vvnnQ | iconv -t 'utf-8' -c +estimated_duration: 1.322 +_description: Attaches very verbose lspci output (with central database Query). + +id: lspci_standard_config_attachment +_summary: Attach PCI configuration space hex dump +plugin: attachment +category_id: com.canonical.plainbox::info +command: lspci -x +estimated_duration: 0.1 +_description: + Attaches a hex dump of the standard part of the PCI configuration + space for all PCI devices. + +id: lsusb_attachment +plugin: attachment +category_id: com.canonical.plainbox::info +user: root +command: + if [[ -v SNAP ]]; then + lsusb.py -f $SNAP/var/lib/usbutils/usb.ids + else + lsusb -vv | iconv -t 'utf-8' -c + fi +estimated_duration: 0.700 +flags: also-after-suspend +_summary: Attach output of lsusb +_description: Attaches a list of detected USB devices. + +id: meminfo_attachment +plugin: attachment +category_id: com.canonical.plainbox::info +command: cat /proc/meminfo +estimated_duration: 0.043 +_summary: Attach copy of /proc/meminfo +_description: Attaches info on system memory as seen in /proc/meminfo. + +id: modprobe_attachment +plugin: attachment +category_id: com.canonical.plainbox::info +command: find /etc/modprobe.* -name \*.conf | xargs cat +estimated_duration: 0.015 +_description: Attaches the contents of the various modprobe conf files. +_summary: Attach the contents of /etc/modprobe.* + +id: modinfo_attachment +plugin: attachment +category_id: com.canonical.plainbox::info +_summary: Attach modinfo information +command: + for mod in $(lsmod | cut -f 1 -d " ") + do + printf "%-16s%s\n" "name:" "$mod" + modinfo $mod + echo + done +estimated_duration: 1.5 +_description: Attaches modinfo information for all currently loaded modules + +id: modules_attachment +plugin: attachment +category_id: com.canonical.plainbox::info +command: cat /etc/modules +estimated_duration: 0.004 +_description: Attaches the contents of the /etc/modules file. +_summary: Attach the contents of /etc/modules + +id: sysctl_attachment +plugin: attachment +category_id: com.canonical.plainbox::info +command: find /etc/sysctl.* -name \*.conf | xargs cat +estimated_duration: 0.014 +_description: Attaches the contents of various sysctl config files. +_summary: Attach sysctl configuration files. + +id: sysfs_attachment +plugin: attachment +category_id: com.canonical.plainbox::info +_description: Attaches a report of sysfs attributes. +command: + for i in `udevadm info --export-db | sed -n 's/^P: //p'`; do + echo "P: $i" + udevadm info --attribute-walk --path=/sys$i 2>/dev/null | sed -n 's/ ATTR{\(.*\)}=="\(.*\)"/A: \1=\2/p' + echo + done +estimated_duration: 6.344 +_summary: Attach detailed sysfs property output from udev +requires: + model_assertion.model != "dragonboard" + +id: udev_attachment +plugin: attachment +category_id: com.canonical.plainbox::info +command: udevadm info --export-db | xml_sanitize +estimated_duration: 1.465 +_description: Attaches a dump of the udev database showing system hardware information. +_summary: Attach dump of udev database + +id: udev_resource_attachment +plugin: attachment +category_id: com.canonical.plainbox::info +command: udev_resource +estimated_duration: 0.432 +_description: Attaches the output of udev_resource, for debugging purposes + +id: lsblk_attachment +estimated_duration: 0.1 +plugin: attachment +command: lsblk -i -n -P -o KNAME,TYPE,MOUNTPOINT +requires: + package.name == "util-linux" or executable.name == 'lsblk' +_description: Attaches disk block devices mount points +_summary: Attach info block devices and their mount points + +id: gcov_attachment +plugin: attachment +category_id: com.canonical.plainbox::info +requires: package.name == 'lcov' +user: root +command: gcov_tarball +_description: Attaches a tarball of gcov data if present. + +id: lsmod_attachment +plugin: attachment +category_id: com.canonical.plainbox::info +command: lsmod_info +estimated_duration: 0.5 +_description: Attaches a list of the currently running kernel modules. +_summary: Attach a list of currently running kernel modules + +id: dkms_info_attachment +plugin: attachment +category_id: com.canonical.plainbox::info +requires: + package.name == 'dkms' +command: dkms_info --format json +_description: Attaches json dumps of installed dkms package information. +_summary: Attaches json dumps of installed dkms package information. + +plugin: attachment +category_id: com.canonical.plainbox::info +id: acpi_sleep_attachment +command: [ -e /proc/acpi/sleep ] && cat /proc/acpi/sleep || echo "No /proc/acpi/sleep found" +estimated_duration: 0.5 +_description: Attaches the contents of /proc/acpi/sleep if it exists. + +plugin: shell +category_id: com.canonical.plainbox::info +id: info/bootchart +_description: Bootchart information. +requires: package.name == 'bootchart' or package.name == 'pybootchartgui' +user: root +command: + process_wait -u root bootchart collector ureadahead; \ + [ `ls /var/log/bootchart/*.tgz 2>/dev/null | wc -l` -lt 2 ] && reboot && sleep 100 + +unit: template +template-resource: device +template-filter: device.category == 'DISK' +plugin: attachment +category_id: com.canonical.plainbox::info +id: info/hdparm_{name}.txt +estimated_duration: 1.0 +requires: + package.name == 'hdparm' or executable.name == 'hdparm' + block_device.state != 'removable' and block_device.name == '{name}' +user: root +command: hdparm -I /dev/{name} +_summary: Attaches info from hdparm about {name} + +plugin: attachment +category_id: com.canonical.plainbox::info +id: bootchart.png +depends: info/bootchart +requires: package.name == 'pybootchartgui' +_description: Attaches the bootchart png file for bootchart runs +command: + file=`ls /var/log/bootchart/*.png 2>/dev/null | tail -1`; \ + [ -e "$file" ] && cat "$file" + +plugin: attachment +category_id: com.canonical.plainbox::info +id: bootchart.tgz +depends: info/bootchart +_description: Attaches the bootchart log for bootchart test runs. +command: + file=`ls /var/log/bootchart/*.tgz 2>/dev/null | tail -1`; \ + [ -e "$file" ] && cat "$file" + +plugin: attachment +category_id: com.canonical.plainbox::info +id: installer_bootchart.tgz +command: [ -f /var/log/installer/bootchart.tgz ] && base64 /var/log/installer/bootchart.tgz +_description: Attach the installer bootchart tarball if it exists. + +plugin: attachment +category_id: com.canonical.plainbox::info +id: installer_debug.gz +user: root +command: [ -f /var/log/installer/debug ] && gzip -9 -c /var/log/installer/debug | base64 +estimated_duration: 0.1 +_description: Attaches the installer debug log if it exists. + +plugin: attachment +category_id: com.canonical.plainbox::info +id: info/touchpad_driver +requires: device.category == 'TOUCHPAD' +command: touchpad_driver_info +estimated_duration: 0.384 +_description: + Returns the name, driver name and driver version of any touchpad discovered on + the system. + +plugin: attachment +category_id: com.canonical.plainbox::info +id: info/audio_device_driver +requires: + package.name == 'pulseaudio-utils' + package.name == 'kmod' or package.name == 'module-init-tools' + device.category == 'AUDIO' +command: audio_driver_info +estimated_duration: 0.177 +_description: Lists the device driver and version for all audio devices. + +plugin: attachment +category_id: com.canonical.plainbox::info +id: info/network_devices +command: network_device_info +estimated_duration: 0.550 +_description: Provides information about network devices + +plugin: attachment +category_id: com.canonical.plainbox::info +id: info/mobile_broadband_info +command: broadband_info +estimated_duration: 0.5 +_description: Provides information about wwan/mobile broadband devices + +plugin: attachment +category_id: com.canonical.plainbox::info +id: info/xrandr +command: xrandr -q --verbose +_description: Provides information about displays attached to the system + +plugin: attachment +category_id: com.canonical.plainbox::info +id: info/disk_partitions +estimated_duration: 1.0 +user: root +command: parted -l -s +_summary: Attaches info about disk partitions +_description: Attaches information about disk partitions + +plugin: attachment +category_id: com.canonical.plainbox::info +id: info/buildstamp +estimated_duration: 0.1 +_description: Attaches the buildstamp identifier for the preinstalled OS +_summary: Attach /etc/buildstamp +command: [ -s /etc/buildstamp ] && cat /etc/buildstamp + +plugin: shell +category_id: com.canonical.plainbox::info +id: info/detect_recovery +requires: package.name == 'dell-recovery' or package.name == 'ubuntu-recovery' +user: root +estimated_duration: 0.2 +_summary: Check existence of recovery partition +_description: Check existence of recovery partition +command: recovery_info + +plugin: shell +category_id: com.canonical.plainbox::info +id: info/detect_dell_recovery +depends: info/detect_recovery +requires: package.name == 'dell-recovery' +user: root +estimated_duration: 0.2 +_summary: Check the recovery type is dell or not +_description: Check the recovery type is dell or not +command: recovery_info checktype DELL + +plugin: attachment +category_id: com.canonical.plainbox::info +id: recovery_info_attachment +depends: info/detect_recovery +requires: package.name == 'dell-recovery' or package.name == 'ubuntu-recovery' +user: root +estimated_duration: 0.2 +_summary: Attach the recovery partition versions +_description: + Attach the recovery partition version + image_version is the preinstalled OS image version + bto_version is only for dell_recovery + Example: + image_version: somerville-trusty-amd64-20140620-0 + bto_version: A00_dell-bto-trusty-houston-15-A11-iso-20141203-0.iso +command: recovery_info version + +plugin: attachment +category_id: com.canonical.plainbox::info +id: dell_bto_xml_attachment +depends: info/detect_dell_recovery +requires: package.name == 'dell-recovery' +user: root +estimated_duration: 0.5 +_summary: Attach the bto.xml in Dell recovery partition +_description: + bto.xml is basically a Fish manifest + The information include: + - fish packages + - dell recovery stage 2 boot log +command: recovery_info file bto.xml + +plugin: attachment +category_id: com.canonical.plainbox::info +id: info/kvm_output +estimated_duration: 0.1 +_summary: + Attaches console log from the kvm_check_vm test +_description: + Attaches the debug log from the virtualization/kvm_check_vm test + to the results submission. +command: [ -f $PLAINBOX_SESSION_SHARE/virt_debug ] && cat $PLAINBOX_SESSION_SHARE/virt_debug + +id: kernel_cmdline_attachment +plugin: attachment +category_id: com.canonical.plainbox::info +command: cat /proc/cmdline +estimated_duration: 0.005 +_description: Attaches the kernel command line used to boot +_summary: Attach a copy of /proc/cmdline diff --git a/units/info/packaging.pxu b/units/info/packaging.pxu new file mode 100644 index 0000000..26d76cb --- /dev/null +++ b/units/info/packaging.pxu @@ -0,0 +1,14 @@ +# The dkms_info script requires python3-debian package +unit: packaging meta-data +os-id: debian +Depends: python3-debian + +# The dkms_info script requires python3-guacamole package +unit: packaging meta-data +os-id: debian +Depends: python3-guacamole (>= 0.9) + +# This is for lsblk attachment and disk/detect +unit: packaging meta-data +os-id: debian +Depends: util-linux diff --git a/units/hwsubmit.pxu b/units/info/test-plan.pxu index b49f5fe..c01d47e 100644 --- a/units/hwsubmit.pxu +++ b/units/info/test-plan.pxu @@ -1,35 +1,32 @@ -id: hwsubmit -_name: All HW Information Submit Tests +id: info-attachment-cert-full unit: test plan +_name: Info attachment jobs +_description: Info attachment jobs include: - cdimage - cpuinfo - device - dmi - dpkg - gconf - lsb - meminfo - module - package - uname - usb - info/buildstamp + acpi_sleep_attachment + codecs_attachment cpuinfo_attachment + dkms_info_attachment dmesg_attachment dmi_attachment dmidecode_attachment + efi_attachment + info/buildstamp + info/disk_partitions + info/hdparm_.*.txt + info/touchpad_driver + installer_debug.gz kernel_cmdline_attachment - recovery_info_attachment - dell_bto_xml_attachment lsmod_attachment - dkms_info_attachment lspci_attachment lspci_standard_config_attachment + lsusb_attachment + meminfo_attachment modinfo_attachment modprobe_attachment modules_attachment - sysfs_attachment sysctl_attachment + sysfs_attachment udev_attachment - +bootstrap_include: + device diff --git a/units/input/jobs.pxu b/units/input/jobs.pxu new file mode 100644 index 0000000..0bf880d --- /dev/null +++ b/units/input/jobs.pxu @@ -0,0 +1,77 @@ +unit: template +template-resource: device +template-filter: device.category == 'MOUSE' or device.category == 'TOUCHPAD' or device.category == 'TOUCHSCREEN' +plugin: manual +category_id: com.canonical.plainbox::input +id: input/pointing_{product_slug}_{category}_{__index__} +estimated_duration: 30.0 +_summary: Check pointing functionality for {product} +_purpose: + This will test your {product} device +_steps: + 1. Move the cursor with your {product}. +_verification: + Did the cursor move? + +plugin: manual +category_id: com.canonical.plainbox::input +id: input/mouse +estimated_duration: 30.0 +_description: + PURPOSE: + This test will test your pointing device + STEPS: + 1. Move the cursor using the pointing device or touch the screen. + 2. Perform some single/double/right click operations. + VERIFICATION: + Did the pointing device work as expected? + +plugin: user-interact-verify +category_id: com.canonical.plainbox::input +id: input/keyboard +estimated_duration: 30.0 +command: keyboard_test +requires: device.category == 'KEYBOARD' +_description: + PURPOSE: + This test will test your keyboard + STEPS: + 1. Click on Test + 2. On the open text area, use your keyboard to type something + VERIFICATION: + Is your keyboard working properly? + +plugin: user-interact +category_id: com.canonical.plainbox::input +id: input/accelerometer +estimated_duration: 60.0 +user: root +requires: device.category == 'ACCELEROMETER' +command: accelerometer_test -m +_description: + PURPOSE: + This test will test your accelerometer to see if it is detected + and operational as a joystick device. + STEPS: + 1. Click on Test + 2. Tilt your hardware in the directions onscreen until the axis threshold is met. + VERIFICATION: + Is your accelerometer properly detected? Can you use the device? + +unit: template +template-resource: device +template-filter: device.category == 'MOUSE' or device.category == 'TOUCHPAD' +plugin: manual +category_id: com.canonical.plainbox::input +id: input/clicking_{product_slug}_{category}_{__index__} +estimated_duration: 30.0 +_summary: Check button functionality for {product} +_purpose: + This will test the buttons of your {product} device +_steps: + 1. Click the left button with your {product}. + 2. Click the right button with your {product}. + 3. Click the middle button with your {product} (if available). + 4. Double-click the left button with your {product}. +_verification: + Did these buttons work as expected? diff --git a/units/input/test-plan.pxu b/units/input/test-plan.pxu new file mode 100644 index 0000000..8033883 --- /dev/null +++ b/units/input/test-plan.pxu @@ -0,0 +1,34 @@ +id: input-cert-full +unit: test plan +_name: Input tests +_description: Input tests +include: + input/accelerometer certification-status=non-blocker + input/pointing_.* certification-status=blocker + input/clicking_.* certification-status=blocker + input/keyboard certification-status=blocker + +id: after-suspend-input-cert-full +unit: test plan +_name: Input tests (after suspend) +_description: Input tests (after suspend) +include: + suspend/pointing-after-suspend_.* certification-status=blocker + suspend/clicking-after-suspend_.* certification-status=blocker + +id: input-cert-blockers +unit: test plan +_name: Input tests (certification blockers only) +_description: Input tests (certification blockers only) +include: + input/pointing_.* certification-status=blocker + input/clicking_.* certification-status=blocker + input/keyboard certification-status=blocker + +id: after-suspend-input-cert-blockers +unit: test plan +_name: Input tests (after suspend, certification blockers only) +_description: Input tests (after suspend, certification blockers only) +include: + suspend/pointing-after-suspend_.* certification-status=blocker + suspend/clicking-after-suspend_.* certification-status=blocker diff --git a/units/install/jobs.pxu b/units/install/jobs.pxu new file mode 100644 index 0000000..ae22103 --- /dev/null +++ b/units/install/jobs.pxu @@ -0,0 +1,10 @@ +plugin: shell +id: install/apt-get-gets-updates +requires: package.name == 'apt' +user: root +command: apt-get -d -y --force-yes dist-upgrade +_description: + Tests to see that apt can access repositories and get updates (does + not install updates). This is done to confirm that you could recover + from an incomplete or broken update. + diff --git a/units/keys/jobs.pxu b/units/keys/jobs.pxu new file mode 100644 index 0000000..891281f --- /dev/null +++ b/units/keys/jobs.pxu @@ -0,0 +1,266 @@ +plugin: user-interact +category_id: com.canonical.plainbox::keys +id: keys/lock-screen +estimated_duration: 30.0 +imports: from com.canonical.plainbox import manifest +requires: manifest.has_special_keys == 'True' +command: lock_screen_watcher +_description: + PURPOSE: + This test will test the screen lock key + STEPS: + 1. Commence the test. If there is no such key, please skip this test. + 2. Press the lock screen button on the keyboard in 30 seconds. + 3. If the screen is locked, move the mouse or press any key to activate the prompt. + 4. Input the password to unlock the screen. + VERIFICATION: + The verification of this test is automated. Do not change the + automatically selected result. + +plugin: manual +category_id: com.canonical.plainbox::keys +id: keys/brightness +estimated_duration: 30.0 +imports: from com.canonical.plainbox import manifest +requires: manifest.has_special_keys == 'True' +_description: + PURPOSE: + This test will test the brightness key + STEPS: + 1. Press the brightness buttons on the keyboard + VERIFICATION: + Did the brightness change following to your key presses? + +plugin: user-interact-verify +category_id: com.canonical.plainbox::keys +id: keys/volume +estimated_duration: 30.0 +imports: from com.canonical.plainbox import manifest +requires: manifest.has_special_keys == 'True' +user: root +command: key_test -s '0xe02e,0xe0ae:Volume Down' '0xe030,0xe0b0:Volume Up' +_description: + PURPOSE: + This test will test the volume keys of your keyboard + STEPS: + Skip this test if your computer has no volume keys. + 1. Click test to open a window on which to test the volume keys. + 2. If all the keys work, the test will be marked as passed. + VERIFICATION: + Do the keys work as expected? + +plugin: user-interact-verify +category_id: com.canonical.plainbox::keys +id: keys/mute +estimated_duration: 30.0 +imports: from com.canonical.plainbox import manifest +requires: manifest.has_special_keys == 'True' +user: root +command: key_test -s '0xe020,0xe0a0:Mute' +_description: + PURPOSE: + This test will test the mute key of your keyboard + STEPS: + 1. Click test to open a window on which to test the mute key. + 2. If the key works, the test will pass and the window will close. + VERIFICATION: + Does the mute key work as expected? + +plugin: manual +category_id: com.canonical.plainbox::keys +id: keys/sleep +estimated_duration: 90.0 +imports: from com.canonical.plainbox import manifest +requires: manifest.has_special_keys == 'True' +_description: + PURPOSE: + This test will test the sleep key + STEPS: + 1. Press the sleep key on the keyboard + 2. Wake your system up by pressing the power button + VERIFICATION: + Did the system go to sleep after pressing the sleep key? + +plugin: user-interact-verify +category_id: com.canonical.plainbox::keys +id: keys/battery-info +estimated_duration: 30.0 +imports: from com.canonical.plainbox import manifest +requires: manifest.has_special_keys == 'True' +user: root +command: key_test -s '0xe071,0xef1:Battery Info' +_description: + PURPOSE: + This test will test the battery information key + STEPS: + Skip this test if you do not have a Battery Button. + 1. Click Test to begin + 2. Press the Battery Info button (or combo like Fn+F3) + 3: Close the Power Statistics tool if it opens + VERIFICATION: + Did the Battery Info key work as expected? + +plugin: manual +category_id: com.canonical.plainbox::keys +id: keys/wireless +estimated_duration: 90.0 +imports: from com.canonical.plainbox import manifest +requires: manifest.has_special_keys == 'True' +_description: + PURPOSE: + This test will test the wireless key + STEPS: + 1. Press the wireless key on the keyboard + 2. Check that the wifi LED turns off or changes color + 3. Check that wireless is disabled + 4. Press the same key again + 5. Check that the wifi LED turns on or changes color + 6. Check that wireless is enabled + VERIFICATION: + Did the wireless turn off on the first press and on again on the second? + (NOTE: the LED functionality will be reviewed in a following test. Please + only consider the functionality of the wifi itself here.) + +plugin: user-interact +category_id: com.canonical.plainbox::keys +id: keys/media-control +estimated_duration: 30.0 +imports: from com.canonical.plainbox import manifest +requires: manifest.has_special_keys == 'True' +user: root +command: key_test -s 0xe010,0xe090:Previous 0xe024,0xe0a4:Stop 0xe019,0xe099:Next 0xe022,0xe0a2:Play +_description: + PURPOSE: + This test will test the media keys of your keyboard + STEPS: + Skip this test if your computer has no media keys. + 1. Click test to open a window on which to test the media keys. + 2. If all the keys work, the test will be marked as passed. + VERIFICATION: + Do the keys work as expected? + +plugin: user-interact +category_id: com.canonical.plainbox::keys +id: keys/super +estimated_duration: 30.0 +imports: from com.canonical.plainbox import manifest +requires: manifest.has_special_keys == 'True' +user: root +command: key_test -s '0xe05b,0xe0db:Left Super Key' +_description: + PURPOSE: + This test will test the super key of your keyboard + STEPS: + 1. Click test to open a window on which to test the super key. + 2. If the key works, the test will pass and the window will close. + VERIFICATION: + Does the super key work as expected? + +plugin: manual +category_id: com.canonical.plainbox::keys +id: keys/video-out +estimated_duration: 120.0 +imports: from com.canonical.plainbox import manifest +requires: manifest.has_special_keys == 'True' +_description: + PURPOSE: + Validate that the External Video hot key is working as expected + STEPS: + 1. Plug in an external monitor + 2. Press the display hot key to change the monitors configuration + VERIFICATION: + Check that the video signal can be mirrored, extended, displayed on external or onboard only. + +plugin: manual +category_id: com.canonical.plainbox::keys +id: keys/touchpad +estimated_duration: 30.0 +imports: from com.canonical.plainbox import manifest +requires: manifest.has_special_keys == 'True' +_description: + PURPOSE: + Verify touchpad hotkey toggles touchpad functionality on and off + STEPS: + 1. Verify the touchpad is functional + 2. Tap the touchpad toggle hotkey + 3. Tap the touchpad toggle hotkey again + VERIFICATION: + Verify the touchpad has been disabled and re-enabled. + +plugin: manual +category_id: com.canonical.plainbox::keys +id: keys/keyboard-backlight +estimated_duration: 30.0 +imports: from com.canonical.plainbox import manifest +requires: manifest.has_special_keys == 'True' +_description: + PURPOSE: + Verify that the keyboard backlight toggle key works properly + STEPS: + 1. Tap the keyboard backlight key + 2. Confirm that the keyboard backlight was toggled to the opposite state + 3. Tap the keyboard backlight key again + 4. Confirm that the keyboard backlight was toggled to the opposite state + VERIFICATION: + Did the keyboard backlight state change on each press? + +plugin: user-interact-verify +category_id: com.canonical.plainbox::keys +id: keys/microphone-mute +estimated_duration: 60.0 +imports: from com.canonical.plainbox import manifest +requires: + manifest.has_special_keys == 'True' + device.category == 'AUDIO' + package.name == 'alsa-base' + package.name == 'pulseaudio-utils' + package.name == 'gstreamer1.0-plugins-good' or package.name == 'gstreamer0.10-plugins-good' +command: + audio_settings store --file=$PLAINBOX_SESSION_SHARE/pulseaudio_settings + audio_settings set --device=pci --volume=50 + alsa_record_playback + EXIT_CODE=$? + audio_settings restore --file=$PLAINBOX_SESSION_SHARE/pulseaudio_settings + exit $EXIT_CODE +_description: + PURPOSE: + This test will test the mute key for your microphone + STEPS: + 1. Click "Test" then speak: "Imagination is more important than knowledge" (or anything else) into your microphone. + 2. While you are speaking, please press the mute key for the microphone to mute it and press it again to unmute. + 3. After a few seconds, your speech will be played back to you. If the key works, your speech should be interrupted for a few seconds. + VERIFICATION: + Does the microphone mute key work as expected? + +plugin: manual +category_id: com.canonical.plainbox::keys +id: keys/hibernate +estimated_duration: 120.0 +imports: from com.canonical.plainbox import manifest +requires: manifest.has_special_keys == 'True' +_description: + PURPOSE: + This test will test the hibernate key + STEPS: + 1. Press the hibernate key on the keyboard + 2. Check that the system hibernated correctly + 3. Wake your system after hibernating by pressing the power button + VERIFICATION: + Did the system go to hibernate after pressing the hibernate key? + +plugin: manual +category_id: com.canonical.plainbox::keys +id: keys/keyboard-overhead-light +estimated_duration: 30.0 +imports: from com.canonical.plainbox import manifest +requires: manifest.has_special_keys == 'True' +_description: + PURPOSE: + This test will test the keyboard overhead light key or switch + STEPS: + 1. Press the keyboard overhead light key or swtich on the light + 2. Check the the keyboard overhead light turn on correctly + 3. Press the key or switch again to toggle off the light + VERIFICATION: + Did the keyboard overhead light key or switch turns on and off the light? + diff --git a/units/keys/manifest.pxu b/units/keys/manifest.pxu new file mode 100644 index 0000000..2cf755c --- /dev/null +++ b/units/keys/manifest.pxu @@ -0,0 +1,4 @@ +unit: manifest entry +id: has_special_keys +_name: Special keys Support +value-type: bool diff --git a/units/keys/test-plan.pxu b/units/keys/test-plan.pxu new file mode 100644 index 0000000..ebd1696 --- /dev/null +++ b/units/keys/test-plan.pxu @@ -0,0 +1,32 @@ +id: keys-cert-full +unit: test plan +_name: Special keys tests +_description: + Special keys tests (w/o sleep/hibernate keys, see Power Management test plans) +include: + keys/lock-screen certification-status=blocker + keys/super certification-status=blocker + keys/battery-info certification-status=blocker + keys/brightness certification-status=blocker + keys/media-control certification-status=blocker + keys/mute certification-status=blocker + keys/volume certification-status=blocker + keys/video-out certification-status=blocker + keys/wireless certification-status=blocker + +id: keys-cert-blockers +unit: test plan +_name: Special keys tests (certification blockers only) +_description: + Special keys tests (w/o sleep/hibernate keys, see Power Management test plans) + (certification blockers only) +include: + keys/lock-screen certification-status=blocker + keys/super certification-status=blocker + keys/battery-info certification-status=blocker + keys/brightness certification-status=blocker + keys/media-control certification-status=blocker + keys/mute certification-status=blocker + keys/volume certification-status=blocker + keys/video-out certification-status=blocker + keys/wireless certification-status=blocker diff --git a/units/led/jobs.pxu b/units/led/jobs.pxu new file mode 100644 index 0000000..17ce655 --- /dev/null +++ b/units/led/jobs.pxu @@ -0,0 +1,249 @@ +id: led/power +_summary: Power LED behavior when powered +_purpose: + Check power led is on when system is powered on +_steps: + 1. Check power led when system is powered on +_verification: + Power led is on when system is powered on +plugin: manual +category_id: com.canonical.plainbox::led +estimated_duration: 15.0 + +id: led/power-blink-suspend +_summary: Power LED behavior when suspended +_purpose: + Check power led is blinking when system is in suspend +_steps: + 1. Suspend the system + 2. Check power led when system is in suspend +_verification: + Power led is blinking when system is in suspend +plugin: manual +category_id: com.canonical.plainbox::led +estimated_duration: 180 + +plugin: manual +category_id: com.canonical.plainbox::led +id: led/suspend +estimated_duration: 30.0 +depends: suspend/suspend_advanced +_description: + PURPOSE: + Suspend LED verification. + STEPS: + Skip this test if your system does not have a dedicated Suspend LED. + 1. The Suspend LED should blink or change color while the system is + suspended + VERIFICATION + Did the Suspend LED blink or change color while the system was suspended? + +plugin: manual +category_id: com.canonical.plainbox::led +id: led/battery-charging +estimated_duration: 120.0 +_description: + PURPOSE: + Validate that the battery light shows charging status + STEPS: + 1. Let system run on battery for a while + 2. Plug in AC plug + VERIFICATION: + Did the battery indicator LED turn orange? + +plugin: manual +category_id: com.canonical.plainbox::led +id: led/battery-charged +estimated_duration: 240.0 +_description: + PURPOSE: + Validate that the battery LED properly displays charged status + STEPS: + 1. Let system run on battery for a short time + 2. Plug in AC + 3. Let system run on AC + VERIFICATION: + Does the orange battery LED shut off when system is fully charged? + +plugin: manual +category_id: com.canonical.plainbox::led +id: led/battery-low +estimated_duration: 30.0 +_description: + PURPOSE: + Validate that the battery LED indicated low power + STEPS: + 1. Let system run on battery for several hours + 2. Monitor battery LED carefully + VERIFICATION: + Does the LED light orange when battery is low? + +plugin: user-interact-verify +category_id: com.canonical.plainbox::led +id: led/hdd +estimated_duration: 30.0 +command: led_hdd_test +_description: + PURPOSE: + HDD LED verification + STEPS: + 1. Select "Test" to write and read a temporary file for a few seconds + 2. HDD LED should light when writing to/reading from HDD + VERIFICATION: + Did the HDD LED light? + +plugin: user-interact-verify +category_id: com.canonical.plainbox::led +id: led/numeric-keypad +estimated_duration: 30.0 +command: keyboard_test +_description: + PURPOSE: + Numeric keypad LED verification + STEPS: + 1. Press "Block Num" key to toggle numeric keypad LED + 2. Click on the "Test" button to open a window to verify your typing + 3. Type using the numeric keypad both when the LED is on and off + VERIFICATION: + 1. Numeric keypad LED status should toggle everytime the "Block Num" key is pressed + 2. Numbers should only be entered in the keyboard verification window when the LED is on + +plugin: manual +category_id: com.canonical.plainbox::led +id: led/caps-lock +estimated_duration: 30.0 +_description: + PURPOSE: + Block cap keys LED verification + STEPS: + 1. Press "Block Cap Keys" to activate/deactivate cap keys blocking + 2. Cap Keys LED should be switched on/off every time the key is pressed + VERIFICATION: + Did the Cap Keys LED light as expected? + +plugin: manual +category_id: com.canonical.plainbox::led +id: led/wlan +depends: keys/wireless +estimated_duration: 60.0 +_description: + PURPOSE: + WLAN LED verification + STEPS: + 1. During the keys/wireless test you should have observed the + wireless LED while turning wireless back on. + 2. WLAN LED should light or change color when wireless is turned on + VERIFICATION: + Did the WLAN LED turn on or change color as expected? + +plugin: manual +category_id: com.canonical.plainbox::led +id: led/wlan-disabled +estimated_duration: 30.0 +depends: keys/wireless +_description: + PURPOSE: + Validate that WLAN LED shuts off when disabled + STEPS: + 1. During the keys/wireless test you should have observed the WLAN LED + while performing that test after turning wireless off. + 2. WLAN LED should turn off or change color when wireless is turned off + VERIFICATION: + Did the WLAN LED turn off or change color as expected? + +id: led/bluetooth +_summary: Bluetooth LED behavior +_purpose: + Check bluetooth LED behavior is correct +_steps: + 1. Run following command to start bluetoothctl console: + sudo bluetoothctl -a + ***Following steps are run in bluetoothctl console*** + 2. Power on the device: + power on + 3. Power off the device: + power off + 4. Quit bluetoothctl console + quit +_verification: + Bluetooth LED is turned on in step 2, and turned off in step 3. +plugin: manual +category_id: com.canonical.plainbox::led +estimated_duration: 180 + +plugin: user-interact-verify +category_id: com.canonical.plainbox::led +id: led/camera +estimated_duration: 20.0 +depends: camera/detect +command: camera_test led +_description: + PURPOSE: + Camera LED verification + STEPS: + 1. Select Test to activate camera + 2. Camera LED should light for a few seconds + VERIFICATION: + Did the camera LED light? + +plugin: manual +category_id: com.canonical.plainbox::led +id: led/touchpad +estimated_duration: 30.0 +_description: + PURPOSE: + Touchpad LED verification + STEPS: + 1. Click on the touchpad button or press key combination to enable/disable touchpad button + 2. Slide your finger on the touchpad + VERIFICATION: + 1. Touchpad LED status should toggle everytime the button is clicked or the key combination is pressed + 2. When the LED is on, the mouse pointer should move on touchpad usage + 3. When the LED is off, the mouse pointer should not move on touchpad usage + +plugin: manual +category_id: com.canonical.plainbox::led +id: led/wireless +estimated_duration: 120.0 +_description: + PURPOSE: + Wireless (WLAN + Bluetooth) LED verification + STEPS: + 1. Make sure WLAN connection is established and Bluetooth is enabled. + 2. WLAN/Bluetooth LED should light + 3. Switch WLAN and Bluetooth off from a hardware switch (if present) + 4. Switch them back on + 5. Switch WLAN and Bluetooth off from the panel applet + 6. Switch them back on + VERIFICATION: + Did the WLAN/Bluetooth LED light as expected? + +plugin: manual +category_id: com.canonical.plainbox::led +id: led/mute +depends: keys/mute +estimated_duration: 30.0 +_description: + PURPOSE: + Audio Mute LED verification. + STEPS: + Skip this test if your system does not have a special Audio Mute LED. + 1. Press the Mute key twice and observe the Audio LED to determine if it + either turned off and on or changed colors. + VERIFICATION: + Did the Audio LED turn on and off or change color as expected? + +plugin: manual +category_id: com.canonical.plainbox::led +id: led/microphone-mute +estimated_duration: 30.0 +depends: keys/microphone-mute +_description: + PURPOSE: + Microphone Mute LED verification. + STEPS: + Skip this test if your system does not have a special Microphone Mute LED. + 1. Press the Microphone Mute key twice and observe the Microphone LED to + determine if it either turned off and on or changed colors. + VERIFICATION: + Did the Microphone Mute LED turn on and off or change color as expected? diff --git a/units/led/test-plan.pxu b/units/led/test-plan.pxu new file mode 100644 index 0000000..4f59af9 --- /dev/null +++ b/units/led/test-plan.pxu @@ -0,0 +1,30 @@ +id: led-cert-full +unit: test plan +_name: LED tests +_description: + LED tests + Notes: - led/power-blink-suspend and led/suspend are used later in the + power management testplan just after a number of suspend tests. + - led/wireless is redundant given that we have led/wlan and + led/bluetooth already. +include: + led/battery-charged + led/battery-charging + led/battery-low + led/camera + led/caps-lock + led/power + led/touchpad + +id: after-suspend-led-cert-full +unit: test plan +_name: LED tests (after suspend) +_description: LED tests (after suspend) +include: + suspend/led_after_suspend/battery-charged + suspend/led_after_suspend/battery-charging + suspend/led_after_suspend/battery-low + suspend/led_after_suspend/camera + suspend/led_after_suspend/caps-lock + suspend/led_after_suspend/power + suspend/led_after_suspend/touchpad diff --git a/units/mediacard/jobs.pxu b/units/mediacard/jobs.pxu new file mode 100644 index 0000000..ab00396 --- /dev/null +++ b/units/mediacard/jobs.pxu @@ -0,0 +1,677 @@ +plugin: user-interact +category_id: com.canonical.plainbox::mediacard +id: mediacard/mmc-insert +estimated_duration: 30.0 +command: + if [[ -v SNAP ]]; then + timeout 20s bash -c 'grep -m 1 -oP "\Kadd.*?mmc.*?\s" <( exec journalctl -u snap.udisks2.udisksd -f -q -S -1s ); kill $!' + else + removable_storage_watcher --memorycard insert sdio usb scsi + fi +imports: from com.canonical.plainbox import manifest +requires: + package.name == 'udisks2' or snap.name == 'udisks2' + manifest.has_card_reader == 'True' +user: root +_description: + PURPOSE: + This test will check that the systems media card reader can + detect the insertion of a Multimedia Card (MMC) media + STEPS: + 1. Commence the test and then insert an MMC card into the reader. + (Note: this test will time-out after 20 seconds.) + 2. Do not remove the device after this test. + VERIFICATION: + The verification of this test is automated. Do not change the + automatically selected result. + +plugin: shell +category_id: com.canonical.plainbox::mediacard +id: mediacard/mmc-storage +estimated_duration: 30.0 +depends: mediacard/mmc-insert +user: root +flags: preserve-cwd +command: removable_storage_test -s 67120000 --memorycard sdio usb scsi --auto-reduce-size +requires: + package.name == 'udisks2' or (snap.name == 'core' and int(snap.revision) >= 1804) +_description: + This test is automated and executes after the mediacard/mmc-insert + test is run. It tests reading and writing to the MMC card. + +plugin: user-interact +category_id: com.canonical.plainbox::mediacard +id: mediacard/mmc-remove +estimated_duration: 30.0 +depends: mediacard/mmc-insert +command: + if [[ -v SNAP ]]; then + timeout 20s bash -c 'grep -m 1 -oP "\Kremove.*?mmc.*?\s" <( exec journalctl -u snap.udisks2.udisksd -f -q -S -1s ); kill $!' + else + removable_storage_watcher --memorycard remove sdio usb scsi + fi +user: root +_description: + PURPOSE: + This test will check that the system correctly detects + the removal of the MMC card from the systems card reader. + STEPS: + 1. Commence the test and then remove the MMC card from the reader. + (Note: this test will time-out after 20 seconds.) + VERIFICATION: + The verification of this test is automated. Do not change the + automatically selected result. + +plugin: user-interact +category_id: com.canonical.plainbox::mediacard +id: mediacard/sd-insert +estimated_duration: 30.0 +command: + if [[ -v SNAP ]]; then + timeout 20s bash -c 'grep -m 1 -oP "\Kadd.*?mmc.*?\s" <( exec journalctl -u snap.udisks2.udisksd -f -q -S -1s ); kill $!' + else + removable_storage_watcher --memorycard insert sdio usb scsi + fi +imports: from com.canonical.plainbox import manifest +requires: + package.name == 'udisks2' or snap.name == 'udisks2' + manifest.has_card_reader == 'True' +user: root +_description: + PURPOSE: + This test will check that the systems media card reader can + detect the insertion of an UNLOCKED Secure Digital (SD) media card + STEPS: + 1. Commence the test and then insert an UNLOCKED SD card into the reader. + (Note: this test will time-out after 20 seconds.) + 2. Do not remove the device after this test. + VERIFICATION: + The verification of this test is automated. Do not change the + automatically selected result. + +plugin: shell +category_id: com.canonical.plainbox::mediacard +id: mediacard/sd-storage +estimated_duration: 30.0 +depends: mediacard/sd-insert +user: root +flags: preserve-cwd +command: removable_storage_test -s 268400000 --memorycard sdio usb scsi +requires: + package.name == 'udisks2' or (snap.name == 'core' and int(snap.revision) >= 1804) +_description: + This test is automated and executes after the mediacard/sd-insert + test is run. It tests reading and writing to the SD card. + +plugin: user-interact +category_id: com.canonical.plainbox::mediacard +id: mediacard/sd-remove +estimated_duration: 30.0 +depends: mediacard/sd-insert +command: + if [[ -v SNAP ]]; then + timeout 20s bash -c 'grep -m 1 -oP "\Kremove.*?mmc.*?\s" <( exec journalctl -u snap.udisks2.udisksd -f -q -S -1s ); kill $!' + else + removable_storage_watcher --memorycard remove sdio usb scsi + fi +user: root +_description: + PURPOSE: + This test will check that the system correctly detects + the removal of an SD card from the systems card reader. + STEPS: + 1. Commence the test and then remove the SD card from the reader. + (Note: this test will time-out after 20 seconds.) + VERIFICATION: + The verification of this test is automated. Do not change the + automatically selected result. + +plugin: shell +category_id: com.canonical.plainbox::mediacard +id: mediacard/sd-preinserted +estimated_duration: 30.0 +user: root +flags: preserve-cwd +command: removable_storage_test -s 268400000 --memorycard -l sdio usb scsi && removable_storage_test --memorycard sdio usb scsi +imports: from com.canonical.plainbox import manifest +requires: + package.name == 'udisks2' or snap.name == 'udisks2' + package.name == 'udisks2' or (snap.name == 'core' and int(snap.revision) >= 1804) + manifest.has_card_reader == 'True' +_description: + This is a fully automated version of mediacard/sd-automated and assumes that the + system under test has a memory card device plugged in prior to checkbox execution. + It is intended for SRU automated testing. + +plugin: user-interact +category_id: com.canonical.plainbox::mediacard +id: mediacard/sdhc-insert +estimated_duration: 30.0 +command: + if [[ -v SNAP ]]; then + timeout 20s bash -c 'grep -m 1 -oP "\Kadd.*?mmc.*?\s" <( exec journalctl -u snap.udisks2.udisksd -f -q -S -1s ); kill $!' + else + removable_storage_watcher --memorycard insert sdio usb scsi + fi +imports: from com.canonical.plainbox import manifest +requires: + package.name == 'udisks2' or snap.name == 'udisks2' + manifest.has_card_reader == 'True' +user: root +_description: + PURPOSE: + This test will check that the systems media card reader can + detect the insertion of a UNLOCKED Secure Digital High-Capacity + (SDHC) media card + STEPS: + 1. Commence the test and then insert an UNLOCKED SDHC card into the reader. + (Note: this test will time-out after 20 seconds.) + 2. Do not remove the device after this test. + VERIFICATION: + The verification of this test is automated. Do not change the + automatically selected result. + +plugin: shell +category_id: com.canonical.plainbox::mediacard +id: mediacard/sdhc-storage +estimated_duration: 30.0 +depends: mediacard/sdhc-insert +user: root +flags: preserve-cwd +command: removable_storage_test -s 268400000 --memorycard sdio usb scsi +requires: + package.name == 'udisks2' or (snap.name == 'core' and int(snap.revision) >= 1804) +_description: + This test is automated and executes after the mediacard/sdhc-insert + test is run. It tests reading and writing to the SDHC card. + +plugin: user-interact +category_id: com.canonical.plainbox::mediacard +id: mediacard/sdhc-remove +estimated_duration: 30.0 +depends: mediacard/sdhc-insert +command: + if [[ -v SNAP ]]; then + timeout 20s bash -c 'grep -m 1 -oP "\Kremove.*?mmc.*?\s" <( exec journalctl -u snap.udisks2.udisksd -f -q -S -1s ); kill $!' + else + removable_storage_watcher --memorycard remove sdio usb scsi + fi +user: root +_description: + PURPOSE: + This test will check that the system correctly detects + the removal of an SDHC card from the systems card reader. + STEPS: + 1. Commence the test and then remove the SDHC card from the reader. + (Note: this test will time-out after 20 seconds.) + VERIFICATION: + The verification of this test is automated. Do not change the + automatically selected result. + +plugin: user-interact +category_id: com.canonical.plainbox::mediacard +id: mediacard/cf-insert +estimated_duration: 30.0 +command: + if [[ -v SNAP ]]; then + timeout 20s bash -c 'grep -m 1 -oP "\Kadd.*?mmc.*?\s" <( exec journalctl -u snap.udisks2.udisksd -f -q -S -1s ); kill $!' + else + removable_storage_watcher --memorycard insert sdio usb scsi + fi +imports: from com.canonical.plainbox import manifest +requires: + package.name == 'udisks2' or snap.name == 'udisks2' + manifest.has_card_reader == 'True' +user: root +_description: + PURPOSE: + This test will check that the systems media card reader can + detect the insertion of a Compact Flash (CF) media card + STEPS: + 1. Commence the test and then insert a CF card into the reader. + (Note: this test will time-out after 20 seconds.) + 2. Do not remove the device after this test. + VERIFICATION: + The verification of this test is automated. Do not change the + automatically selected result. + +plugin: shell +category_id: com.canonical.plainbox::mediacard +id: mediacard/cf-storage +estimated_duration: 30.0 +depends: mediacard/cf-insert +user: root +command: removable_storage_test -s 268400000 --memorycard sdio usb scsi +requires: + package.name == 'udisks2' or (snap.name == 'core' and int(snap.revision) >= 1804) +_description: + This test is automated and executes after the mediacard/cf-insert + test is run. It tests reading and writing to the CF card. + +plugin: user-interact +category_id: com.canonical.plainbox::mediacard +id: mediacard/cf-remove +depends: mediacard/cf-storage +estimated_duration: 30.0 +command: + if [[ -v SNAP ]]; then + timeout 20s bash -c 'grep -m 1 -oP "\Kremove.*?mmc.*?\s" <( exec journalctl -u snap.udisks2.udisksd -f -q -S -1s ); kill $!' + else + removable_storage_watcher --memorycard remove sdio usb scsi + fi +user: root +_description: + PURPOSE: + This test will check that the system correctly detects + the removal of a CF card from the systems card reader. + STEPS: + 1. Commence the test and then remove the CF card from the reader. + (Note: this test will time-out after 20 seconds.) + VERIFICATION: + The verification of this test is automated. Do not change the + automatically selected result. + +plugin: user-interact +category_id: com.canonical.plainbox::mediacard +id: mediacard/sdxc-insert +estimated_duration: 30.0 +command: + if [[ -v SNAP ]]; then + timeout 20s bash -c 'grep -m 1 -oP "\Kadd.*?mmc.*?\s" <( exec journalctl -u snap.udisks2.udisksd -f -q -S -1s ); kill $!' + else + removable_storage_watcher --memorycard insert sdio usb scsi + fi +imports: from com.canonical.plainbox import manifest +requires: + package.name == 'udisks2' or snap.name == 'udisks2' + manifest.has_card_reader == 'True' +user: root +_description: + PURPOSE: + This test will check that the systems media card reader can + detect the insertion of a Secure Digital Extended Capacity (SDXC) media card + STEPS: + 1. Commence the test and then insert an UNLOCKED SDXC card into the reader. + (Note: this test will time-out after 20 seconds.) + 2. Do not remove the device after this test. + VERIFICATION: + The verification of this test is automated. Do not change the + automatically selected result. + +plugin: shell +category_id: com.canonical.plainbox::mediacard +id: mediacard/sdxc-storage +estimated_duration: 30.0 +depends: mediacard/sdxc-insert +user: root +command: removable_storage_test -s 268400000 --memorycard sdio usb scsi +requires: + package.name == 'udisks2' or (snap.name == 'core' and int(snap.revision) >= 1804) +_description: + This test is automated and executes after the mediacard/sdxc-insert + test is run. It tests reading and writing to the SDXC card. + +plugin: user-interact +category_id: com.canonical.plainbox::mediacard +id: mediacard/sdxc-remove +estimated_duration: 30.0 +depends: mediacard/sdxc-insert +command: + if [[ -v SNAP ]]; then + timeout 20s bash -c 'grep -m 1 -oP "\Kremove.*?mmc.*?\s" <( exec journalctl -u snap.udisks2.udisksd -f -q -S -1s ); kill $!' + else + removable_storage_watcher --memorycard remove sdio usb scsi + fi +user: root +_description: + PURPOSE: + This test will check that the system correctly detects + the removal of a SDXC card from the systems card reader. + STEPS: + 1. Commence the test and then remove the SDXC card from the reader. + (Note: this test will time-out after 20 seconds.) + VERIFICATION: + The verification of this test is automated. Do not change the + automatically selected result. + +plugin: user-interact +category_id: com.canonical.plainbox::mediacard +id: mediacard/ms-insert +estimated_duration: 30.0 +command: + if [[ -v SNAP ]]; then + timeout 20s bash -c 'grep -m 1 -oP "\Kadd.*?mmc.*?\s" <( exec journalctl -u snap.udisks2.udisksd -f -q -S -1s ); kill $!' + else + removable_storage_watcher --memorycard insert sdio usb scsi + fi +imports: from com.canonical.plainbox import manifest +requires: + package.name == 'udisks2' or snap.name == 'udisks2' + manifest.has_card_reader == 'True' +user: root +_description: + PURPOSE: + This test will check that the systems media card reader can + detect the insertion of a Memory Stick (MS) media card + STEPS: + 1. Commence the test and then insert a MS card into the reader. + (Note: this test will time-out after 20 seconds.) + 2. Do not remove the device after this test. + VERIFICATION: + The verification of this test is automated. Do not change the + automatically selected result. + +plugin: shell +category_id: com.canonical.plainbox::mediacard +id: mediacard/ms-storage +estimated_duration: 30.0 +depends: mediacard/ms-insert +user: root +command: removable_storage_test -s 268400000 --memorycard sdio usb scsi +requires: + package.name == 'udisks2' or (snap.name == 'core' and int(snap.revision) >= 1804) +_description: + This test is automated and executes after the mediacard/ms-insert + test is run. It tests reading and writing to the MS card. + +plugin: user-interact +category_id: com.canonical.plainbox::mediacard +id: mediacard/ms-remove +estimated_duration: 30.0 +depends: mediacard/ms-insert +command: + if [[ -v SNAP ]]; then + timeout 20s bash -c 'grep -m 1 -oP "\Kremove.*?mmc.*?\s" <( exec journalctl -u snap.udisks2.udisksd -f -q -S -1s ); kill $!' + else + removable_storage_watcher --memorycard remove sdio usb scsi + fi +user: root +_description: + PURPOSE: + This test will check that the system correctly detects + the removal of a MS card from the systems card reader. + STEPS: + 1. Commence the test and then remove the MS card from the reader. + (Note: this test will time-out after 20 seconds.) + VERIFICATION: + The verification of this test is automated. Do not change the + automatically selected result. + +plugin: user-interact +category_id: com.canonical.plainbox::mediacard +id: mediacard/msp-insert +estimated_duration: 30.0 +command: + if [[ -v SNAP ]]; then + timeout 20s bash -c 'grep -m 1 -oP "\Kadd.*?mmc.*?\s" <( exec journalctl -u snap.udisks2.udisksd -f -q -S -1s ); kill $!' + else + removable_storage_watcher --memorycard insert sdio usb scsi + fi +user: root +imports: from com.canonical.plainbox import manifest +requires: + package.name == 'udisks2' or snap.name == 'udisks2' + manifest.has_card_reader == 'True' +_description: + PURPOSE: + This test will check that the systems media card reader can + detect the insertion of a Memory Stick Pro (MSP) media card + STEPS: + 1. Commence the test and then insert a MSP card into the reader. + (Note: this test will time-out after 20 seconds.) + 2. Do not remove the device after this test. + VERIFICATION: + The verification of this test is automated. Do not change the + automatically selected result. + +plugin: shell +category_id: com.canonical.plainbox::mediacard +id: mediacard/msp-storage +estimated_duration: 30.0 +depends: mediacard/msp-insert +user: root +command: removable_storage_test -s 268400000 --memorycard sdio usb scsi +requires: + package.name == 'udisks2' or (snap.name == 'core' and int(snap.revision) >= 1804) +_description: + This test is automated and executes after the mediacard/msp-insert + test is run. It tests reading and writing to the MSP card. + +plugin: user-interact +category_id: com.canonical.plainbox::mediacard +id: mediacard/msp-remove +estimated_duration: 30.0 +depends: mediacard/msp-insert +command: + if [[ -v SNAP ]]; then + timeout 20s bash -c 'grep -m 1 -oP "\Kremove.*?mmc.*?\s" <( exec journalctl -u snap.udisks2.udisksd -f -q -S -1s ); kill $!' + else + removable_storage_watcher --memorycard remove sdio usb scsi + fi +user: root +_description: + PURPOSE: + This test will check that the system correctly detects + the removal of a MSP card from the systems card reader. + STEPS: + 1. Commence the test and remove the MSP card from the reader. + (Note: this test will time-out after 20 seconds.) + VERIFICATION: + The verification of this test is automated. Do not change the + automatically selected result. + +plugin: user-interact +category_id: com.canonical.plainbox::mediacard +id: mediacard/xd-insert +estimated_duration: 30.0 +command: + if [[ -v SNAP ]]; then + timeout 20s bash -c 'grep -m 1 -oP "\Kadd.*?mmc.*?\s" <( exec journalctl -u snap.udisks2.udisksd -f -q -S -1s ); kill $!' + else + removable_storage_watcher --memorycard insert sdio usb scsi + fi +imports: from com.canonical.plainbox import manifest +requires: + package.name == 'udisks2' or snap.name == 'udisks2' + manifest.has_card_reader == 'True' +user: root +_description: + PURPOSE: + This test will check that the systems media card reader can + detect the insertion of a Extreme Digital (xD) media card + STEPS: + 1. Commence the test and then insert a xD card into the reader. + (Note: this test will time-out after 20 seconds.) + 2. Do not remove the device after this test. + VERIFICATION: + The verification of this test is automated. Do not change the + automatically selected result. + +plugin: shell +category_id: com.canonical.plainbox::mediacard +id: mediacard/xd-storage +estimated_duration: 30.0 +depends: mediacard/xd-insert +user: root +command: removable_storage_test -s 268400000 --memorycard sdio usb scsi +requires: + package.name == 'udisks2' or (snap.name == 'core' and int(snap.revision) >= 1804) +_description: + This test is automated and executes after the mediacard/xd-insert + test is run. It tests reading and writing to the xD card. + +plugin: user-interact +category_id: com.canonical.plainbox::mediacard +id: mediacard/xd-remove +estimated_duration: 30.0 +depends: mediacard/xd-insert +command: + if [[ -v SNAP ]]; then + timeout 20s bash -c 'grep -m 1 -oP "\Kremove.*?mmc.*?\s" <( exec journalctl -u snap.udisks2.udisksd -f -q -S -1s ); kill $!' + else + removable_storage_watcher --memorycard remove sdio usb scsi + fi +user: root +_description: + PURPOSE: + This test will check that the system correctly detects + the removal of a xD card from the systems card reader. + STEPS: + 1. Commence the test and then remove the xD card from the reader. + (Note: this test will time-out after 20 seconds.) + VERIFICATION: + The verification of this test is automated. Do not change the + automatically selected result. + +plugin: user-interact-verify +category_id: com.canonical.plainbox::mediacard +id: mediacard/sd-performance-manual +depends: mediacard/sd-insert +estimated_duration: 120.0 +user: root +command: removable_storage_test -s 268400000 --memorycard sdio usb | cat <(echo "Working...") - <(echo; echo "Verify the result and decide on the outcome") +requires: + package.name == 'udisks2' or (snap.name == 'core' and int(snap.revision) >= 1804) +environ: LD_LIBRARY_PATH GI_TYPELIB_PATH +_description: + PURPOSE: + This test will check your Media Card data transfer performance. + STEPS: + 1. Insert SD card into the reader slot on this computer. + 2. Commence the test. + VERIFICATION: + Did the results of the test match the expected performance of the inserted device? + +plugin: user-interact-verify +category_id: com.canonical.plainbox::mediacard +id: mediacard/sdhc-performance-manual +depends: mediacard/sdhc-insert +estimated_duration: 120.0 +user: root +command: removable_storage_test -s 268400000 --memorycard sdio usb | cat <(echo "Working...") - <(echo; echo "Verify the result and decide on the outcome") +requires: + package.name == 'udisks2' or (snap.name == 'core' and int(snap.revision) >= 1804) +environ: LD_LIBRARY_PATH GI_TYPELIB_PATH +_description: + PURPOSE: + This test will check your Media Card data transfer performance. + STEPS: + 1. Insert SDHC card into the reader slot on this computer. + 2. Commence the test. + VERIFICATION: + Did the results of the test match the expected performance of the inserted device? + +plugin: user-interact-verify +category_id: com.canonical.plainbox::mediacard +id: mediacard/mmc-performance-manual +depends: mediacard/mmc-insert +estimated_duration: 120.0 +user: root +command: removable_storage_test -s 209715200 --memorycard sdio usb | cat <(echo "Working...") - <(echo; echo "Verify the result and decide on the outcome") +requires: + package.name == 'udisks2' or (snap.name == 'core' and int(snap.revision) >= 1804) +environ: LD_LIBRARY_PATH GI_TYPELIB_PATH +_description: + PURPOSE: + This test will check your Media Card data transfer performance. + STEPS: + 1. Insert MMC card into the reader slot on this computer. + 2. Commence the test. + VERIFICATION: + Did the results of the test match the expected performance of the inserted device? + +plugin: user-interact-verify +category_id: com.canonical.plainbox::mediacard +id: mediacard/cf-performance-manual +depends: mediacard/cf-insert +estimated_duration: 120.0 +user: root +command: removable_storage_test -s 268400000 --memorycard sdio usb | cat <(echo "Working...") - <(echo; echo "Verify the result and decide on the outcome") +requires: + package.name == 'udisks2' or (snap.name == 'core' and int(snap.revision) >= 1804) +environ: LD_LIBRARY_PATH GI_TYPELIB_PATH +_description: + PURPOSE: + This test will check your Media Card data transfer performance. + STEPS: + 1. Insert CF card into the reader slot on this computer. + 2. Commence the test. + VERIFICATION: + Did the results of the test match the expected performance of the inserted device? + +plugin: user-interact-verify +category_id: com.canonical.plainbox::mediacard +id: mediacard/sdxc-performance-manual +depends: mediacard/sdxc-insert +estimated_duration: 120.0 +user: root +command: removable_storage_test -s 268400000 --memorycard sdio usb | cat <(echo "Working...") - <(echo; echo "Verify the result and decide on the outcome") +requires: + package.name == 'udisks2' or (snap.name == 'core' and int(snap.revision) >= 1804) +environ: LD_LIBRARY_PATH GI_TYPELIB_PATH +_description: + PURPOSE: + This test will check your Media Card data transfer performance. + STEPS: + 1. Insert SDXC card into the reader slot on this computer. + 2. Commence the test. + VERIFICATION: + Did the results of the test match the expected performance of the inserted device? + +plugin: user-interact-verify +category_id: com.canonical.plainbox::mediacard +id: mediacard/ms-performance-manual +depends: mediacard/ms-insert +estimated_duration: 120.0 +user: root +command: removable_storage_test -s 268400000 --memorycard sdio usb | cat <(echo "Working...") - <(echo; echo "Verify the result and decide on the outcome") +requires: + package.name == 'udisks2' or (snap.name == 'core' and int(snap.revision) >= 1804) +environ: LD_LIBRARY_PATH GI_TYPELIB_PATH +_description: + PURPOSE: + This test will check your Media Card data transfer performance. + STEPS: + 1. Insert MS card into the reader slot on this computer. + 2. Commence the test. + VERIFICATION: + Did the results of the test match the expected performance of the inserted device? + +plugin: user-interact-verify +category_id: com.canonical.plainbox::mediacard +id: mediacard/msp-performance-manual +depends: mediacard/msp-insert +estimated_duration: 120.0 +user: root +command: removable_storage_test -s 268400000 --memorycard sdio usb | cat <(echo "Working...") - <(echo; echo "Verify the result and decide on the outcome") +requires: + package.name == 'udisks2' or (snap.name == 'core' and int(snap.revision) >= 1804) +environ: LD_LIBRARY_PATH GI_TYPELIB_PATH +_description: + PURPOSE: + This test will check your Media Card data transfer performance. + STEPS: + 1. Insert MSP card into the reader slot on this computer. + 2. Commence the test. + VERIFICATION: + Did the results of the test match the expected performance of the inserted device? + +plugin: user-interact-verify +category_id: com.canonical.plainbox::mediacard +id: mediacard/xd-performance-manual +depends: mediacard/xd-insert +estimated_duration: 120.0 +user: root +command: removable_storage_test -s 268400000 --memorycard sdio usb | cat <(echo "Working...") - <(echo; echo "Verify the result and decide on the outcome") +requires: + package.name == 'udisks2' or (snap.name == 'core' and int(snap.revision) >= 1804) +environ: LD_LIBRARY_PATH GI_TYPELIB_PATH +_description: + PURPOSE: + This test will check your Media Card data transfer performance. + STEPS: + 1. Insert xD card into the reader slot on this computer. + 2. Commence the test. + VERIFICATION: + Did the results of the test match the expected performance of the inserted device? + diff --git a/units/mediacard/manifest.pxu b/units/mediacard/manifest.pxu new file mode 100644 index 0000000..a0199db --- /dev/null +++ b/units/mediacard/manifest.pxu @@ -0,0 +1,4 @@ +unit: manifest entry +id: has_card_reader +_name: Media Card Reader +value-type: bool diff --git a/units/mediacard/test-plan.pxu b/units/mediacard/test-plan.pxu new file mode 100644 index 0000000..087e556 --- /dev/null +++ b/units/mediacard/test-plan.pxu @@ -0,0 +1,47 @@ +id: mediacard-cert-full +unit: test plan +_name: Mediacard tests +_description: Mediacard tests +include: + mediacard/sd-insert certification-status=blocker + mediacard/sd-storage certification-status=blocker + mediacard/sd-remove certification-status=blocker + mediacard/sdhc-insert certification-status=blocker + mediacard/sdhc-storage certification-status=blocker + mediacard/sdhc-remove certification-status=blocker + +id: after-suspend-mediacard-cert-full +unit: test plan +_name: Mediacard tests (after suspend) +_description: Mediacard tests (after suspend) +include: + suspend/sd-insert-after-suspend certification-status=blocker + suspend/sd-storage-after-suspend certification-status=blocker + suspend/sd-remove-after-suspend certification-status=blocker + suspend/sdhc-insert-after-suspend certification-status=blocker + suspend/sdhc-storage-after-suspend certification-status=blocker + suspend/sdhc-remove-after-suspend certification-status=blocker + +id: mediacard-cert-blockers +unit: test plan +_name: Mediacard tests (certification blockers only) +_description: Mediacard tests (certification blockers only) +include: + mediacard/sd-insert certification-status=blocker + mediacard/sd-storage certification-status=blocker + mediacard/sd-remove certification-status=blocker + mediacard/sdhc-insert certification-status=blocker + mediacard/sdhc-storage certification-status=blocker + mediacard/sdhc-remove certification-status=blocker + +id: after-suspend-mediacard-cert-blockers +unit: test plan +_name: Mediacard tests (after suspend, certification blockers only) +_description: Mediacard tests (after suspend, certification blockers only) +include: + suspend/sd-insert-after-suspend certification-status=blocker + suspend/sd-storage-after-suspend certification-status=blocker + suspend/sd-remove-after-suspend certification-status=blocker + suspend/sdhc-insert-after-suspend certification-status=blocker + suspend/sdhc-storage-after-suspend certification-status=blocker + suspend/sdhc-remove-after-suspend certification-status=blocker diff --git a/units/memory/jobs.pxu b/units/memory/jobs.pxu new file mode 100644 index 0000000..48d543c --- /dev/null +++ b/units/memory/jobs.pxu @@ -0,0 +1,56 @@ +plugin: shell +category_id: com.canonical.plainbox::memory +id: memory/info +estimated_duration: 5.0 +user: root +command: memory_compare +_summary: + Check amount of memory reported by meminfo against DMI +_description: + This test checks the amount of memory which is reporting in meminfo against + the size of the memory modules detected by DMI. + +plugin: shell +category_id: com.canonical.plainbox::memory +id: memory/check +estimated_duration: 1000.0 +user: root +requires: + uname.name == 'Linux' +command: memory_test +_summary: + Run memory stress test including swapping to disk +_description: + Test to perform some basic stress and exercise of system memory. This test + also includes an over-commit function to force swapping to disk, thus SUTs + should have suitably large swap files for the amount of RAM they have + installed. + +plugin: shell +category_id: com.canonical.plainbox::memory +id: memory/meminfo_before_suspend +estimated_duration: 1.2 +_summary: Store memory info before suspending +command: meminfo_resource > $PLAINBOX_SESSION_SHARE/meminfo_before_suspend +_siblings: [ + { "id": "after-suspend-memory/meminfo_after_suspend", + "_summary": "Compare memory info to the state prior to suspend", + "command": "meminfo_resource | diff $PLAINBOX_SESSION_SHARE/meminfo_before_suspend -", + "depends": "com.canonical.certification::suspend/suspend_advanced_auto"} + ] + +plugin: shell +category_id: com.canonical.plainbox::memory +id: memory/memory_stress_ng +estimated_duration: 11000.0 +user: root +requires: + package.name == 'stress-ng' or executable.name == 'stress-ng' +command: memory_stress_ng +_summary: Stress test of system memory +_description: + Test to perform some basic stress and exercise of system memory via the + stress_ng tool. This test also includes an over-commit function to force + swapping to disk, thus SUTs should have suitably large swap files for the + amount of RAM they have installed. + diff --git a/units/miscellanea/jobs.pxu b/units/miscellanea/jobs.pxu new file mode 100644 index 0000000..a253cd4 --- /dev/null +++ b/units/miscellanea/jobs.pxu @@ -0,0 +1,381 @@ +plugin: shell +category_id: com.canonical.plainbox::miscellanea +id: miscellanea/submission-resources +depends: + cdimage + cpuinfo + dmi + dpkg + efi + environment + lsb + meminfo + module + package + snap + requirements + uname + dmi_attachment + lsblk_attachment + sysfs_attachment + udev_attachment + dkms_info_json + udev_json + raw_devices_dmi_json + modprobe_json + lspci_standard_config_json +estimated_duration: 1.0 +command: true +_summary: + Check that data for a complete result are present +_description: + A meta-job that verifies the data necessary for a complete result + submission are present. Failure indicates that the results are incomplete + and may be rejected. + +plugin: manual +category_id: com.canonical.plainbox::miscellanea +id: miscellanea/tester-info +_description: + PURPOSE: + Keep tester related information in the report + STEPS: + 1. Tester Information + 2. Please enter the following information in the comments field: + a. Name + b. Email Address + c. Reason for this test run + VERIFICATION: + Nothing to verify for this test + +plugin: user-interact-verify +category_id: com.canonical.plainbox::miscellanea +id: miscellanea/chvt +requires: + package.name == 'kbd' +user: root +command: cycle_vts +_description: + PURPOSE: + This test will check that the system can switch to a virtual terminal and back to X + STEPS: + 1. Click "Test" to switch to another virtual terminal and then back to X + VERIFICATION: + Did your screen change temporarily to a text console and then switch back to your current session? + +plugin: shell +category_id: com.canonical.plainbox::miscellanea +id: miscellanea/fwts_test +estimated_duration: 1.2 +requires: + package.name == 'fwts' +user: root +_description: + Run Firmware Test Suite (fwts) automated tests. +environ: PLAINBOX_SESSION_SHARE +command: + checkbox-support-fwts_test -l $PLAINBOX_SESSION_SHARE/fwts_results.log + +plugin: shell +category_id: com.canonical.plainbox::miscellanea +id: miscellanea/fwupdate +estimated_duration: 1.0 +depends: miscellanea/efi_boot_mode +requires: + cpuinfo.platform in ("i386", "x86_64", "aarch64", "armhf") + package.name == 'fwupdate' or executable.name == 'fwupdate' +_description: + Determine if EFI firmware supports update from OS. +_summary: + Check for firmware update support. +command: fwupdate -s + +plugin: attachment +category_id: com.canonical.plainbox::miscellanea +id: miscellanea/fwts_results.log +requires: + package.name == 'fwts' +command: + [[ -e ${PLAINBOX_SESSION_SHARE}/fwts_results.log ]] && xz -c ${PLAINBOX_SESSION_SHARE}/fwts_results.log | base64 +_description: Attaches the miscellanes/fwts_test results log to the submission. + +plugin: attachment +category_id: com.canonical.plainbox::miscellanea +estimated_duration: 0.5 +id: miscellanea/fwts_results_hwe.log.gz +requires: + package.name == 'fwts' +command: + [ -f $PLAINBOX_SESSION_SHARE/fwts_results_hwe.log ] && gzip -c $PLAINBOX_SESSION_SHARE/fwts_results_hwe.log | base64 +_description: Attaches the FWTS results log to the submission (to HWE) +_summary: Attach FWTS log to submission (to HWE) + +plugin: shell +category_id: com.canonical.plainbox::miscellanea +id: miscellanea/ipmi_test +requires: + package.name == 'ipmitool' or executable.name == 'ipmitool' + cpuinfo.platform != 's390x' +user: root +command: ipmi_test +_summary: + Test IPMI in-band communications +_description: + This will run some basic commands in-band against a BMC, verifying that IPMI + works. Use of MAAS to deploy the system implicitly tests out-of-band BMC + control. + +plugin: shell +category_id: com.canonical.plainbox::miscellanea +estimated_duration: 0.5 +id: miscellanea/efi_boot_mode +requires: + cpuinfo.platform in ("i386", "x86_64", "aarch64") +_summary: Test that system booted in EFI mode +_description: + Test to verify that the system booted in EFI mode with Secure Boot active. +command: boot_mode_test efi + +plugin: shell +category_id: com.canonical.plainbox::miscellanea +estimated_duration: 0.5 +id: miscellanea/secure_boot_mode +requires: + cpuinfo.platform in ("i386", "x86_64", "aarch64") +depends: miscellanea/efi_boot_mode +_summary: Test that system booted with Secure Boot active +_description: + Test to verify that the system booted in Secure Boot active. +command: boot_mode_test secureboot + +plugin: shell +category_id: com.canonical.plainbox::miscellanea +estimated_duration: 0.5 +user: root +id: miscellanea/efi_pxeboot +requires: + cpuinfo.platform in ("i386", "x86_64", "aarch64") +depends: miscellanea/efi_boot_mode +_summary: Test that system booted from the network +_description: + Test to verify that the system booted from the network. + Works only on EFI-based systems. +command: efi-pxeboot + +plugin: shell +category_id: com.canonical.plainbox::miscellanea +id: miscellanea/bmc_info +requires: + package.name == 'ipmitool' or executable.name == 'ipmitool' + cpuinfo.platform != 's390x' +estimated_duration: 0.5 +user: root +command: bmc_info +_description: + This will gather some info about the BMC itself for diagnostic purposes. This + will not works on non-IPMI systems like AMT and blade/sled type systems. +_summary: + Gather BMC identification info + +plugin: shell +category_id: com.canonical.plainbox::miscellanea +id: miscellanea/cpus_are_not_samples +requires: + package.name == 'dmidecode' or executable.name == 'dmidecode' + dmi_present.state == 'supported' +estimated_duration: 0.5 +user: root +command: dmitest cpu-check +_description: + Sanity check of CPU information; fails if CPU is an engineering sample +_summary: + Test DMI data for CPUs + +plugin: shell +category_id: com.canonical.plainbox::miscellanea +id: miscellanea/dmitest_server +requires: + package.name == 'dmidecode' or executable.name == 'dmidecode' + dmi_present.state == 'supported' +estimated_duration: 0.5 +user: root +command: dmitest --test_versions server +_description: + Sanity check of DMI system identification data (for servers) +_summary: + Test DMI identification data (servers) + +plugin: shell +category_id: com.canonical.plainbox::miscellanea +id: miscellanea/dmitest_client +requires: + package.name == 'dmidecode' + dmi_present.state == 'supported' +estimated_duration: 0.5 +user: root +command: dmitest --test_versions --test_serials desktop +_description: + Sanity check of DMI system identification data (for desktops & laptops) +_summary: + DMI ID sanity check (desktops/laptops) + +plugin: shell +category_id: com.canonical.plainbox::miscellanea +id: miscellanea/is_laptop +user: root +_description: + Determine if we need to run tests specific to portable computers that may not apply to desktops. +command: check_is_laptop + +plugin: shell +category_id: com.canonical.plainbox::miscellanea +id: miscellanea/apport-directory +requires: package.name == 'apport' +command: if [ -d /var/crash ]; then if [ $(find /var/crash -type f | grep -v .lock | wc -l) -eq 0 ]; then echo "/var/crash is empty"; else echo `ls /var/crash`; false; fi; else echo "/var/crash does not exist"; fi +_description: + Test that the /var/crash directory doesn't contain anything. Lists the files contained within if it does, or echoes the status of the directory (doesn't exist/is empty) + +plugin: shell +category_id: com.canonical.plainbox::miscellanea +id: miscellanea/sources-list +command: sources_test $SOURCES_LIST "$REPOSITORIES" +_description: Checks that a specified sources list file contains the requested repositories + +unit: template +template-resource: device +template-filter: device.category == 'NETWORK' +plugin: manual +category_id: com.canonical.plainbox::miscellanea +id: miscellanea/pxe_boot_device{__index__}_{interface} +_summary: PXE verification test of device {__index__} ({interface}) +estimated_duration: 30.0 +_purpose: + This test will verify that you have attempted to PXE boot this machine from + the network device {__index__} ({interface}). +_steps: + 1. Prior to running this test, you should have attempted to boot this system via PXE on every Network Port available. +_verification: + 1. Select Yes if you successfully used PXE to boot this system using the NIC {interface} + 2. Select No if you attempted to PXE boot via {interface} and it failed for some reason. + 3. Select Skip if you did not attempt to PXE boot this system using the NIC {interface} + +unit: template +template-resource: device +template-filter: device.category == 'NETWORK' +plugin: manual +category_id: com.canonical.plainbox::miscellanea +id: miscellanea/remote_shared_ipmi_device{__index__}_{interface} +_summary: Remote Shared IPMI verification test for device {__index__} ({interface}) +estimated_duration: 30.0 +_purpose: + This test will verify that you have attempted IPMI control of this machine from the network device {__index__} ({interface}). +_steps: + 1. Prior to running this test, you should have configured and used IPMI to power this machine off and on using every Network Port available. +_verification: + 1. Select Yes if you successfully used IPMI to remotely power this system off and on using the NIC {interface} + 2. Select No if you did not attempt to use IPMI to remotely power this sytem off and on via the NIC {interface} + 3. Select No if you attempted to use IPMI to remotely power off/on this system via {interface} and it failed for some reason. + +plugin: manual +category_id: com.canonical.plainbox::miscellanea +id: miscellanea/remote_dedicated_ipmi +_description: + PURPOSE: + Some systems do not share IPMI over all NICs but instead have a dedicated management port directly connected to the BMC. This test verifies that you have used that port for remote IPMI connections and actions. + STEPS: + 1. Prior to running the test, you should have configured and used the Dedicated Management Port to remotely power off/on this sytem. + VERIFICATION: + Skip this test if this system ONLY uses shared management/ethernet ports OR if this system does not have a Management Controller. + 1. Select Yes if you successfully used IPMI to remotely power this system off and on using the dedicated management port. + 2. Select No if you attempted to use the dedicated management port to remotely power this system off/on and it failed for some reason. + +plugin:shell +id: miscellanea/oops +estimated_duration: 10.0 +requires: + package.name == 'fwts' +user: root +_description: + Run Firmware Test Suite (fwts) oops tests. +environ: PLAINBOX_SESSION_SHARE +command: + checkbox-support-fwts_test -l $PLAINBOX_SESSION_SHARE/fwts_oops_results.log -t oops + +plugin: attachment +category_id: com.canonical.plainbox::miscellanea +estimated_duration: 0.5 +id: miscellanea/oops_results.log +command: + [ -e ${PLAINBOX_SESSION_SHARE}/fwts_oops_results.log ] && xz -c ${PLAINBOX_SESSION_SHARE}/fwts_oops_results.log | base64 +_description: Attaches the FWTS oops results log to the submission + +plugin: shell +category_id: com.canonical.plainbox::miscellanea +estimated_duration: 0.1 +id: miscellanea/get_maas_version +command: maas-version-check +_description: If system was installed via MAAS from a cert server, the MAAS version used should be contained in /etc/installed-by-maas +_summary: Verify MAAS version used to deploy the SUT + +plugin: shell +category_id: com.canonical.plainbox::miscellanea +estimated_duration: 30.0 +id: miscellanea/get_make_and_model +user: root +command: get_make_and_model +requires: + dmi_present.state == 'supported' +_description: Retrieve the computer's make and model for easier access than digging through the dmidecode output. +_summary: Gather info on the SUT's make and model + +plugin: user-interact-verify +category_id: com.canonical.plainbox::miscellanea +estimated_duration: 5.0 +id: miscellanea/device_check +command: udev_resource -l VIDEO NETWORK WIRELESS DISK ACCELEROMETER | tee >([[ $DISPLAY ]] && zenity --text-info --title="Device report") +_summary: Device Check +_purpose: + Device check +_steps: + 1. Commence the test + 2. Compare items on System Manifest to the devices known to udev +_verification: + Do the devices reported by udev match the devices on the Manifest? + +plugin: shell +category_id: com.canonical.plainbox::miscellanea +estimated_duration: 20.0 +id: miscellanea/sosreport +user: root +requires: executable.name == 'sosreport' +command: sosreport --batch --tmp-dir $PLAINBOX_SESSION_SHARE +_summary: Generate baseline sosreport +_description: Generates a baseline sosreport of logs and system data + +plugin: attachment +category_id: com.canonical.plainbox::miscellanea +estimated_duration: 5.0 +id: miscellanea/sosreport_attachment +depends: miscellanea/sosreport +user: root +command: + SOSFILE=`ls -t $PLAINBOX_SESSION_SHARE/sosreport*xz | head -1`; [ -e ${SOSFILE} ] && base64 $SOSFILE +_summary: Attach the baseline sosreport file + +plugin: shell +category_id: com.canonical.plainbox::miscellanea +estimated_duration: 0.5 +id: miscellanea/call-trace-check +command: if [ -n "$(grep "Call Trace:" /var/log/syslog)" ]; then echo "Call Trace detected in syslog"; false; else true; fi +_summary: Check syslog for call traces +_description: Checks syslog for call traces after testing is complete. +flags: preserve-locale + +plugin: attachment +category_id: com.canonical.plainbox::miscellanea +estimated_duration: 0.5 +id: miscellanea/attach-syslog +command: if [ -n "$(grep "Call Trace:" /var/log/syslog)" ]; then cat /var/log/syslog; fi +_summary: Attach syslog if Call Trace appears +_description: Attaches a copy of syslog if call traces are present after testing is complete. +flags: preserve-locale diff --git a/units/miscellanea/test-plan.pxu b/units/miscellanea/test-plan.pxu new file mode 100644 index 0000000..86f502c --- /dev/null +++ b/units/miscellanea/test-plan.pxu @@ -0,0 +1,44 @@ +id: misc-client-cert-full +unit: test plan +_name: Miscellaneous client tests +_description: Miscellaneous client tests (fwts, dmi, oops) +include: +nested_part: + misc-client-cert-automated + +id: misc-client-cert-automated +unit: test plan +_name: Miscellaneous client tests (automated) +_description: + Miscellaneous client tests (fwts, dmi, oops) + (automated) +include: + install/apt-get-gets-updates + memory/info certification-status=blocker + firmware/fwts_desktop_diagnosis + firmware/fwts_desktop_diagnosis_hwe + firmware/fwts_desktop_diagnosis_results.log.gz + firmware/fwts_desktop_diagnosis_results_hwe.log.gz + firmware/fwts_wakealarm.* certification-status=blocker + firmware/fwts_uefirtvariable.* certification-status=blocker + firmware/no_ACPI_REV_interface + miscellanea/oops certification-status=blocker + miscellanea/oops_results.log + miscellanea/dmitest_client +bootstrap_include: + fwts + +id: misc-client-cert-blockers +unit: test plan +_name: Miscellaneous client tests (certification blockers only) +_description: + Miscellaneous client tests (fwts, dmi, oops) + (certification blockers only) +include: + memory/info certification-status=blocker + firmware/fwts_wakealarm.* certification-status=blocker + firmware/fwts_uefirtvariable.* certification-status=blocker + miscellanea/oops certification-status=blocker + miscellanea/oops_results.log +bootstrap_include: + fwts diff --git a/units/mobilebroadband/jobs.pxu b/units/mobilebroadband/jobs.pxu new file mode 100644 index 0000000..0c83428 --- /dev/null +++ b/units/mobilebroadband/jobs.pxu @@ -0,0 +1,106 @@ +plugin: shell +category_id: com.canonical.plainbox::mobilebroadband +id: mobilebroadband/gsm_connection +estimated_duration: 20.0 +requires: + package.name == 'network-manager' + package.name == 'modemmanager' + mobilebroadband.gsm == 'supported' +user: root +environ: GSM_CONN_NAME GSM_APN GSM_USERNAME GSM_PASSWORD +command: + if [ -n "${GSM_APN}" ]; then + trap "nmcli con delete id $GSM_CONN_NAME" EXIT + create_connection mobilebroadband gsm \ + `[ -n "${GSM_APN}" ] && echo "--apn=$GSM_APN"` \ + `[ -n "${GSM_CONN_NAME}" ] && echo "--name=$GSM_CONN_NAME"` \ + `[ -n "${GSM_USERNAME}" ] && echo "--username=$GSM_USERNAME"` \ + `[ -n "${GSM_PASSWORD}" ] && echo "--password=$GSM_PASSWORD"` || exit 1 + fi + INTERFACE=`(nmcli -f GENERAL -t dev list 2>/dev/null || nmcli -f GENERAL -t dev show) | tr '\n' ' ' | grep -oP 'TYPE:\Kgsm.*' | sed 's/GENERAL.TYPE:.*//' | grep -oP 'GENERAL.IP-IFACE:\K\S*'` + echo "connected GSM interface seems to be $INTERFACE" + [ -z $INTERFACE ] && exit 1 + curl http://start.ubuntu.com/connectivity-check.html --interface $INTERFACE + EXIT_CODE=$? + if [ -n "${GSM_APN}" ] && [ "`nmcli dev status | awk '/gsm/ {print $3}'`" == "connected" ]; then + nmcli con down id `[ ${GSM_CONN_NAME} ] && echo "$GSM_CONN_NAME" || echo "MobileBB"` + fi + exit $EXIT_CODE +_description: Creates a mobile broadband connection for a GSM based modem and checks the connection to ensure it's working. + +plugin: shell +category_id: com.canonical.plainbox::mobilebroadband +id: mobilebroadband/cdma_connection +estimated_duration: 20.0 +requires: + package.name == 'network-manager' + package.name == 'modemmanager' + mobilebroadband.cdma == 'supported' +user: root +environ: CDMA_CONN_NAME CDMA_USERNAME CDMA_PASSWORD +command: + if [ -n "${CDMA_USERNAME}" ]; then + trap "nmcli con delete id $CDMA_CONN_NAME" EXIT + create_connection mobilebroadband cdma \ + `[ -n "${CDMA_CONN_NAME}" ] && echo "--name=$CDMA_CONN_NAME"` \ + `[ -n "${CDMA_USERNAME}" ] && echo "--username=$CDMA_USERNAME"` \ + `[ -n "${CDMA_PASSWORD}" ] && echo "--password=$CDMA_PASSWORD"` || exit 1 + fi + INTERFACE=`(nmcli -f GENERAL -t dev list 2>/dev/null || nmcli -f GENERAL -t dev show) | tr '\n' ' ' | grep -oP 'TYPE:\Kcdma.*' | sed 's/GENERAL.TYPE:.*//' | grep -oP 'GENERAL.IP-IFACE:\K\S*'` + echo "connected CDMA interface seems to be $INTERFACE" + [ -z $INTERFACE ] && exit 1 + curl http://start.ubuntu.com/connectivity-check.html --interface $INTERFACE + EXIT_CODE=$? + if [ -n "${CDMA_USERNAME}" ] && [ "`nmcli dev status | awk '/cdma/ {print $3}'`" == "connected" ]; then + nmcli con down id `[ ${CDMA_CONN_NAME} ] && echo "$CDMA_CONN_NAME" || echo "MobileBB"` + fi + exit $EXIT_CODE +_description: Creates a mobile broadband connection for a CDMA based modem and checks the connection to ensure it's working. + +unit: template +template-resource: device +template-filter: device.category == 'NETWORK' +plugin: user-interact-verify +category_id: com.canonical.plainbox::mobilebroadband +id: mobilebroadband/maximum_bandwidth_gsm_device{__index__}_{interface} +depends: mobilebroadband/gsm_connection +estimated_duration: 330.0 +requires: + package.name == 'zenity' + package.name == 'iperf' +environ: TEST_TARGET_FTP TEST_TARGET_IPERF TEST_USER TEST_PASS +user: root +command: network test -i {interface} -t iperf 2>&1 | cat - <(echo; echo "Verify the result and click OK to decide on the outcome") | zenity --text-info --title 'mobile broadband max bw {interface}' +_purpose: + User verification of whether the observed transfer throughput is acceptable + for the type and maximum speed of device {__index__} ({interface}). +_steps: + 1. Click "Test". + 2. Read the network test summary and confirm that the throughput is acceptable. + 3. If needed, click "Test" again to repeat the transfer test. +_verification: + Was the reported throughput acceptable for the type and maximum speed of this interface? + +unit: template +template-resource: device +template-filter: device.category == 'NETWORK' +plugin: user-interact-verify +category_id: com.canonical.plainbox::mobilebroadband +id: mobilebroadband/maximum_bandwidth_cdma_device{__index__}_{interface} +depends: mobilebroadband/cdma_connection +estimated_duration: 330.0 +requires: + package.name == 'zenity' + package.name == 'iperf' +environ: TEST_TARGET_FTP TEST_TARGET_IPERF TEST_USER TEST_PASS +user: root +command: network test -i {interface} -t iperf 2>&1 | cat - <(echo; echo "Verify the result and click OK to decide on the outcome") | zenity --text-info --title 'mobile broadband max bw {interface}' +_purpose: + User verification of whether the observed transfer throughput is acceptable + for the type and maximum speed of device {__index__} ({interface}). +_steps: + 1. Click "Test". + 2. Read the network test summary and confirm that the throughput is acceptable. + 3. If needed, click "Test" again to repeat the transfer test. +_verification: + Was the reported throughput acceptable for the type and maximum speed of this interface? diff --git a/units/mobilebroadband/test-plan.pxu b/units/mobilebroadband/test-plan.pxu new file mode 100644 index 0000000..166a297 --- /dev/null +++ b/units/mobilebroadband/test-plan.pxu @@ -0,0 +1,15 @@ +id: mobilebroadband-cert-full +unit: test plan +_name: Mobile broadband tests +_description: Mobile broadband tests +include: +nested_part: + mobilebroadband-cert-automated + +id: mobilebroadband-cert-automated +unit: test plan +_name: Mobile broadband tests (automated) +_description: Mobile broadband tests (automated) +include: + mobilebroadband/gsm_connection certification-status=non-blocker + mobilebroadband/cdma_connection certification-status=non-blocker diff --git a/units/monitor/jobs.pxu b/units/monitor/jobs.pxu new file mode 100644 index 0000000..36425db --- /dev/null +++ b/units/monitor/jobs.pxu @@ -0,0 +1,203 @@ +unit: template +template-resource: graphics_card +template-filter: graphics_card.prime_gpu_offload == 'Off' +id: monitor/{index}_vga_{product_slug} +requires: display.vga == 'supported' or display.dp == 'supported' +plugin: manual +category_id: com.canonical.plainbox::monitor +_purpose: + This test will check your VGA port. +_steps: + Skip this test if your system does not have a VGA port. + 1. Connect a display (if not already connected) to the VGA port on your system +_verification: + Was the desktop displayed correctly on both screens? + +unit: template +template-resource: graphics_card +template-filter: graphics_card.prime_gpu_offload == 'Off' +id: monitor/{index}_dvi_{product_slug} +requires: display.dvi == 'supported' +plugin: manual +category_id: com.canonical.plainbox::monitor +_purpose: + This test will check your DVI port. +_steps: + Skip this test if your system does not have a DVI port. + 1. Connect a display (if not already connected) to the DVI port on your system +_verification: + Was the desktop displayed correctly on both screens? + +unit: template +template-resource: graphics_card +template-filter: graphics_card.prime_gpu_offload == 'Off' +id: monitor/{index}_displayport_{product_slug} +requires: display.dp == 'supported' +plugin: manual +category_id: com.canonical.plainbox::monitor +_purpose: + This test will check your DisplayPort port. +_steps: + Skip this test if your system does not have a DisplayPort port. + 1. Connect a display (if not already connected) to the DisplayPort port on your system +_verification: + Was the desktop displayed correctly on both screens? + +unit: template +template-resource: graphics_card +template-filter: graphics_card.prime_gpu_offload == 'Off' +id: monitor/{index}_hdmi_{product_slug} +requires: display.hdmi == 'supported' +plugin: manual +category_id: com.canonical.plainbox::monitor +_purpose: + This test will check your HDMI port. +_steps: + Skip this test if your system does not have a HDMI port. + 1. Connect a display (if not already connected) to the HDMI port on your system +_verification: + Was the desktop displayed correctly on both screens? + +unit: template +template-resource: graphics_card +template-filter: graphics_card.prime_gpu_offload == 'Off' +id: monitor/{index}_svideo_{product_slug} +requires: display.svideo == 'supported' +plugin: manual +category_id: com.canonical.plainbox::monitor +_purpose: + This test will check your S-VIDEO port. +_steps: + Skip this test if your system does not have a S-VIDEO port. + 1. Connect a display (if not already connected) to the S-VIDEO port on your system +_verification: + Was the desktop displayed correctly on both screens? + +unit: template +template-resource: graphics_card +template-filter: graphics_card.prime_gpu_offload == 'Off' +id: monitor/{index}_rca_{product_slug} +requires: display.rca == 'supported' +plugin: manual +category_id: com.canonical.plainbox::monitor +_purpose: + This test will check your RCA port. +_steps: + Skip this test if your system does not have a RCA port. + 1. Connect a display (if not already connected) to the RCA port on your system +_verification: + Was the desktop displayed correctly on both screens? + +unit: template +template-resource: graphics_card +template-filter: graphics_card.prime_gpu_offload == 'Off' +id: monitor/{index}_multi-head_{product_slug} +requires: dmi.product in ['Desktop','Low Profile Desktop','Tower','Mini Tower', 'Space-saving'] +plugin: manual +category_id: com.canonical.plainbox::monitor +_purpose: + This test verifies that multi-monitor output works on your desktop system. This is NOT the same test as the external monitor tests you would run on your laptop. You will need two monitors to perform this test. +_steps: + Skip this test if your video card does not support multiple monitors. + 1. If your second monitor is not already connected, connect it now + 2. Open the "Displays" tool (open the dash and search for "Displays") + 3. Configure your output to provide one desktop across both monitors + 4. Open any application and drag it from one monitor to the next. +_verification: + Was the stretched desktop displayed correctly across both screens? + +unit: template +template-resource: graphics_card +template-filter: graphics_card.prime_gpu_offload == 'Off' +id: monitor/{index}_powersaving_{product_slug} +plugin: user-interact-verify +category_id: com.canonical.plainbox::monitor +command: xset dpms force off +_purpose: + This test will check your monitor power saving capabilities +_steps: + 1. Click "Test" to try the power saving capabilities of your monitor + 2. Press any key or move the mouse to recover +_verification: + Did the monitor go blank and turn on again? + +unit: template +template-resource: graphics_card +template-filter: graphics_card.prime_gpu_offload == 'Off' +id: monitor/{index}_dim_brightness_{product_slug} +requires: dmi.product in ['Notebook','Laptop','Portable','All In One','All-In-One','AIO'] +plugin: user-interact-verify +category_id: com.canonical.plainbox::monitor +user: root +command: brightness_test +_purpose: + This test will test changes to screen brightness +_steps: + 1. Click "Test" to try to dim the screen. + 2. Check if the screen was dimmed approximately to half of the maximum brightness. + 3. The screen will go back to the original brightness in 2 seconds. +_verification: + Was your screen dimmed approximately to half of the maximum brightness? + +unit: template +template-resource: graphics_card +template-filter: graphics_card.prime_gpu_offload == 'Off' +id: monitor/{index}_thunderbolt_{product_slug} +imports: from com.canonical.plainbox import manifest +requires: manifest.has_thunderbolt == 'True' +estimated_duration: 15.0 +plugin: manual +category_id: com.canonical.plainbox::monitor +_summary: Display connected via Thunderbolt for {vendor} {product} +_purpose: + This test will check your Thunderbolt port as a monitor interconnect for {vendor} {product}. +_steps: + 1. Connect a display (if not already connected) to the Thunderbolt port on + your system + 2. Switch display modes between in your Display Settings, check if it can be + set to mirrored, extended, displayed on external or onboard only +_verification: + Was the desktop displayed correctly with {vendor} {product} on the Thunderbolt-connected + screen in every mode? + +unit: template +template-resource: graphics_card +template-filter: graphics_card.prime_gpu_offload == 'Off' +id: monitor/{index}_thunderbolt3_{product_slug} +imports: from com.canonical.plainbox import manifest +requires: manifest.has_thunderbolt3 == 'True' +estimated_duration: 15.0 +plugin: manual +category_id: com.canonical.plainbox::monitor +_summary: Display connected via Thunderbolt 3 for {vendor} {product} +_purpose: + This test will check your Thunderbolt 3 port as a monitor interconnect for {vendor} {product}. +_steps: + 1. Connect a display (if not already connected) to the Thunderbolt 3 port on + your system + 2. Switch display modes between in your Display Settings, check if it can be + set to mirrored, extended, displayed on external or onboard only +_verification: + Was the desktop displayed correctly with {vendor} {product} on the Thunderbolt-connected + screen in every mode? + +unit: template +template-resource: graphics_card +id: monitor/{index}_type-c_displayport_{product_slug} +template-filter: graphics_card.prime_gpu_offload == 'Off' +imports: from com.canonical.plainbox import manifest +requires: manifest.has_usb_type_c == 'True' +estimated_duration: 15.0 +plugin: manual +category_id: com.canonical.plainbox::monitor +_summary: Display connected via DisplayPort using an USB Type-C port for {vendor} {product} +_purpose: + This test will check the connection of a screen using a "USB Type-C to DisplayPort" adapter for {vendor} {product}. +_steps: + 1. Connect a display (if not already connected) to the USB Type-C port on + your system using a "USB Type-C to DisplayPort" adapter + 2. Switch display modes between in your Display Settings, check if it can be + set to mirrored, extended, displayed on external or onboard only +_verification: + Was the desktop displayed correctly with {vendor} {product} on the screen + connected using a "USB Type-C to DisplayPort" adapter in every mode? diff --git a/units/monitor/test-plan.pxu b/units/monitor/test-plan.pxu new file mode 100644 index 0000000..10976c9 --- /dev/null +++ b/units/monitor/test-plan.pxu @@ -0,0 +1,86 @@ +id: monitor-integrated-gpu-cert-full +unit: test plan +_name: Monitor tests (integrated GPU) +_description: Monitor tests (integrated GPU) +include: + monitor/1_powersaving_.* certification-status=blocker + monitor/1_dim_brightness_.* certification-status=blocker + monitor/1_displayport_.* certification-status=blocker + monitor/1_type-c_displayport_.* certification-status=non-blocker + audio/1_playback_displayport_.* certification-status=blocker + audio/1_playback_type-c_displayport_.* certification-status=non-blocker + monitor/1_dvi_.* certification-status=blocker + monitor/1_hdmi_.* certification-status=blocker + audio/1_playback_hdmi_.* certification-status=blocker + monitor/1_thunderbolt_.* certification-status=blocker + monitor/1_thunderbolt3_.* certification-status=non-blocker + audio/1_playback_thunderbolt_.* certification-status=blocker + audio/1_playback_thunderbolt3_.* certification-status=non-blocker + thunderbolt/daisy-chain certification-status=blocker + thunderbolt3/daisy-chain certification-status=non-blocker + monitor/1_vga_.* certification-status=blocker + monitor/1_multi-head_.* certification-status=blocker +bootstrap_include: + graphics_card + +id: monitor-discrete-gpu-cert-full +unit: test plan +_name: Monitor tests (discrete GPU) +_description: Monitor tests (discrete GPU) +include: + monitor/2_powersaving_.* certification-status=blocker + monitor/2_dim_brightness_.* certification-status=blocker + monitor/2_displayport_.* certification-status=blocker + monitor/2_type-c_displayport_.* certification-status=non-blocker + audio/2_playback_displayport_.* certification-status=blocker + audio/2_playback_type-c_displayport_.* certification-status=non-blocker + monitor/2_dvi_.* certification-status=blocker + monitor/2_hdmi_.* certification-status=blocker + audio/2_playback_hdmi_.* certification-status=blocker + monitor/2_thunderbolt_.* certification-status=blocker + monitor/2_thunderbolt3_.* certification-status=non-blocker + audio/2_playback_thunderbolt_.* certification-status=blocker + audio/2_playback_thunderbolt3_.* certification-status=non-blocker + monitor/2_vga_.* certification-status=blocker + monitor/2_multi-head_.* certification-status=blocker +bootstrap_include: + graphics_card + +id: monitor-integrated-gpu-cert-blockers +unit: test plan +_name: Monitor tests (integrated GPU, certification blockers only) +_description: Monitor tests (integrated GPU, certification blockers only) +include: + monitor/1_powersaving_.* certification-status=blocker + monitor/1_dim_brightness_.* certification-status=blocker + monitor/1_displayport_.* certification-status=blocker + audio/1_playback_displayport_.* certification-status=blocker + monitor/1_dvi_.* certification-status=blocker + monitor/1_hdmi_.* certification-status=blocker + audio/1_playback_hdmi_.* certification-status=blocker + monitor/1_thunderbolt_.* certification-status=blocker + audio/1_playback_thunderbolt_.* certification-status=blocker + thunderbolt/daisy-chain certification-status=blocker + monitor/1_vga_.* certification-status=blocker + monitor/1_multi-head_.* certification-status=blocker +bootstrap_include: + graphics_card + +id: monitor-discrete-gpu-cert-blockers +unit: test plan +_name: Monitor tests (discrete GPU, certification blockers only) +_description: Monitor tests (discrete GPU, certification blockers only) +include: + monitor/2_powersaving_.* certification-status=blocker + monitor/2_dim_brightness_.* certification-status=blocker + monitor/2_displayport_.* certification-status=blocker + audio/2_playback_displayport_.* certification-status=blocker + monitor/2_dvi_.* certification-status=blocker + monitor/2_hdmi_.* certification-status=blocker + audio/2_playback_hdmi_.* certification-status=blocker + monitor/2_thunderbolt_.* certification-status=blocker + audio/2_playback_thunderbolt_.* certification-status=blocker + monitor/2_vga_.* certification-status=blocker + monitor/2_multi-head_.* certification-status=blocker +bootstrap_include: + graphics_card diff --git a/units/networking/jobs.pxu b/units/networking/jobs.pxu new file mode 100644 index 0000000..92eadd4 --- /dev/null +++ b/units/networking/jobs.pxu @@ -0,0 +1,79 @@ +plugin: shell +category_id: com.canonical.plainbox::networking +id: networking/gateway_ping +depends: ethernet/detect +command: gateway_ping_test +estimated_duration: 2.000 +_description: Tests whether the system has a working Internet connection. + +unit: template +template-resource: device +template-filter: device.category == 'NETWORK' +plugin: user-interact-verify +category_id: com.canonical.plainbox::networking +id: networking/info_device{__index__}_{interface} +_summary: Network Information of device {__index__} ({interface}) +estimated_duration: 1.0 +command: network_info {interface} | zenity --text-info --title="{interface}" +_description: + PURPOSE: + This test will check the network device {__index__} ({interface}) + STEPS: + 1. Click "Test" to verify the information for this network device + VERIFICATION: + Is this correct? + +plugin: user-interact-verify +category_id: com.canonical.plainbox::networking +id: networking/modem_connection +command: network_check +_description: + PURPOSE: + This test will check that a DSL modem can be configured and connected. + STEPS: + 1. Connect the telephone line to the computer + 2. Click on the Network icon on the top panel. + 3. Select "Edit Connections" + 4. Select the "DSL" tab + 5. Click on "Add" button + 6. Configure the connection parameters properly + 7. Click "Test" to verify that it's possible to establish an HTTP connection + VERIFICATION: + Did a notification show and was the connection correctly established? + +plugin: shell +category_id: com.canonical.plainbox::networking +id: networking/ping +command: gateway_ping_test $CHECKBOX_SERVER +_description: + Automated test case to verify availability of some system on the network using ICMP ECHO packets. + +plugin: shell +category_id: com.canonical.plainbox::networking +id: networking/http +command: wget -SO /dev/null http://$TRANSFER_SERVER +_description: + Automated test case to make sure that it's possible to download files through HTTP + +plugin: shell +category_id: com.canonical.plainbox::networking +id: networking/ntp +requires: package.name == 'ntpdate' +user: root +command: network_ntp_test +_description: Test to see if we can sync local clock to an NTP server + +plugin: shell +category_id: com.canonical.plainbox::networking +id: networking/ssh +requires: package.name == 'openssh-client' +command: if [ $CHECKBOX_SERVER ]; then ssh -q -o 'StrictHostKeyChecking=no' -o "UserKnownHostsFile=/tmp/ssh_test_$$" -l ubuntu $CHECKBOX_SERVER "uname -a" && rm /tmp/ssh_test_$$; fi +_description: Verify that an installation of checkbox-server on the network can be reached over SSH. + +plugin: shell +category_id: com.canonical.plainbox::networking +id: networking/printer +requires: package.name == 'cups-client' +command: network_printer_test -s $CHECKBOX_SERVER +_description: Try to enable a remote printer on the network and print a test page. + diff --git a/units/networking/packaging.pxu b/units/networking/packaging.pxu new file mode 100644 index 0000000..b71caa7 --- /dev/null +++ b/units/networking/packaging.pxu @@ -0,0 +1,3 @@ +unit: packaging meta-data +os-id: debian +Depends: ntpdate, net-tools diff --git a/units/networking/test-plan.pxu b/units/networking/test-plan.pxu new file mode 100644 index 0000000..ade662a --- /dev/null +++ b/units/networking/test-plan.pxu @@ -0,0 +1,34 @@ +id: networking-cert-full +unit: test plan +_name: Networking tests +_description: Networking tests +include: +nested_part: + networking-cert-automated + networking-cert-manual + +id: networking-cert-manual +unit: test plan +_name: Networking tests (manual) +_description: Networking tests (manual) +include: + networking/info.* certification-status=blocker + +id: networking-cert-automated +unit: test plan +_name: Networking tests (automated) +_description: Networking tests (automated) +include: + ethernet/detect certification-status=blocker + networking/gateway_ping certification-status=blocker + networking/ntp certification-status=blocker + +id: networking-cert-blockers +unit: test plan +_name: Networking tests (certification blockers only) +_description: Networking tests (certification blockers only) +include: + ethernet/detect certification-status=blocker + networking/gateway_ping certification-status=blocker + networking/info.* certification-status=blocker + networking/ntp certification-status=blocker diff --git a/units/optical/jobs.pxu b/units/optical/jobs.pxu new file mode 100644 index 0000000..707c02d --- /dev/null +++ b/units/optical/jobs.pxu @@ -0,0 +1,198 @@ +plugin: shell +category_id: com.canonical.plainbox::optical +id: optical/detect +requires: device.category == 'CDROM' +estimated_duration: 1.2 +_summary: Displays discovered optical drives +_description: Detects optical drives (CD/DVD) attached to the system. +command: udev_resource -l CDROM + +unit: template +template-resource: device +template-filter: device.category == 'CDROM' +plugin: user-interact-verify +category_id: com.canonical.plainbox::optical +id: optical/read_{name} +estimated_duration: 120.0 +user: root +command: optical_read_test /dev/{name} +_description: + PURPOSE: + This test will check your {product} device's ability to read CD media + STEPS: + 1. Insert appropriate non-blank media into your optical drive(s). Movie and Audio Disks may not work. Self-created data disks have the greatest chance of working. + 2. If a file browser window opens, you can safely close or ignore that window. + 3. Click "Test" to begin the test. + VERIFICATION: + This test should automatically select "Yes" if it passes, "No" if it fails. + +unit: template +template-resource: device +template-filter: device.category == 'CDROM' +plugin: shell +category_id: com.canonical.plainbox::optical +id: optical/read-automated_{name} +estimated_duration: 120.0 +user: root +command: optical_read_test /dev/{name} +_summary: + Tests read functionality of optical drive {name} +_description: + This is an automated version of optical/read. It assumes you have already inserted a data CD into your optical drive prior to running Checkbox. + +unit: template +template-resource: device +template-filter: device.category == 'CDROM' +plugin: user-interact-verify +category_id: com.canonical.plainbox::optical +id: optical/cdrom-write_{name} +estimated_duration: 120.0 +requires: + optical_drive_{name}.cd_write == 'supported' +user: root +command: set -o pipefail; optical_write_test /dev/{name} cd | ansi_parser +_description: + PURPOSE: + This test will check your system's {product} CD writing capabilities. This test requires a blank CD-R or CD+R. + STEPS: + Skip this test if you do not have a blank CD disk. + 1. Insert a blank CD-R or CD+R into your drive + 2. Click "Test" to begin. + 3. When the CD tray ejects the media after burning, close it (DO NOT remove the disk, it is needed for the second portion of the test). Note, you must close the drive within 5 minutes or the test will time out. + VERIFICATION: + This test should automatically select "Yes" if it passes, "No" if it fails. + +unit: template +template-resource: device +template-filter: device.category == 'CDROM' +plugin: shell +category_id: com.canonical.plainbox::optical +id: optical/cdrom-write-automated_{name} +estimated_duration: 120.0 +requires: + optical_drive_{name}.cd_write == 'supported' +user: root +command: set -o pipefail; optical_write_test /dev/{name} cd | ansi_parser +_description: + This is an automated version of optical/cdrom-write. It assumes you have already inserted a data CD into your {product} optical drive prior to running Checkbox. + +unit: template +template-resource: device +template-filter: device.category == 'CDROM' +plugin: manual +category_id: com.canonical.plainbox::optical +id: optical/cdrom-audio-playback_{name} +depends: optical/read_{name} +estimated_duration: 120.0 +_description: + PURPOSE: + This test will check your {product} CD audio playback capabilities + STEPS: + 1. Insert an audio CD in your optical drive + 2. When prompted, launch the Music Player + 3. Locate the CD in the display of the Music Player + 4. Select the CD in the Music Player + 5. Click the Play button to listen to the music on the CD + 6. Stop playing after some time + 7. Right click on the CD icon and select "Eject Disc" + 8. The CD should be ejected + 9. Close the Music Player + VERIFICATION: + Did all the steps work? + +unit: template +template-resource: device +template-filter: device.category == 'CDROM' +plugin: user-interact-verify +category_id: com.canonical.plainbox::optical +id: optical/dvd-write_{name} +requires: + optical_drive_{name}.dvd_write == 'supported' +estimated_duration: 120.0 +user: root +command: set -o pipefail; optical_write_test /dev/{name} dvd | ansi_parser +_description: + PURPOSE: + This test will check your system's {product} writing capabilities. This test requires a blank DVD-R or DVD+R. + STEPS: + Skip this test if you do not have a blank DVD disk. + 1. Enter a blank DVD-R or DVD+R into your drive + 2. Click "Test" to begin. + 3. When the CD tray ejects the media after burning, close it (DO NOT remove the disk, it is needed for the second portion of the test). Note, you must close the drive within 5 minutes or the test will time out. + VERIFICATION: + This test should automatically select "Yes" if it passes, "No" if it fails. + +unit: template +template-resource: device +template-filter: device.category == 'CDROM' +plugin: shell +category_id: com.canonical.plainbox::optical +id: optical/dvd-write-automated_{name} +estimated_duration: 120.0 +requires: + optical_drive_{name}.dvd_write == 'supported' +user: root +command: set -o pipefail; optical_write_test /dev/{name} dvd | ansi_parser +_description: + This is an automated version of optical/dvd-write. It assumes you have already inserted a data DVD into your {product} optical drive prior to running Checkbox. + +plugin: user-interact-verify +category_id: com.canonical.plainbox::optical +id: optical/dvd_playback +command: totem /media/cdrom +estimated_duration: 120.0 +requires: + device.category == 'CDROM' + package.name == 'totem' +_description: + PURPOSE: + This test will check your DVD playback capabilities + STEPS: + 1. Insert a DVD that contains any movie in your optical drive + 2. Click "Test" to play the DVD in Totem + VERIFICATION: + Did the file play? + +unit: template +template-resource: device +template-filter: device.category == 'CDROM' +plugin: user-interact +category_id: com.canonical.plainbox::optical +id: optical/bluray-read_{name} +estimated_duration: 120.0 +requires: + optical_drive_{name}.bd_read == "supported" +user: root +command: optical_read_test /dev/{name} +_description: + PURPOSE: + This test will check your {product} device's ability to read Blu-Ray (BD) media + STEPS: + 1. Insert appropriate non-blank media into your Blu-Ray drive. Movie and Audio Disks may not work. Self-created data disks have the greatest chance of working. + 2. If a file browser window opens, you can safely close or ignore that window. + 3. Click "Test" to begin the test. + VERIFICATION: + This test should automatically select "Yes" if it passes, "No" if it fails. + +unit: template +template-resource: device +template-filter: device.category == 'CDROM' +plugin: user-interact +category_id: com.canonical.plainbox::optical +id: optical/bluray-write_{name} +requires: + optical_drive_{name}.bd_write == "supported" + package.name == "growisofs" +user: root +command: set -o pipefail; optical_write_test /dev/{name} bd | ansi_parser +estimated_duration: 120.00 +_description: + PURPOSE: + This test will check your {product} device's ability to write Blu-Ray (BD) media + STEPS: + Skip this test if you do not have a blank BD-R disc + 1. Insert appropriate writable media into your Blu-Ray drive. + 2. Click "Test" to begin the test. + VERIFICATION: + This test should automatically select "Yes" if it passes, "No" if it fails. + diff --git a/units/optical/test-plan.pxu b/units/optical/test-plan.pxu new file mode 100644 index 0000000..ba63c28 --- /dev/null +++ b/units/optical/test-plan.pxu @@ -0,0 +1,27 @@ +id: optical-cert-full +unit: test plan +_name: Optical drive tests +_description: Optical drive tests +include: + optical/detect certification-status=blocker + optical/read_.* certification-status=blocker + optical/cdrom-write_.* certification-status=blocker + optical/dvd-write_.* certification-status=blocker + optical/bluray-read_.* certification-status=blocker + optical/bluray-write_.* certification-status=blocker +bootstrap_include: + device + +id: optical-cert-blockers +unit: test plan +_name: Optical drive tests (certification blockers only) +_description: Optical drive tests (certification blockers only) +include: + optical/detect certification-status=blocker + optical/read_.* certification-status=blocker + optical/cdrom-write_.* certification-status=blocker + optical/dvd-write_.* certification-status=blocker + optical/bluray-read_.* certification-status=blocker + optical/bluray-write_.* certification-status=blocker +bootstrap_include: + device diff --git a/units/power-management/jobs.pxu b/units/power-management/jobs.pxu new file mode 100644 index 0000000..2521f81 --- /dev/null +++ b/units/power-management/jobs.pxu @@ -0,0 +1,257 @@ +plugin: manual +category_id: com.canonical.plainbox::power-management +id: power-management/shutdown-boot +estimated_duration: 300.0 +_description: + PURPOSE: + This test will check your system shutdown/booting cycle. + STEPS: + 1. Shutdown your machine. + 2. Boot your machine. + 3. Repeat steps 1 and 2 at least 5 times. + VERIFICATION: + Did the system shutdown and rebooted correctly? + +plugin: shell +category_id: com.canonical.plainbox::power-management +id: power-management/fwts_wakealarm +environ: PLAINBOX_SESSION_SHARE +estimated_duration: 25.0 +user: root +_description: Test ACPI Wakealarm (fwts wakealarm) +requires: + package.name == 'fwts' or executable.name == 'fwts' +command: checkbox-support-fwts_test -f aborted -t wakealarm -l $PLAINBOX_SESSION_SHARE/fwts-wakealarm.log + +plugin: attachment +category_id: com.canonical.plainbox::power-management +id: power-management/fwts_wakealarm-log-attach +depends: power-management/fwts_wakealarm +estimated_duration: 2.0 +_description: Attach log from fwts wakealarm test +command: [ -e ${PLAINBOX_SESSION_SHARE}/fwts-wakealarm.log ] && cat ${PLAINBOX_SESSION_SHARE}/fwts-wakealarm.log + +plugin: shell +category_id: com.canonical.plainbox::power-management +id: power-management/poweroff +estimated_duration: 120.0 +depends: power-management/fwts_wakealarm +user: root +environ: PLAINBOX_SESSION_SHARE +requires: + package.name == 'fwts' +command: pm_test --silent --checkbox-respawn-cmd $PLAINBOX_SESSION_SHARE/__respawn_checkbox poweroff --log-level=debug --log-dir=$PLAINBOX_SESSION_SHARE +flags: noreturn autorestart +_description: + This test will check the system's ability to power-off and boot. + +plugin: attachment +category_id: com.canonical.plainbox::power-management +id: power-management/poweroff-log-attach +estimated_duration: 1.0 +command: + set -o pipefail + tar cvz $PLAINBOX_SESSION_SHARE/*poweroff.1.log | base64 +_description: + This will attach any logs from the power-management/poweroff test to the results. + +plugin: shell +category_id: com.canonical.plainbox::power-management +id: power-management/reboot +estimated_duration: 120.0 +user: root +environ: PLAINBOX_SESSION_SHARE +requires: + package.name == 'fwts' +command: pm_test --silent --checkbox-respawn-cmd $PLAINBOX_SESSION_SHARE/__respawn_checkbox reboot --log-level=debug --log-dir=$PLAINBOX_SESSION_SHARE +flags: noreturn autorestart +_description: + This test will check the system's ability to reboot cleanly. + +plugin: attachment +category_id: com.canonical.plainbox::power-management +id: power-management/reboot-log-attach +estimated_duration: 1.0 +command: + set -o pipefail + tar cvz $PLAINBOX_SESSION_SHARE/*reboot.1.log | base64 +_description: + This will attach any logs from the power-management/reboot test to the results. + +plugin: manual +category_id: com.canonical.plainbox::power-management +id: power-management/lid +estimated_duration: 120.0 +requires: dmi.product in ['Notebook','Laptop','Portable'] +_description: + PURPOSE: + This test will check your lid sensors. + STEPS: + 1. Close your laptop lid. + VERIFICATION: + Does closing your laptop lid cause your system to suspend? + +plugin: user-interact-verify +category_id: com.canonical.plainbox::power-management +id: power-management/lid_close +estimated_duration: 60.0 +requires: device.product == 'Lid Switch' +command: + for i in `seq 20`; do + state=`cat /proc/acpi/button/lid/*/state | awk '{print $2}'` + [ "$state" = "closed" ] && exit 0 || sleep 0.5 + done + exit 1 +_description: + PURPOSE: + This test will check your lid sensors + STEPS: + 1. Press "Enter". + 2. Close and open the lid. + VERIFICATION: + Did the screen turn off while the lid was closed? + +plugin: user-interact-verify +category_id: com.canonical.plainbox::power-management +id: power-management/lid_open +estimated_duration: 60.0 +requires: device.product == 'Lid Switch' +command: + for i in `seq 20`; do + state=`cat /proc/acpi/button/lid/*/state | awk '{print $2}'` + [ "$state" = "open" ] && exit 0 || sleep 0.5 + done + exit 1 +_description: + PURPOSE: + This test will check your lid sensors. + STEPS: + 1. Press "Enter". + 2. Close the lid. + 3. Wait 5 seconds with the lid closed. + 4. Open the lid. + VERIFICATION: + Did the system resume when the lid was opened? + +plugin: shell +category_id: com.canonical.plainbox::power-management +id: power-management/rtc +requires: + rtc.state == 'supported' + executable.name == 'hwclock' + cpuinfo.other != 'emulated by qemu' +user: root +command: hwclock -r +estimated_duration: 0.02 +_summary: Test that RTC functions properly (if present) +_description: + Verify that the Real-time clock (RTC) device functions properly, if present. + +plugin: shell +category_id: com.canonical.plainbox::power-management +id: power-management/tickless_idle +estimated_duration: 1.0 +requires: cpuinfo.platform in ('i386', 'x86_64', 'ppc64el', 'pSeries') +_description: Check to see if CONFIG_NO_HZ is set in the kernel (this is just a simple regression check) +command: + zgrep 'CONFIG_NO_HZ=y' /boot/config-`uname -r` >/dev/null 2>&1 || ( echo "WARNING: Tickless Idle is NOT set" >&2 && exit 1 ) + +plugin: manual +category_id: com.canonical.plainbox::power-management +id: power-management/unplug_ac +estimated_duration: 30.0 +_description: + PURPOSE: + This test will ensure that the AC is unplugged for the battery drain tests to run. + STEPS: + 1. Unplug laptop from AC. + VERIFICATION: + Was the laptop unplugged from AC? + +plugin: shell +category_id: com.canonical.plainbox::power-management +id: power-management/battery_drain_idle +estimated_duration: 90.0 +requires: package.name == 'upower' +depends: power-management/unplug_ac +_description: Checks the battery drain during idle. Reports time and capacity until empty. +command: + battery_test -t 90 --idle + +plugin: shell +category_id: com.canonical.plainbox::power-management +id: power-management/battery_drain_movie +estimated_duration: 90.0 +requires: + package.name == 'upower' +depends: power-management/unplug_ac +_description: Checks the battery drain while watching a movie. Reports time and capacity until empty. Requires MOVIE_VAR to be set. +command: + battery_test -t 90 --movie $MOVIE_VAR + +plugin: shell +category_id: com.canonical.plainbox::power-management +id: power-management/battery_drain_sleep +estimated_duration: 140.0 +user: root +requires: + package.name == 'upower' + package.name == 'fwts' +depends: power-management/unplug_ac +_description: Checks the battery drain during suspend. Reports time and capacity until empty. +command: + battery_test -t 120 --sleep + +plugin: manual +category_id: com.canonical.plainbox::power-management +id: power-management/plug_ac +estimated_duration: 30.0 +depends: power-management/battery_drain_idle power-management/battery_drain_movie power-management/battery_drain_sleep +_description: + PURPOSE: + This test will ensure that the AC is plugged back in after the battery tests. + STEPS: + 1. Plug laptop into AC. + VERIFICATION: + Was the laptop plugged into AC? + +plugin: user-interact-verify +category_id: com.canonical.plainbox::power-management +id: power-management/reboot_manual +estimated_duration: 120.0 +user:root +command: shutdown -r now +_description: + PURPOSE: + This test will verify that your system can successfully reboot. + STEPS: + 1. Press 'Enter' to initiate a system reboot. + 2. When the grub boot menu is displayed, boot into Ubuntu (Or allow the + system to automatically boot on its own). + 3. Once the system has restarted, log in and restart checkbox-certification-server. + 4. Select 'Re-Run' to return to this test. + 5. Select Pass to indicate the test has passed if the system rebooted + successfully, otherwise, select 'Fail' to indicate there was a problem. + VERIFICATION: + Did the system reboot correctly? + +plugin: user-interact-verify +category_id: com.canonical.plainbox::power-management +id: power-management/shutdown_manual +estimated_duration: 120.0 +user: root +command: shutdown -h now +_description: + PURPOSE: + This test will check your system shutdown/booting cycle + STEPS: + 1. Press 'Enter' to initiate a system shutdown. + 2. Power the system back on. + 3. From the grub menu, boot into the Xen Hypervisor. + 4. When the system has restarted, log in and restart checkbox-certification-server. + 5. Select 'Re-Run' to return to this test. + 6. Select Pass to indicate the test has passed if the machine shut down + successfully otherwise, Select 'Fail' to indicate there was a problem. + VERIFICATION: + Did the system shutdown and boot correctly? + diff --git a/units/power-management/test-plan.pxu b/units/power-management/test-plan.pxu new file mode 100644 index 0000000..2cd1a76 --- /dev/null +++ b/units/power-management/test-plan.pxu @@ -0,0 +1,59 @@ +id: power-management-precheck-cert +unit: test plan +_name: Power Management precheck tests +_description: Power Management precheck tests +include: +nested_part: + power-management-precheck-cert-automated + +id: power-management-reboot-poweroff-cert +unit: test plan +_name: Power Management reboot and power off tests +_description: Power Management reboot and power off tests +include: +nested_part: + power-management-reboot-poweroff-cert-automated + +id: power-management-precheck-cert-automated +unit: test plan +_name: Power Management precheck tests (automated) +_description: Power Management precheck tests (automated) +include: + power-management/rtc certification-status=blocker + power-management/tickless_idle certification-status=blocker + power-management/fwts_wakealarm certification-status=blocker + power-management/fwts_wakealarm-log-attach + +id: power-management-reboot-poweroff-cert-automated +unit: test plan +_name: Power Management reboot and power off tests (automated) +_description: Power Management reboot and power off tests (automated) +include: + power-management/poweroff certification-status=blocker + power-management/poweroff-log-attach + power-management/reboot certification-status=blocker + power-management/reboot-log-attach + +id: power-management-precheck-cert-blockers +unit: test plan +_name: + Power Management precheck tests (certification blockers only) +_description: + Power Management precheck tests (certification blockers only) +include: + power-management/rtc certification-status=blocker + power-management/tickless_idle certification-status=blocker + power-management/fwts_wakealarm certification-status=blocker + power-management/fwts_wakealarm-log-attach + +id: power-management-reboot-poweroff-cert-blockers +unit: test plan +_name: + Power Management reboot and power off tests (certification blockers only) +_description: + Power Management reboot and power off tests (certification blockers only) +include: + power-management/poweroff certification-status=blocker + power-management/poweroff-log-attach + power-management/reboot certification-status=blocker + power-management/reboot-log-attach diff --git a/units/smoke/jobs.pxu b/units/smoke/jobs.pxu new file mode 100644 index 0000000..0e16299 --- /dev/null +++ b/units/smoke/jobs.pxu @@ -0,0 +1,51 @@ +plugin: shell +id: smoke/true +command: true +_description: + Check success result from shell test case + +plugin: shell +id: smoke/false +command: false +_description: + Check failed result from shell test case + +plugin: shell +id: smoke/dependency/good +depends: smoke/true +command: true +_description: + Check job is executed when dependency succeeds + +plugin: shell +id: smoke/dependency/bad +depends: smoke/false +command: true +_description: + Check job result is set to uninitiated when dependency fails + +plugin: shell +id: smoke/requirement/good +requires: package.name == "checkbox" +command: true +_description: + Check job is executed when requirements are met + +plugin: shell +id: smoke/requirement/bad +requires: package.name == "unknown-package" +command: true +_description: + Check job result is set to "not required on this system" when requirements are not met + +plugin: manual +id: smoke/manual +_description: + PURPOSE: + This test checks that the manual plugin works fine + STEPS: + 1. Add a comment + 2. Set the result as passed + VERIFICATION: + Check that in the report the result is passed and the comment is displayed + diff --git a/units/smoke.pxu b/units/smoke/test-plan.pxu index bcd69c6..bcd69c6 100644 --- a/units/smoke.pxu +++ b/units/smoke/test-plan.pxu diff --git a/units/stress/jobs.pxu b/units/stress/jobs.pxu new file mode 100644 index 0000000..d1a8146 --- /dev/null +++ b/units/stress/jobs.pxu @@ -0,0 +1,456 @@ +plugin: shell +category_id: com.canonical.plainbox::stress +id: stress/cpu_stress_test +estimated_duration: 7200.0 +requires: + package.name == 'stress' +user: root +command: num_vm=$(awk '/MemTotal/ {x=$2/262144; print ((x == int(x)) ? x : int(x) +1)}' /proc/meminfo); vm_bytes=$(($(awk '/MemTotal/ {print int($2/1024)}' /proc/meminfo)/$num_vm/4))M; stress --cpu `cpuinfo_resource | awk '/count:/ {print $2}'` --vm $num_vm --vm-bytes $vm_bytes --timeout 7200s +_description: + Simulate high system load using the 'stress' tool to exercise the CPU for + several hours. The test is considered passed if the system does not freeze or + abend. + +plugin: shell +category_id: com.canonical.plainbox::stress +id: stress/cpu_stress_ng_test +estimated_duration: 7200.0 +requires: + package.name == 'stress-ng' or executable.name == 'stress-ng' +user: root +command: cpu_stress --runtime 7200 +_summary: + Stress of CPUs (very long runtime) +_description: + Impose a high system load using the 'stress_ng' tool to exercise the CPU for + several hours. The test is considered passed if the system does not freeze and + if the stress_ng tool does not report errors. + +plugin: shell +category_id: com.canonical.plainbox::stress +id: power-management/hibernate_30_cycles +estimated_duration: 5400.00 +depends: + power-management/hibernate_advanced +requires: + sleep.disk == 'supported' + rtc.state == 'supported' +environ: PLAINBOX_SESSION_SHARE +user: root +command: + if type -P fwts >/dev/null; then + echo "Calling fwts" + checkbox-support-fwts_test -l $PLAINBOX_SESSION_SHARE/hibernate_30_cycles -f none -s s4 --s4-device-check --s4-device-check-delay=45 --s4-sleep-delay=120 --s4-multiple=30 + else + echo "Calling sleep_test" + set -o pipefail; sleep_test -s disk -i 30 -w 120 | tee $PLAINBOX_SESSION_SHARE/hibernate_30_cycles.log + fi +_description: + PURPOSE: + This is an automated stress test that will force the system to hibernate/resume for 30 cycles + +plugin: shell +category_id: com.canonical.plainbox::stress +id: power-management/hibernate-30-cycles-log-check +estimated_duration: 1.0 +command: [ -e $PLAINBOX_SESSION_SHARE/hibernate_30_cycles.log ] && sleep_test_log_check -v s4 $PLAINBOX_SESSION_SHARE/hibernate_30_cycles.log +_description: + Automated check of the 30 cycle hibernate log for errors detected by fwts. + +plugin: attachment +category_id: com.canonical.plainbox::stress +id: power-management/hibernate-30-cycle-log-attach +estimated_duration: 1.0 +command: [ -e $PLAINBOX_SESSION_SHARE/hibernate_30_cycles.log ] && cat $PLAINBOX_SESSION_SHARE/hibernate_30_cycles.log +_description: + Attaches the log from the 30 cycle Hibernate/Resume test if it exists + +plugin: shell +category_id: com.canonical.plainbox::stress +id: power-management/suspend_30_cycles_with_reboots +estimated_duration: 5400.0 +depends: + power-management/rtc + suspend/suspend_advanced +requires: + executable.name == 'x-terminal-emulator' +flags: noreturn autorestart +user: root +command: + pm_test reboot --checkbox-respawn-cmd $PLAINBOX_SESSION_SHARE/__respawn_checkbox --fwts --log-level=debug --log-dir=$PLAINBOX_SESSION_SHARE --suspends-before-reboot=30 -r 3 --silent +_description: + This is an automated stress test that will run a sequence of '30 suspend/resume cycles and one reboot' 3 times. + +plugin: shell +category_id: com.canonical.plainbox::stress +id: power-management/suspend_30_cycles +estimated_duration: 2400.0 +depends: + power-management/rtc + suspend/suspend_advanced +environ: PLAINBOX_SESSION_SHARE +user: root +command: + if type -P fwts >/dev/null; then + echo "Calling fwts" + set -o pipefail; checkbox-support-fwts_test -l $PLAINBOX_SESSION_SHARE/suspend_30_cycles -f none -s s3 --s3-device-check --s3-device-check-delay=45 --s3-sleep-delay=30 --s3-multiple=30 | tee $PLAINBOX_SESSION_SHARE/suspend_30_cycles_times.log + else + echo "Calling sleep_test" + set -o pipefail; sleep_test -p -s mem -i 30 | tee $PLAINBOX_SESSION_SHARE/suspend_30_cycles.log + fi +_description: + PURPOSE: + This is an automated stress test that will force the system to suspend/resume for 30 cycles. + +plugin: shell +category_id: com.canonical.plainbox::stress +id: power-management/suspend-30-cycles-log-check +depends: power-management/suspend_30_cycles +estimated_duration: 1.0 +command: [ -e $PLAINBOX_SESSION_SHARE/suspend_30_cycles.log ] && sleep_test_log_check -v s3 $PLAINBOX_SESSION_SHARE/suspend_30_cycles.log +_description: + Automated check of the 30 cycle suspend log for errors detected by fwts. + +plugin: shell +category_id: com.canonical.plainbox::stress +id: power-management/suspend-30-cycles-log-check-with-reboots +depends: power-management/suspend_30_cycles_with_reboots +estimated_duration: 1.0 +command: [ -e $PLAINBOX_SESSION_SHARE/pm_test.reboot.3.log ] && sleep_test_log_check -v s3 $PLAINBOX_SESSION_SHARE/pm_test.reboot.3.log +_description: + Automated check of the '30 cycle suspend and 1 reboot times 3' logs for errors detected by fwts. + +plugin: attachment +category_id: com.canonical.plainbox::stress +id: power-management/suspend-30-cycle-log-attach +estimated_duration: 1.0 +depends: power-management/suspend_30_cycles +command: [ -e $PLAINBOX_SESSION_SHARE/suspend_30_cycles.log ] && xz -c $PLAINBOX_SESSION_SHARE/suspend_30_cycles.log | base64 +_description: + Attaches the log from the 30 cycle Suspend/Resume test if it exists + +plugin: attachment +category_id: com.canonical.plainbox::stress +id: power-management/suspend-30-cycle-log-attach-with-reboots +estimated_duration: 1.0 +depends: power-management/suspend_30_cycles_with_reboots +command: [ -e $PLAINBOX_SESSION_SHARE/pm_test.reboot.3.log ] && xz -c $PLAINBOX_SESSION_SHARE/pm_test.reboot.3.log | base64 +_description: + Attaches the log from the '30 cycle Suspend/Resume and one reboot times 3' test if it exists + +plugin: shell +category_id: com.canonical.plainbox::stress +id: power-management/suspend-30-cycles-time-check +estimated_duration: 1.0 +depends: power-management/suspend_30_cycles +command: [ -e $PLAINBOX_SESSION_SHARE/suspend_30_cycles_times.log ] && sleep_time_check $PLAINBOX_SESSION_SHARE/suspend_30_cycles_times.log +_description: + Checks the sleep times to ensure that a machine suspends and resumes within a given threshold + +plugin: shell +category_id: com.canonical.plainbox::stress +id: power-management/suspend-30-cycles-time-check-with-reboots +estimated_duration: 1.0 +depends: power-management/suspend_30_cycles_with_reboots +command: [ -e $PLAINBOX_SESSION_SHARE/pm_test.reboot.3.log ] && sleep_time_check $PLAINBOX_SESSION_SHARE/pm_test.reboot.3.log +_description: + Checks the sleep times to ensure that a machine suspends and resumes within a given threshold + +plugin: shell +category_id: com.canonical.plainbox::stress +id: stress/hibernate_250_cycles +estimated_duration: 43400.0 +depends: power-management/rtc +environ: PLAINBOX_SESSION_SHARE +user: root +command: + if type -P fwts >/dev/null; then + echo "Calling fwts" + checkbox-support-fwts_test -l $PLAINBOX_SESSION_SHARE/hibernate_250_cycles -s s4 --s4-device-check --s4-device-check-delay=45 --s4-sleep-delay=120 --s4-multiple=250 + else + echo "Calling sleep_test" + set -o pipefail; sleep_test -s disk -i 250 -w 120 | tee $PLAINBOX_SESSION_SHARE/hibernate_250_cycles.log + fi +_description: + PURPOSE: + This is an automated stress test that will force the system to hibernate/resume for 250 cycles + +plugin: attachment +category_id: com.canonical.plainbox::stress +id: stress/hibernate-250-cycle-log-attach +estimated_duration: 1.0 +command: [ -e $PLAINBOX_SESSION_SHARE/hibernate_250_cycles.log ] && cat $PLAINBOX_SESSION_SHARE/hibernate_250_cycles.log +_description: + Attaches the log from the 250 cycle Hibernate/Resume test if it exists + +plugin: shell +category_id: com.canonical.plainbox::stress +id: stress/suspend_250_cycles +estimated_duration: 18750.0 +depends: power-management/rtc +environ: PLAINBOX_SESSION_SHARE +user: root +command: + if type -P fwts >/dev/null; then + echo "Calling fwts" + set -o pipefail; checkbox-support-fwts_test -l $PLAINBOX_SESSION_SHARE/suspend_250_cycles -s s3 --s3-device-check --s3-device-check-delay=45 --s3-sleep-delay=30 --s3-multiple=250 | tee $PLAINBOX_SESSION_SHARE/suspend_250_cycles_times.log + else + echo "Calling sleep_test" + set -o pipefail; sleep_test -p -s mem -i 250 | tee $PLAINBOX_SESSION_SHARE/suspend_250_cycles.log + fi +_description: + PURPOSE: + This is an automated stress test that will force the system to suspend/resume for 250 cycles. + +plugin: attachment +category_id: com.canonical.plainbox::stress +id: stress/suspend-250-cycle-log-attach +estimated_duration: 1.0 +command: [ -e $PLAINBOX_SESSION_SHARE/suspend_250_cycles.log ] && cat $PLAINBOX_SESSION_SHARE/suspend_250_cycles.log +_description: + Attaches the log from the 250 cycle Suspend/Resume test if it exists + +plugin: shell +category_id: com.canonical.plainbox::stress +id: stress/suspend-250-cycles-time-check +estimated_duration: 1.0 +command: [ -e $PLAINBOX_SESSION_SHARE/suspend_250_cycles_times.log ] && sleep_time_check $PLAINBOX_SESSION_SHARE/suspend_250_cycles_times.log +_description: + Checks the sleep times to ensure that a machine suspends and resumes within a given threshold + +plugin: shell +category_id: com.canonical.plainbox::stress +id: stress/reboot +estimated_duration: 4500.0 +requires: + package.name == 'fwts' +command: pm_test --checkbox-respawn-cmd $PLAINBOX_SESSION_SHARE/__respawn_checkbox -r 100 --silent --log-level=notset reboot --log-dir=$PLAINBOX_SESSION_SHARE +flags: noreturn autorestart +user: root +environ: PLAINBOX_SESSION_SHARE +_description: + Stress reboot system (100 cycles) + +plugin: attachment +category_id: com.canonical.plainbox::stress +id: stress/reboot_log +estimated_duration: 1.0 +depends: stress/reboot +command: + set -o pipefail + tar cvz $PLAINBOX_SESSION_SHARE/*reboot.100.log | base64 + +plugin: shell +category_id: com.canonical.plainbox::stress +id: stress/reboot_30 +requires: + package.name == 'fwts' + executable.name == 'x-terminal-emulator' +command: pm_test --checkbox-respawn-cmd $PLAINBOX_SESSION_SHARE/__respawn_checkbox -r 30 --silent --log-level=notset reboot --log-dir=$PLAINBOX_SESSION_SHARE +flags: noreturn autorestart +estimated_duration: 2700 +user: root +environ: PLAINBOX_SESSION_SHARE +_description: + Stress reboot system (30 cycles) + +plugin: attachment +category_id: com.canonical.plainbox::stress +id: stress/reboot_30_log +depends: stress/reboot_30 +command: + set -o pipefail + tar cvz $PLAINBOX_SESSION_SHARE/*reboot.30.log | base64 + +plugin: shell +category_id: com.canonical.plainbox::stress +id: stress/poweroff +estimated_duration: 4500.0 +requires: + package.name == 'fwts' + executable.name == 'x-terminal-emulator' +command: pm_test --checkbox-respawn-cmd $PLAINBOX_SESSION_SHARE/__respawn_checkbox --r 100 --silent --log-level=notset poweroff --log-dir=$PLAINBOX_SESSION_SHARE +flags: noreturn autorestart +user: root +environ: PLAINBOX_SESSION_SHARE +_description: + Stress poweroff system (100 cycles) + +plugin: attachment +category_id: com.canonical.plainbox::stress +id: stress/poweroff_log +estimated_duration: 1.0 +depends: stress/poweroff +command: + set -o pipefail + tar cvz $PLAINBOX_SESSION_SHARE/*poweroff.100.log | base64 + +plugin: shell +category_id: com.canonical.plainbox::stress +id: stress/poweroff_30 +requires: + package.name == 'fwts' + executable.name == 'x-terminal-emulator' +command: pm_test --checkbox-respawn-cmd $PLAINBOX_SESSION_SHARE/__respawn_checkbox --r 30 --silent --log-level=notset poweroff --log-dir=$PLAINBOX_SESSION_SHARE +flags: noreturn autorestart +estimated_duration: 3600 +user: root +environ: PLAINBOX_SESSION_SHARE +_description: + Stress poweroff system (30 cycles) + +plugin: attachment +category_id: com.canonical.plainbox::stress +id: stress/poweroff_30_log +depends: stress/poweroff_30 +command: + set -o pipefail + tar cvz $PLAINBOX_SESSION_SHARE/*poweroff.30.log | base64 + +plugin: shell +category_id: com.canonical.plainbox::stress +id: stress/reboot_30_check +estimated_duration: 1.0 +depends: stress/reboot_30 +command: pm_log_check --log-level=notset $PLAINBOX_SESSION_SHARE/pm_test.reboot.30.log $PLAINBOX_SESSION_SHARE/pm_log_check_reboot.30.log +_description: Check logs for the stress reboot (30 cycles) test case + +plugin: attachment +category_id: com.canonical.plainbox::stress +id: stress/reboot_30_check_log +estimated_duration: 1.0 +depends: stress/reboot_30_check +command: + set -o pipefail + tar cvz $PLAINBOX_SESSION_SHARE/pm_log_check_reboot.30.log | base64 + +plugin: shell +category_id: com.canonical.plainbox::stress +id: stress/poweroff_30_check +estimated_duration: 1.0 +depends: stress/poweroff_30 +command: pm_log_check --log-level=notset $PLAINBOX_SESSION_SHARE/pm_test.poweroff.30.log $PLAINBOX_SESSION_SHARE/pm_log_check_poweroff.30.log +_description: Check logs for the stress poweroff (30 cycles) test case + +plugin: attachment +category_id: com.canonical.plainbox::stress +id: stress/poweroff_30_check_log +estimated_duration: 1.0 +depends: stress/poweroff_30_check +command: + set -o pipefail + tar cvz $PLAINBOX_SESSION_SHARE/pm_log_check_poweroff.30.log | base64 + +plugin: shell +category_id: com.canonical.plainbox::stress +id: stress/reboot_check +estimated_duration: 1.0 +depends: stress/reboot +command: pm_log_check --log-level=notset $PLAINBOX_SESSION_SHARE/pm_test.reboot.100.log $PLAINBOX_SESSION_SHARE/pm_log_check_reboot.100.log +_description: Check logs for the stress reboot (100 cycles) test case + +plugin: attachment +category_id: com.canonical.plainbox::stress +id: stress/reboot_check_log +estimated_duration: 1.0 +depends: stress/reboot_check +command: + set -o pipefail + tar cvz $PLAINBOX_SESSION_SHARE/pm_log_check_reboot.100.log | base64 + +plugin: shell +category_id: com.canonical.plainbox::stress +id: stress/poweroff_check +estimated_duration: 1.0 +depends: stress/poweroff +command: pm_log_check --log-level=notset $PLAINBOX_SESSION_SHARE/pm_test.poweroff.100.log $PLAINBOX_SESSION_SHARE/pm_log_check_poweroff.100.log +_description: Check logs for the stress poweroff (100 cycles) test case + +plugin: attachment +category_id: com.canonical.plainbox::stress +id: stress/poweroff_check_log +estimated_duration: 1.0 +depends: stress/poweroff_check +command: + set -o pipefail + tar cvz $PLAINBOX_SESSION_SHARE/pm_log_check_poweroff.100.log | base64 + +plugin: shell +category_id: com.canonical.plainbox::stress +id: stress/graphics +estimated_duration: 600.0 +requires: + package.name == 'x11-apps' +user: root +environ: PLAINBOX_SESSION_SHARE +command: graphics_stress_test --iterations 20 -b repeat -d -o $PLAINBOX_SESSION_SHARE/graphics-stress-results && echo "Graphics Stress Test completed successfully" || echo "Graphics Stress Test completed, but there are errors. Please see the log $PLAINBOX_SESSION_SHARE/graphics-stress-results for details" && false +_description: + Run the graphics stress test. This test can take a few minutes. + +plugin: attachment +category_id: com.canonical.plainbox::stress +id: stress/graphics-tarball +estimated_duration: 1.0 +requires: + package.name == 'x11-apps' +command: [ -f $PLAINBOX_SESSION_SHARE/graphics-stress-results ] && tar cvz $PLAINBOX_SESSION_SHARE/graphics-stress-results | base64 +_description: Attaches the graphics stress results to the submission. + +plugin: shell +category_id: com.canonical.plainbox::stress +id: stress/usb +estimated_duration: 710.0 +user: root +command: removable_storage_test -s 10240000 -c 100 -i 3 usb +_description: Runs a test that transfers 100 10MB files 3 times to usb. + +plugin: user-interact +category_id: com.canonical.plainbox::stress +id: stress/sdhc +user: root +_summary: Stress test for SDHC card +estimated_duration: 780.0 +command: removable_storage_test -s 10240000 -c 100 -i 3 sdio scsi usb --memorycard +_description: + PURPOSE: + This test will transfers 100 10MB files 3 times to a SDHC card, to + check that the systems media card reader can transfer large amounts + of data. + STEPS: + 1. Insert a SDHC card into the reader and then Click "Test". + If a file browser opens up, you can safely close it. + 2. Do not remove the device during this test. + VERIFICATION: + The verification of this test is automated. Do not change the + automatically selected result. + +plugin: shell +category_id: com.canonical.plainbox::stress +id: stress/network_restart +estimated_duration: 1500.0 +user: root +environ: PLAINBOX_SESSION_SHARE +command: network_restart -t 100 -o $PLAINBOX_SESSION_SHARE +_description: Ping ubuntu.com and restart network interfaces 100 times + +plugin: attachment +category_id: com.canonical.plainbox::stress +id: stress/network_restart_log +estimated_duration: 1.0 +depends: stress/network_restart +command: file=$PLAINBOX_SESSION_SHARE/network_restart.log; if [ -e "$file" ]; then iconv -t 'ascii' -c "$file"; fi + +plugin: manual +category_id: com.canonical.plainbox::stress +id: stress/wireless_hotkey +estimated_duration: 60.0 +requires: dmi.product in ['Notebook','Laptop','Portable'] +_description: + PURPOSE: + To make sure that stressing the wifi hotkey does not cause applets to disappear from the panel or the system to lock up + STEPS: + 1. Log in to desktop + 2. Press wifi hotkey at a rate of 1 press per second and slowly increase the speed of the tap, until you are tapping as fast as possible + VERIFICATION: + Verify the system is not frozen and the wifi and bluetooth applets are still visible and functional + diff --git a/units/stress/packaging.pxu b/units/stress/packaging.pxu new file mode 100644 index 0000000..708944e --- /dev/null +++ b/units/stress/packaging.pxu @@ -0,0 +1,4 @@ +# The stress/cpu_stress_ng_test job requires stress-ng package +unit: packaging meta-data +os-id: debian +Depends: stress-ng (>= 0.04) diff --git a/units/stress/test-plan.pxu b/units/stress/test-plan.pxu new file mode 100644 index 0000000..5f72916 --- /dev/null +++ b/units/stress/test-plan.pxu @@ -0,0 +1,55 @@ +id: stress-cert-full +unit: test plan +_name: Stress tests +_description: Stress tests +include: +nested_part: + stress-cert-automated + +id: stress-cert-automated +unit: test plan +_name: Stress tests (automated) +_description: Stress tests (automated) +include: + power-management/suspend_30_cycles certification-status=blocker + power-management/suspend-30-cycle-log-attach + power-management/suspend-30-cycles-time-check certification-status=non-blocker + stress/cpu_stress_ng_test certification-status=blocker + +id: stress-cert-blockers +unit: test plan +_name: Stress tests (certification blockers only) +_description: Stress tests (certification blockers only) +include: + power-management/suspend_30_cycles certification-status=blocker + power-management/suspend-30-cycle-log-attach + stress/cpu_stress_ng_test certification-status=blocker + +id: stress-suspend-30-cycles-with-reboots-automated +unit: test plan +_name: Suspend stress tests (with reboots) +_description: Suspend stress tests (with reboots) +include: + power-management/suspend_30_cycles_with_reboots + power-management/suspend-30-cycles-log-check-with-reboots + power-management/suspend-30-cycle-log-attach-with-reboots + power-management/suspend-30-cycles-time-check-with-reboots + +id: stress-30-reboot-poweroff-automated +unit: test plan +_name: Power Management reboot and power off stress tests (automated) +_description: Power Management reboot and power off stress tests (automated) +include: + stress/reboot_30 + stress/reboot_30_log + stress/poweroff_30 + stress/poweroff_30_log + +id: stress-ng-cert-automated +unit: test plan +_name: Stress NG tests (automated) +_description: Stress NG tests (automated) +include: + stress/cpu_stress_ng_test certification-status=blocker + memory/memory_stress_ng certification-status=blocker + diff --git a/units/submission/jobs.pxu b/units/submission/jobs.pxu new file mode 100644 index 0000000..4c929b0 --- /dev/null +++ b/units/submission/jobs.pxu @@ -0,0 +1,85 @@ +id: dkms_info_json +plugin: attachment +category_id: com.canonical.plainbox::info +requires: + package.name == 'dkms' +command: + dkms_info --format json | plainbox dev parse dkms-info | \ + jq --indent 4 '.dkms_info' +_description: Attaches json dumps of installed dkms package information. +_summary: Attaches json dumps of installed dkms package information. + +id: udev_json +plugin: attachment +category_id: com.canonical.plainbox::info +command: udevadm info --export-db | plainbox dev parse udevadm +_description: Attaches json dumps of udev_resource + +id: raw_devices_dmi_json +plugin: attachment +category_id: com.canonical.plainbox::info +requires: + package.name == 'dmidecode' or executable.name == 'dmidecode' + dmi_present.state == 'supported' +user: root +command: + dmidecode -t bios -t system | plainbox dev parse dmidecode | \ + jq --indent 4 '[.[0]."_attributes" + + {"category": .[0]."category"}, .[1]."_attributes" + + {"category": .[1]."category"}]' +estimated_duration: 1 +_description: Attaches dmidecode output +_summary: Attaches json dumps of udev_resource raw dmi devices + +id: modprobe_json +plugin: attachment +category_id: com.canonical.plainbox::info +command: + find /etc/modprobe.* -name \*.conf | xargs cat | plainbox dev parse modprobe | + jq --indent 4 'to_entries | map({"module": .key, "options": .value})' +estimated_duration: 0.015 +_description: Attaches the contents of the various modprobe conf files. +_summary: Attach the contents of /etc/modprobe.* + +id: lspci_standard_config_json +_summary: Attach PCI configuration space hex dump +plugin: attachment +category_id: com.canonical.plainbox::info +command: + lspci -x | plainbox dev parse pci-subsys-id | \ + jq --indent 4 '.pci_subsystem_id' +estimated_duration: 0.1 +_description: Attaches a hex dump of the standard part of the PCI configuration + space for all PCI devices. + +id: dell_bto_xml_attachment_json +plugin: attachment +category_id: com.canonical.plainbox::info +depends: info/detect_dell_recovery +requires: package.name == 'dell-recovery' +user: root +estimated_duration: 0.5 +_summary: Attach the bto.xml in Dell recovery partition +_description: + bto.xml is basically a Fish manifest + The information include: + - fish packages + - dell recovery stage 2 boot log +command: recovery_info file bto.xml | plainbox dev parse bto + +id: recovery_info_attachment_json +plugin: attachment +category_id: com.canonical.plainbox::info +depends: info/detect_recovery +requires: package.name == 'dell-recovery' or package.name == 'ubuntu-recovery' +user: root +estimated_duration: 0.2 +_summary: Attach the recovery partition versions +_description: + Attach the recovery partition version + image_version is the preinstalled OS image version + bto_version is only for dell_recovery + Example: + image_version: somerville-trusty-amd64-20140620-0 + bto_version: A00_dell-bto-trusty-houston-15-A11-iso-20141203-0.iso +command: recovery_info version | plainbox dev parse recovery-info diff --git a/units/submission/packaging.pxu b/units/submission/packaging.pxu new file mode 100644 index 0000000..ce8cc1c --- /dev/null +++ b/units/submission/packaging.pxu @@ -0,0 +1,7 @@ +unit: packaging meta-data +os-id: debian +Depends: jq + +unit: packaging meta-data +os-id: debian +Depends: plainbox diff --git a/units/submission/test-plan.pxu b/units/submission/test-plan.pxu new file mode 100644 index 0000000..d23d977 --- /dev/null +++ b/units/submission/test-plan.pxu @@ -0,0 +1,10 @@ +id: submission-cert-full +unit: test plan +_name: Submission resources +_description: Submission resources +include: +mandatory_include: + com.canonical.plainbox::collect-manifest + miscellanea/device_check + # Meta-job to include required resources, don't remove. + miscellanea/submission-resources diff --git a/units/suspend/suspend-graphics.pxu b/units/suspend/suspend-graphics.pxu new file mode 100644 index 0000000..c85d147 --- /dev/null +++ b/units/suspend/suspend-graphics.pxu @@ -0,0 +1,201 @@ +unit: template +template-resource: graphics_card +template-filter: graphics_card.prime_gpu_offload == 'Off' +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/{index}_resolution_before_suspend_{product_slug}_xenial +after: graphics/{index}_switch_card_{product_slug}_xenial +estimated_duration: 1.2 +_description: Record the current resolution before suspending. +command: + source graphics_env {driver} {index} + xrandr -q |grep '*'| awk '{{print $1}}' > $PLAINBOX_SESSION_SHARE/{index}_resolution_before_suspend.txt + +unit: template +template-resource: graphics_card +template-filter: graphics_card.prime_gpu_offload == 'Off' +plugin: user-interact-verify +category_id: com.canonical.plainbox::suspend +id: suspend/{index}_suspend_after_switch_to_card_{product_slug}_xenial +requires: + sleep.mem == 'supported' + rtc.state == 'supported' +after: graphics/{index}_switch_card_{product_slug}_xenial +user: root +environ: PLAINBOX_SESSION_SHARE +command: + if type -P fwts >/dev/null; then + echo "Calling fwts" + set -o pipefail; checkbox-support-fwts_test -f none -l $PLAINBOX_SESSION_SHARE/{index}_suspend_single -s s3 --s3-sleep-delay=30 --s3-device-check --s3-device-check-delay=45 | tee $PLAINBOX_SESSION_SHARE/{index}_suspend_single_times.log + else + echo "Calling sleep_test" + set -o pipefail; sleep_test -p | tee $PLAINBOX_SESSION_SHARE/{index}_suspend_single_times.log + fi +estimated_duration: 90.0 +_summary: Test suspend/resume after switching to {vendor} {product} +_description: + PURPOSE: + This test will check suspend and resume after switching to {vendor} {product} graphics card. + STEPS: + 1. Ensure you have switched to {vendor} {product} graphics card. + 2. Click "Test" and your system will suspend for about 30 - 60 seconds + 3. Observe the Power LED to see if it blinks or changes color during suspend + 4. If your system does not wake itself up after 60 seconds, please press the power button momentarily to wake the system manually + 5. If your system fails to wake at all and must be rebooted, restart System Testing after reboot and mark this test as Failed + VERIFICATION: + Did your system suspend and resume correctly after switching to {vendor} {product} graphics card? + (NOTE: Please only consider whether the system successfully suspended and resumed. Power/Suspend LED verification will occur after this test is completed.) + +unit: template +template-resource: graphics_card +template-filter: graphics_card.prime_gpu_offload == 'Off' +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/{index}_resolution_after_suspend_{product_slug}_xenial +estimated_duration: 1.2 +depends: suspend/{index}_suspend_after_switch_to_card_{product_slug}_xenial suspend/{index}_resolution_before_suspend_{product_slug}_xenial +_description: Test to see that we have the same resolution after resuming as before. +command: + source graphics_env {driver} {index} + xrandr -q |grep '*'| awk '{{print $1}}' | diff $PLAINBOX_SESSION_SHARE/{index}_resolution_before_suspend.txt - + +unit: template +template-resource: graphics_card +template-filter: graphics_card.prime_gpu_offload == 'Off' +plugin: manual +category_id: com.canonical.plainbox::suspend +id: suspend/{index}_display_after_suspend_{product_slug}_xenial +depends: suspend/{index}_suspend_after_switch_to_card_{product_slug}_xenial +_summary: Test display function after suspend for {vendor} {product} +_description: + PURPOSE: + This test will check that the display is correct after suspend and resume on the {vendor} {product} graphics card. + STEPS: + 1. Check that your display does not show up visual artifacts after resuming. + VERIFICATION: + Does the display work normally after resuming from suspend using the {vendor} {product} graphics card? + +unit: template +template-resource: graphics_card +template-filter: graphics_card.prime_gpu_offload == 'Off' +plugin: user-interact-verify +category_id: com.canonical.plainbox::suspend +id: suspend/{index}_cycle_resolutions_after_suspend_{product_slug}_xenial +requires: package.name == 'xorg' +depends: suspend/{index}_suspend_after_switch_to_card_{product_slug}_xenial +estimated_duration: 120.0 +command: + source graphics_env {driver} {index} + xrandr_cycle --keyword={index}_after_suspend --screenshot-dir $PLAINBOX_SESSION_SHARE +_description: + PURPOSE: + This test will cycle through the detected display modes + STEPS: + 1. Click "Test" and the display will cycle trough the display modes + VERIFICATION: + Did your display look fine in the detected mode? + +unit: template +template-resource: graphics_card +template-filter: graphics_card.prime_gpu_offload == 'Off' +plugin: attachment +category_id: com.canonical.plainbox::suspend +id: suspend/{index}_xrandr_screens_after_suspend.tar.gz_xenial +depends: suspend/{index}_cycle_resolutions_after_suspend_{product_slug}_xenial +command: [ -f $PLAINBOX_SESSION_SHARE/{index}_xrandr_screens_after_suspend.tgz ] && base64 $PLAINBOX_SESSION_SHARE/{index}_xrandr_screens_after_suspend.tgz +_description: This attaches screenshots from the suspend/cycle_resolutions_after_suspend test to the results submission. + +unit: template +template-resource: graphics_card +template-filter: graphics_card.prime_gpu_offload == 'Off' +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/{index}_compiz_check_after_suspend_{product_slug}_xenial +depends: suspend/{index}_suspend_after_switch_to_card_{product_slug}_xenial +requires: package.name == 'nux-tools' +command: + source graphics_env {driver} {index} + ! /usr/lib/nux/unity_support_test -c -p 2>&1 | ansi_parser | grep -e ":\(\s\+\)no$" -ie "error" +estimated_duration: 0.130 +_description: Check that {product} hardware is able to run compiz after suspend +_summary: Test Compiz support for {product} after suspend + +unit: template +template-resource: graphics_card +plugin: user-interact-verify +category_id: com.canonical.plainbox::suspend +id: suspend/{index}_glxgears_after_suspend_{product_slug}_xenial +depends: suspend/{index}_suspend_after_switch_to_card_{product_slug}_xenial +requires: package.name == 'mesa-utils' +command: + source graphics_env {driver} {index} + glxgears + true +_summary: Test that glxgears works for {vendor} {product} after suspend +_description: + PURPOSE: + This test tests the basic 3D capabilities of your {vendor} {product} video card after suspend + STEPS: + 1. Click "Test" to execute an OpenGL demo. Press ESC at any time to close. + 2. Verify that the animation is not jerky or slow. + VERIFICATION: + 1. Did the 3d animation appear? + 2. Was the animation free from slowness/jerkiness? + +unit: template +template-resource: graphics_card +id: suspend/{index}_video_after_suspend_{product_slug}_xenial +depends: suspend/{index}_suspend_after_switch_to_card_{product_slug}_xenial +plugin: user-interact-verify +category_id: com.canonical.plainbox::suspend +requires: + package.name == 'xorg' + package.name == 'gir1.2-gst-plugins-base-0.10' or package.name == 'gir1.2-gst-plugins-base-1.0' +command: + source graphics_env {driver} {index} + gst_pipeline_test -t 2 'videotestsrc ! videoconvert ! autovideosink' || gst_pipeline_test -t 2 'videotestsrc ! ffmpegcolorspace ! autovideosink' +_summary: Test that video can be displayed after suspend with {vendor} {product} +_description: + PURPOSE: + This test will test the default display after suspend with a sample video + STEPS: + 1. Click "Test" to display a video test. + VERIFICATION: + Do you see color bars and static? + +unit: template +template-resource: graphics_card +template-filter: graphics_card.prime_gpu_offload == 'Off' +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/{index}_driver_version_after_suspend_{product_slug}_xenial +depends: suspend/{index}_suspend_after_switch_to_card_{product_slug}_xenial +command: + source graphics_env {driver} {index} + graphics_driver +estimated_duration: 0.500 +_description: Parses Xorg.0.Log and discovers the running X driver and version after suspend for the {vendor} {product} graphics card +_summary: Test X driver/version after suspend for {vendor} {product} + +unit: template +template-resource: graphics_card +template-filter: graphics_card.prime_gpu_offload == 'Off' +plugin: attachment +category_id: com.canonical.plainbox::suspend +id: suspend/{index}_suspend-single-log-attach_{product_slug}_xenial +depends: suspend/{index}_suspend_after_switch_to_card_{product_slug}_xenial +command: [ -e $PLAINBOX_SESSION_SHARE/{index}_suspend_single.log ] && cat $PLAINBOX_SESSION_SHARE/{index}_suspend_single.log +_description: + Attaches the log from the single suspend/resume test to the results + +unit: template +template-resource: graphics_card +template-filter: graphics_card.prime_gpu_offload == 'Off' +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/{index}_suspend-time-check_{product_slug}_xenial +depends: suspend/{index}_suspend_after_switch_to_card_{product_slug}_xenial +estimated_duration: 1.2 +command: [ -e $PLAINBOX_SESSION_SHARE/{index}_suspend_single_times.log ] && sleep_time_check $PLAINBOX_SESSION_SHARE/{index}_suspend_single_times.log +_description: Checks the sleep times to ensure that a machine suspends and resumes within a given threshold + diff --git a/units/suspend/suspend.pxu b/units/suspend/suspend.pxu new file mode 100644 index 0000000..287071a --- /dev/null +++ b/units/suspend/suspend.pxu @@ -0,0 +1,2898 @@ +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/network_before_suspend +depends: ethernet/detect +estimated_duration: 1.2 +_description: Record the current network before suspending. +command: set -o pipefail; gateway_ping_test | tee $PLAINBOX_SESSION_SHARE/network_before_suspend.txt + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/resolution_before_suspend +estimated_duration: 1.2 +_description: Record the current resolution before suspending. +command: xrandr -q |grep '*'| awk '{print $1}' > $PLAINBOX_SESSION_SHARE/resolution_before_suspend.txt + +unit: template +template-resource: graphics_card +template-filter: graphics_card.prime_gpu_offload == 'Off' +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/{index}_resolution_before_suspend_{product_slug} +depends: graphics/{index}_switch_card_{product_slug} +estimated_duration: 1.2 +_description: Record the current resolution before suspending. +command: xrandr -q |grep '*'| awk '{{print $1}}' > $PLAINBOX_SESSION_SHARE/{index}_resolution_before_suspend.txt + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/audio_before_suspend +estimated_duration: 1.0 +requires: + device.category == 'AUDIO' + package.name == 'alsa-base' +_description: Record mixer settings before suspending. +command: audio_settings store --file=$PLAINBOX_SESSION_SHARE/audio_settings_before_suspend + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/cpu_before_suspend +estimated_duration: 1.2 +_description: Verify that all the CPUs are online before suspending +command: cpuinfo_resource > $PLAINBOX_SESSION_SHARE/cpuinfo_before_suspend + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/memory_before_suspend +estimated_duration: 1.2 +_description: + Dumps memory info to a file for comparison after suspend test has been run +command: meminfo_resource > $PLAINBOX_SESSION_SHARE/meminfo_before_suspend + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/wireless_before_suspend +depends: wireless/wireless_connection +requires: device.category == 'WIRELESS' +command: nmcli -t -f UUID con status > $PLAINBOX_SESSION_SHARE/connections && connect_wireless && gateway_ping_test --interface=`(nmcli dev list 2>/dev/null || nmcli dev show) | grep -B 1 -e 'wireless' -e 'wifi' | grep GENERAL.DEVICE | awk '{print $2}'` && for con in `cat $PLAINBOX_SESSION_SHARE/connections`; do nmcli con up uuid "$con"; done +estimated_duration: 20.0 +_description: + This test disconnects all connections and then connects to the wireless + interface. It then checks the connection to confirm it's working as expected. + +unit: template +template-resource: device +template-filter: device.category == 'NETWORK' +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/iperf_before_suspend_ether_auto_device{__index__}_{interface} +depends: ethernet/detect +estimated_duration: 20.0 +requires: + package.name == 'iperf' +user: root +environ: TEST_TARGET_FTP TEST_TARGET_IPERF TEST_USER TEST_PASS +command: network -i {interface} -t iperf +_description: + This test executes iperf connection performance/stability against device {__index__} ({interface}) before suspend. + +unit: template +template-resource: device +template-filter: device.category == 'WIRELESS' +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/iperf_before_suspend_wifi_auto_device{__index__}_{interface} +depends: wireless/wireless_connection +estimated_duration: 20.0 +requires: + package.name == 'iperf' +user: root +environ: TEST_TARGET_FTP TEST_TARGET_IPERF TEST_USER TEST_PASS +command: network -i {interface} -t iperf +_description: + This test executes iperf connection performance/stability against device {__index__} ({interface}) before suspend. + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/iperf_before_suspend_mobilebroadband_gsm_auto +depends: mobilebroadband/gsm_connection +estimated_duration: 20.0 +user: root +environ: TEST_TARGET_FTP TEST_TARGET_IPERF TEST_USER TEST_PASS +command: + INTERFACE=`(nmcli -t -f GENERAL -m tabular dev list 2>/dev/null || nmcli -t -f GENERAL -m tabular dev show) |grep gsm |cut -d ":" -f 13` + [ -z $INTERFACE ] && exit 1 + network test -i $INTERFACE -t iperf +_description: + This test executes iperf connection performance/stability against the broadband device found on the system before suspend. + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/iperf_before_suspend_mobilebroadband_cdma_auto +depends: mobilebroadband/cdma_connection +estimated_duration: 20.0 +user: root +environ: TEST_TARGET_FTP TEST_TARGET_IPERF TEST_USER TEST_PASS +command: + INTERFACE=`(nmcli -t -f GENERAL -m tabular dev list 2>/dev/null || nmcli -t -f GENERAL -m tabular dev show) |grep cdma |cut -d ":" -f 13` + [ -z $INTERFACE ] && exit 1 + network test -i $INTERFACE -t iperf +_description: + This test executes iperf connection performance/stability against the broadband device found on the system before suspend. + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/bluetooth_obex_send_before_suspend +estimated_duration: 10.0 +requires: + package.name == 'bluez' + package.name == 'obexftp' + device.category == 'BLUETOOTH' +command: + if [ -z "$BTDEVADDR" ] + then + echo "btdevaddr option not set to device address of Bluetooth target in checkbox.ini" + exit 1 + fi + if rfkill list bluetooth | grep -q 'Hard blocked: yes' + then + echo "rfkill list shows BT is hard blocked" + fi + if rfkill list bluetooth | grep -q 'Soft blocked: yes' + then + echo "rfkill list shows BT is soft blocked, removing before testing" + rfkill unblock bluetooth + sleep 3 + fi + set -o pipefail; bluetooth_test $PLAINBOX_PROVIDER_DATA/images/JPEG_Color_Image_Ubuntu.jpg $BTDEVADDR send 2>&1 | ansi_parser +_description: + This is an automated Bluetooth file transfer test. It sends an image to the device specified by the BTDEVADDR environment variable. + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/bluetooth_obex_browse_before_suspend +estimated_duration: 10.0 +requires: + package.name == 'bluez' + package.name == 'obexftp' + device.category == 'BLUETOOTH' +command: + if [ -z "$BTDEVADDR" ] + then + echo "btdevaddr option not set to device address of Bluetooth target in checkbox.ini" + exit 1 + fi + if rfkill list bluetooth | grep -q 'Hard blocked: yes' + then + echo "rfkill list shows BT is hard blocked" + fi + if rfkill list bluetooth | grep -q 'Soft blocked: yes' + then + echo "rfkill list shows BT is soft blocked, removing before testing" + rfkill unblock bluetooth + sleep 3 + fi + set -o pipefail; bluetooth_test $PLAINBOX_PROVIDER_DATA/images/JPEG_Color_Image_Ubuntu.jpg $BTDEVADDR browse 2>&1 | ansi_parser +_description: + This is an automated Bluetooth test. It emulates browsing on a remote device specified by the BTDEVADDR environment variable. + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/bluetooth_obex_get_before_suspend +estimated_duration: 20.0 +requires: + package.name == 'bluez' + package.name == 'obexftp' + device.category == 'BLUETOOTH' +command: + if [ -z "$BTDEVADDR" ] + then + echo "btdevaddr option not set to device address of Bluetooth target in checkbox.ini" + exit 1 + fi + if rfkill list bluetooth | grep -q 'Hard blocked: yes' + then + echo "rfkill list shows BT is hard blocked" + fi + if rfkill list bluetooth | grep -q 'Soft blocked: yes' + then + echo "rfkill list shows BT is soft blocked, removing before testing" + rfkill unblock bluetooth + sleep 3 + fi + set -o pipefail; bluetooth_test $PLAINBOX_PROVIDER_DATA/images/JPEG_Color_Image_Ubuntu.jpg $BTDEVADDR get 2>&1 | ansi_parser +_description: + This is an automated Bluetooth test. It receives the given file from a remote host specified by the BTDEVADDR environment variable + +plugin: user-interact-verify +category_id: com.canonical.plainbox::suspend +id: suspend/suspend_advanced +requires: + sleep.mem == 'supported' + rtc.state == 'supported' +user: root +environ: PLAINBOX_SESSION_SHARE +command: + if type -P fwts >/dev/null; then + echo "Calling fwts" + if [[ -v SNAP ]]; then + export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$SNAP/usr/lib/fwts" + fi + set -o pipefail; checkbox-support-fwts_test -f none -l $PLAINBOX_SESSION_SHARE/suspend_single -s s3 --s3-sleep-delay=30 --s3-device-check --s3-device-check-delay=45 | tee $PLAINBOX_SESSION_SHARE/suspend_single_times.log + else + echo "Calling sleep_test" + set -o pipefail; sleep_test -p | tee $PLAINBOX_SESSION_SHARE/2_suspend_single_times.log + fi +estimated_duration: 90.0 +_description: + PURPOSE: + This test will check suspend and resume + STEPS: + 1. Click "Test" and your system will suspend for about 30 - 60 seconds + 2. Observe the Power LED to see if it blinks or changes color during suspend + 3. If your system does not wake itself up after 60 seconds, please press the power button momentarily to wake the system manually + 4. If your system fails to wake at all and must be rebooted, restart System Testing after reboot and mark this test as Failed + VERIFICATION: + Did your system suspend and resume correctly? + (NOTE: Please only consider whether the system successfully suspended and resumed. Power/Suspend LED verification will occur after this test is completed.) + +unit: template +template-resource: graphics_card +template-filter: graphics_card.prime_gpu_offload == 'Off' +plugin: user-interact-verify +category_id: com.canonical.plainbox::suspend +id: suspend/{index}_suspend_after_switch_to_card_{product_slug} +requires: + sleep.mem == 'supported' + rtc.state == 'supported' +depends: suspend/suspend_advanced graphics/{index}_switch_card_{product_slug} +user: root +environ: PLAINBOX_SESSION_SHARE +command: + if type -P fwts >/dev/null; then + echo "Calling fwts" + if [[ -v SNAP ]]; then + export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$SNAP/usr/lib/fwts" + fi + set -o pipefail; checkbox-support-fwts_test -f none -l $PLAINBOX_SESSION_SHARE/{index}_suspend_single -s s3 --s3-sleep-delay=30 --s3-device-check --s3-device-check-delay=45 | tee $PLAINBOX_SESSION_SHARE/{index}_suspend_single_times.log + else + echo "Calling sleep_test" + set -o pipefail; sleep_test -p | tee $PLAINBOX_SESSION_SHARE/{index}_suspend_single_times.log + fi +estimated_duration: 90.0 +_summary: Test suspend/resume after switching to {vendor} {product} +_description: + PURPOSE: + This test will check suspend and resume after switching to {vendor} {product} graphics card. + STEPS: + 1. Ensure you have switched to {vendor} {product} graphics card. + 2. Click "Test" and your system will suspend for about 30 - 60 seconds + 3. Observe the Power LED to see if it blinks or changes color during suspend + 4. If your system does not wake itself up after 60 seconds, please press the power button momentarily to wake the system manually + 5. If your system fails to wake at all and must be rebooted, restart System Testing after reboot and mark this test as Failed + VERIFICATION: + Did your system suspend and resume correctly after switching to {vendor} {product} graphics card? + (NOTE: Please only consider whether the system successfully suspended and resumed. Power/Suspend LED verification will occur after this test is completed.) + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/suspend_advanced_auto +requires: + sleep.mem == 'supported' + rtc.state == 'supported' +_description: + This is the automated version of suspend/suspend_advanced. +user: root +environ: PLAINBOX_SESSION_SHARE +command: + if [[ -v SNAP ]]; then + export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$SNAP/usr/lib/fwts" + fi + set -o pipefail; checkbox-support-fwts_test -f none -l $PLAINBOX_SESSION_SHARE/suspend_single -s s3 --s3-sleep-delay=30 --s3-device-check --s3-device-check-delay=45 | tee $PLAINBOX_SESSION_SHARE/suspend_single_times.log +estimated_duration: 90.000 + +unit: template +template-resource: graphics_card +plugin: user-interact-verify +category_id: com.canonical.plainbox::suspend +id: suspend/{index}_hybrid_sleep_{product_slug} +user: root +command: + if [[ -v SNAP ]]; then + export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$SNAP/usr/lib/fwts" + fi + set -o pipefail; checkbox-support-fwts_test -f none -l $PLAINBOX_SESSION_SHARE/{index}_hybrid_sleep_single -s s3 --s3-hybrid --s3-sleep-delay=30 --s3-device-check --s3-device-check-delay=45 --pm-method=pm-utils | tee $PLAINBOX_SESSION_SHARE/{index}_hybrid_sleep_single_times.log +estimated_duration: 90.0 +_description: + PURPOSE: + This test will check hybrid sleep and resume + STEPS: + 1. Click "Test" and your system will go into hybrid sleep mode for about 30 - 60 seconds + 2. Observe the Power LED to see if it blinks or changes color during suspend + 3. If your system does not wake itself up after 60 seconds, please press the power button momentarily to wake the system manually + 4. If your system fails to wake at all and must be rebooted, restart System Testing after reboot and mark this test as Failed + VERIFICATION: + Did your system enter hybrid sleep and then resume correctly? + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/suspend-single-log-check +depends: suspend/suspend_advanced_auto +estimated_duration: 1.2 +command: [ -e $PLAINBOX_SESSION_SHARE/suspend_single.log ] && sleep_test_log_check -v s3 $PLAINBOX_SESSION_SHARE/suspend_single.log +_description: + Automated check of the suspend log to look for errors reported by fwts + +plugin: attachment +category_id: com.canonical.plainbox::suspend +id: suspend/suspend-single-log-attach +depends: suspend/suspend_advanced_auto +command: [ -e $PLAINBOX_SESSION_SHARE/suspend_single.log ] && xz -c $PLAINBOX_SESSION_SHARE/suspend_single.log | base64 +_description: + Attaches the log from the single suspend/resume test to the results + +unit: template +template-resource: graphics_card +template-filter: graphics_card.prime_gpu_offload == 'Off' +plugin: attachment +category_id: com.canonical.plainbox::suspend +id: suspend/{index}_suspend-single-log-attach_{product_slug} +depends: suspend/{index}_suspend_after_switch_to_card_{product_slug} +command: [ -e $PLAINBOX_SESSION_SHARE/{index}_suspend_single.log ] && xz -c $PLAINBOX_SESSION_SHARE/{index}_suspend_single.log | base64 +_description: + Attaches the log from the single suspend/resume test to the results + +unit: template +template-resource: graphics_card +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/{index}_hybrid-sleep-single-log-check +depends: suspend/{index}_hybrid_sleep_{product_slug} +estimated_duration: 1.2 +command: [ -e $PLAINBOX_SESSION_SHARE/{index}_hybrid_sleep_single.log ] && sleep_test_log_check -v s3 $PLAINBOX_SESSION_SHARE/{index}_hybrid_sleep_single.log +_description: + Automated check of the hybrid sleep log to look for errors reported by fwts + +unit: template +template-resource: graphics_card +plugin: attachment +category_id: com.canonical.plainbox::suspend +id: suspend/{index}_hybrid-sleep-single-log-attach_{product_slug} +depends: suspend/{index}_hybrid_sleep_{product_slug} +command: [ -e $PLAINBOX_SESSION_SHARE/{index}_hybrid_sleep_single.log ] && xz -c $PLAINBOX_SESSION_SHARE/{index}_hybrid_sleep_single.log | base64 +_description: + Attaches the log from the single hybrid sleep/resume test to the results + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/suspend-time-check +estimated_duration: 1.2 +command: [ -e $PLAINBOX_SESSION_SHARE/suspend_single_times.log ] && sleep_time_check $PLAINBOX_SESSION_SHARE/suspend_single_times.log +_description: + Checks the sleep times to ensure that a machine suspends and resumes within a given threshold + +unit: template +template-resource: graphics_card +template-filter: graphics_card.prime_gpu_offload == 'Off' +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/{index}_suspend-time-check_{product_slug} +depends: suspend/{index}_suspend_after_switch_to_card_{product_slug} +estimated_duration: 1.2 +command: [ -e $PLAINBOX_SESSION_SHARE/{index}_suspend_single_times.log ] && sleep_time_check $PLAINBOX_SESSION_SHARE/{index}_suspend_single_times.log +_description: Checks the sleep times to ensure that a machine suspends and resumes within a given threshold + +plugin: user-interact-verify +category_id: com.canonical.plainbox::suspend +id: power-management/usb_wakeup_keyboard +user: root +depends: suspend/suspend_advanced +estimated_duration: 120.0 +command: pm-suspend +_description: + PURPOSE: + Wake up by USB keyboard + STEPS: + 1. Enable "Wake by USB KB/Mouse" item in BIOS + 2. Press "Test" to enter suspend (S3) mode + 3. Press any key of USB keyboard to wake system up + VERIFICATION: + Did the system wake up from suspend mode when you pressed a keyboard key? + +plugin: user-interact-verify +category_id: com.canonical.plainbox::suspend +id: power-management/usb_wakeup_mouse +user: root +depends: suspend/suspend_advanced +estimated_duration: 120.0 +command: pm-suspend +_description: + PURPOSE: + Wake up by USB mouse + STEPS: + 1. Enable "Wake by USB KB/Mouse" item in BIOS + 2. Press "Test" to enter suspend (S3) mode + 3. Press any button of USB mouse to wake system up + VERIFICATION: + Did the system wake up from suspend mode when you pressed the mouse button? + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/network_after_suspend +estimated_duration: 20.0 +depends: suspend/suspend_advanced suspend/network_before_suspend +_description: Test the network after resuming. +command: network_wait; gateway_ping_test | diff $PLAINBOX_SESSION_SHARE/network_before_suspend.txt - + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/resolution_after_suspend +depends: suspend/suspend_advanced suspend/resolution_before_suspend +estimated_duration: 1.2 +_description: Test to see that we have the same resolution after resuming as before. +command: xrandr -q |grep '*'| awk '{print $1}' | diff $PLAINBOX_SESSION_SHARE/resolution_before_suspend.txt - + +unit: template +template-resource: graphics_card +template-filter: graphics_card.prime_gpu_offload == 'Off' +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/{index}_resolution_after_suspend_{product_slug} +estimated_duration: 1.2 +depends: suspend/{index}_resolution_before_suspend_{product_slug} +_description: Test to see that we have the same resolution after resuming as before. +command: xrandr -q |grep '*'| awk '{{print $1}}' | diff $PLAINBOX_SESSION_SHARE/{index}_resolution_before_suspend.txt - + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/audio_after_suspend +estimated_duration: 1.0 +requires: + device.category == 'AUDIO' + package.name == 'alsa-base' +depends: suspend/suspend_advanced suspend/audio_before_suspend +_description: Verify that mixer settings after suspend are the same as before suspend. +command: + audio_settings store --file=$PLAINBOX_SESSION_SHARE/audio_settings_after_suspend + diff $PLAINBOX_SESSION_SHARE/audio_settings_before_suspend $PLAINBOX_SESSION_SHARE/audio_settings_after_suspend + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/audio_after_suspend_auto +estimated_duration: 1.2 +requires: + device.category == 'AUDIO' + package.name == 'alsa-base' +depends: suspend/suspend_advanced_auto suspend/audio_before_suspend +_description: Verify that mixer settings after suspend are the same as before suspend. +command: + audio_settings store --file=$PLAINBOX_SESSION_SHARE/audio_settings_after_suspend + diff $PLAINBOX_SESSION_SHARE/audio_settings_before_suspend $PLAINBOX_SESSION_SHARE/audio_settings_after_suspend + +plugin: user-interact +category_id: com.canonical.plainbox::suspend +id: suspend/speaker-headphone-plug-detection-after-suspend +depends: suspend/suspend_advanced +estimated_duration: 60.0 +requires: + device.category == 'AUDIO' + package.name == 'pulseaudio-utils' +command: pulse-active-port-change sinks +_description: + PURPOSE: + Check that system detects speakers or headphones being plugged in after suspend + STEPS: + 1. Prepare a pair of headphones or speakers with a standard 3.5mm jack + 2. Locate the speaker / headphone jack on the device under test + 3. Run the test (you have 30 seconds from now on) + 4. Plug headphones or speakers into the appropriate jack + 5. Unplug the device for subsequent tests. + VERIFICATION: + Verification is automatic, no action is required. + The test times out after 30 seconds (and fails in that case). + +plugin: user-interact +category_id: com.canonical.plainbox::suspend +id: suspend/microphone-plug-detection-after-suspend +depends: suspend/suspend_advanced +estimated_duration: 60.0 +requires: + device.category == 'AUDIO' + package.name == 'pulseaudio-utils' +command: pulse-active-port-change sources +_description: + PURPOSE: + Check that system detects a microphone being plugged in after suspend + STEPS: + 1. Prepare a microphone with a standard 3.5mm jack + 2. Locate the microphone jack on the device under test. + Keep in mind that it may be shared with the headphone jack. + 3. Run the test (you have 30 seconds from now on) + 4. Plug the microphone into the appropriate jack + 5. Unplug the device for subsequent tests. + VERIFICATION: + Verification is automatic, no action is required. + The test times out after 30 seconds (and fails in that case). + +plugin: user-interact-verify +category_id: com.canonical.plainbox::suspend +id: suspend/playback_headphones-after-suspend +estimated_duration: 20.0 +depends: audio/list_devices suspend/suspend_advanced +requires: + device.category == 'AUDIO' + package.name == 'alsa-base' + package.name == 'gir1.2-gst-plugins-base-0.10' or package.name == 'gir1.2-gst-plugins-base-1.0' + package.name == 'pulseaudio-utils' +command: + audio_settings store --file=$PLAINBOX_SESSION_SHARE/pulseaudio_settings + audio_settings set --device=pci --volume=50 + gst_pipeline_test -t 2 'audiotestsrc wave=sine freq=512 ! audioconvert ! audioresample ! autoaudiosink' + EXIT_CODE=$? + audio_settings restore --file=$PLAINBOX_SESSION_SHARE/pulseaudio_settings + exit $EXIT_CODE +_description: + PURPOSE: + This test will check that headphones connector works correctly after suspend + STEPS: + 1. Connect a pair of headphones to your audio device + 2. Commence the test to play a sound to your audio device + VERIFICATION: + Did you hear a sound through the headphones and did the sound play without any distortion, clicks or other strange noises from your headphones? + +plugin: user-interact-verify +category_id: com.canonical.plainbox::suspend +id: suspend/alsa_record_playback_external-after-suspend +estimated_duration: 20.0 +depends: suspend/playback_headphones-after-suspend suspend/suspend_advanced +requires: + device.category == 'AUDIO' + package.name == 'alsa-base' + package.name == 'pulseaudio-utils' + package.name == 'gstreamer1.0-plugins-good' or package.name == 'gstreamer0.10-plugins-good' +command: + audio_settings store --file=$PLAINBOX_SESSION_SHARE/pulseaudio_settings + audio_settings set --device=pci --volume=50 + alsa_record_playback + EXIT_CODE=$? + audio_settings restore --file=$PLAINBOX_SESSION_SHARE/pulseaudio_settings + exit $EXIT_CODE +_description: + PURPOSE: + This test will check that recording sound using an external microphone works correctly after suspend + STEPS: + 1. Connect a microphone to your microphone port + 2. Click "Test", then speak into the external microphone + 3. After a few seconds, your speech will be played back to you + VERIFICATION: + Did you hear your speech played back? + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/cpu_after_suspend +estimated_duration: 1.2 +depends: suspend/suspend_advanced suspend/cpu_before_suspend +_description: Verify that all CPUs are online after resuming. +command: cpuinfo_resource | diff $PLAINBOX_SESSION_SHARE/cpuinfo_before_suspend - + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/cpu_after_suspend_auto +estimated_duration: 1.2 +depends: suspend/suspend_advanced_auto suspend/cpu_before_suspend +_description: Verify that all CPUs are online after resuming. +command: cpuinfo_resource | diff $PLAINBOX_SESSION_SHARE/cpuinfo_before_suspend - + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/memory_after_suspend +estimated_duration: 1.2 +depends: suspend/suspend_advanced suspend/memory_before_suspend +_description: + Verify that all memory is available after resuming from suspend. +command: meminfo_resource | diff $PLAINBOX_SESSION_SHARE/meminfo_before_suspend - + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/memory_after_suspend_auto +estimated_duration: 1.2 +depends: suspend/suspend_advanced_auto suspend/memory_before_suspend +_description: + Verify that all memory is available after resuming from suspend. +command: meminfo_resource | diff $PLAINBOX_SESSION_SHARE/meminfo_before_suspend - + +plugin: manual +category_id: com.canonical.plainbox::suspend +id: suspend/display_after_suspend +estimated_duration: 120.0 +depends: suspend/suspend_advanced +_description: + PURPOSE: + This test will check that the display is correct after suspend and resume + STEPS: + 1. Check that your display does not show up visual artifacts after resuming. + VERIFICATION: + Does the display work normally after resuming from suspend? + +unit: template +template-resource: graphics_card +template-filter: graphics_card.prime_gpu_offload == 'Off' +plugin: manual +category_id: com.canonical.plainbox::suspend +id: suspend/{index}_display_after_suspend_{product_slug} +depends: suspend/{index}_suspend_after_switch_to_card_{product_slug} +_summary: Test display function after suspend for {vendor} {product} +_description: + PURPOSE: + This test will check that the display is correct after suspend and resume on the {vendor} {product} graphics card. + STEPS: + 1. Check that your display does not show up visual artifacts after resuming. + VERIFICATION: + Does the display work normally after resuming from suspend using the {vendor} {product} graphics card? + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/wireless_after_suspend +depends: suspend/suspend_advanced suspend/wireless_before_suspend +requires: + device.category == 'WIRELESS' +command: connect_wireless && gateway_ping_test --interface=`(nmcli dev list 2>/dev/null || nmcli dev show) | grep -B 1 -e wireless -e wifi | grep GENERAL.DEVICE | awk '{print $2}'` && for con in `cat $PLAINBOX_SESSION_SHARE/connections`; do nmcli con up uuid "$con"; done +estimated_duration: 20.0 +_description: + This test checks that the wireless interface is working after suspending the system. It + disconnects all interfaces and then connects to the wireless interface and checks that the + connection is working as expected. + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/wireless_connection_after_suspend_wpa_bg +depends: suspend/suspend_advanced +estimated_duration: 20.0 +requires: + device.category == 'WIRELESS' + environment.ROUTERS == 'multiple' +user: root +environ: WPA_BG_SSID WPA_BG_PSK +command: + trap "nmcli con delete id $WPA_BG_SSID" EXIT + if create_connection wifi $WPA_BG_SSID --security=wpa --key=$WPA_BG_PSK; then + connect_wireless # lp:1471663 + INTERFACE=`nmcli dev status | awk '/802-11-wireless|wifi/ {print $1}'` + iw dev $INTERFACE link + gateway_ping_test --interface=$INTERFACE + STATUS=$? + # We reconnect the Ethernet connection if any (lp:1471663) + WIRED=$(nmcli -f UUID,TYPE c | grep -oP ".*(?=\s+.*ethernet)") + if [[ ! -z $WIRED ]]; then + nmcli c up uuid $WIRED + fi + exit $STATUS + else + exit 1 + fi +_description: + Tests that the systems wireless hardware can connect to a router using WPA + security and the 802.11b/g protocols after the system has been suspended. + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/wireless_connection_after_suspend_open_bg +depends: suspend/suspend_advanced +estimated_duration: 1.2 +requires: + device.category == 'WIRELESS' + environment.ROUTERS == 'multiple' +user: root +environ: OPEN_BG_SSID +command: + trap "nmcli con delete id $OPEN_BG_SSID" EXIT + if create_connection wifi $OPEN_BG_SSID; then + connect_wireless # lp:1471663 + INTERFACE=`nmcli dev status | awk '/802-11-wireless|wifi/ {print $1}'` + iw dev $INTERFACE link + gateway_ping_test --interface=$INTERFACE + STATUS=$? + # We reconnect the Ethernet connection if any (lp:1471663) + WIRED=$(nmcli -f UUID,TYPE c | grep -oP ".*(?=\s+.*ethernet)") + if [[ ! -z $WIRED ]]; then + nmcli c up uuid $WIRED + fi + exit $STATUS + else + exit 1 + fi +_description: + Tests that the systems wireless hardware can connect to a router using no + security and the 802.11b/g protocols after the system has been suspended. + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/wireless_connection_after_suspend_wpa_n +depends: suspend/suspend_advanced +estimated_duration: 1.2 +requires: + device.category == 'WIRELESS' + environment.ROUTERS == 'multiple' +user: root +environ: WPA_N_SSID WPA_N_PSK +command: + trap "nmcli con delete id $WPA_N_SSID" EXIT + if create_connection wifi $WPA_N_SSID --security=wpa --key=$WPA_N_PSK; then + connect_wireless # lp:1471663 + INTERFACE=`nmcli dev status | awk '/802-11-wireless|wifi/ {print $1}'` + iw dev $INTERFACE link + gateway_ping_test --interface=$INTERFACE + STATUS=$? + # We reconnect the Ethernet connection if any (lp:1471663) + WIRED=$(nmcli -f UUID,TYPE c | grep -oP ".*(?=\s+.*ethernet)") + if [[ ! -z $WIRED ]]; then + nmcli c up uuid $WIRED + fi + exit $STATUS + else + exit 1 + fi +_description: + Tests that the systems wireless hardware can connect to a router using WPA + security and the 802.11n protocol after the system has been suspended. + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/wireless_connection_after_suspend_open_n +depends: suspend/suspend_advanced +estimated_duration: 1.2 +requires: + device.category == 'WIRELESS' + environment.ROUTERS == 'multiple' +user: root +environ: OPEN_N_SSID +command: + trap "nmcli con delete id $OPEN_N_SSID" EXIT + if create_connection wifi $OPEN_N_SSID; then + connect_wireless # lp:1471663 + INTERFACE=`nmcli dev status | awk '/802-11-wireless|wifi/ {print $1}'` + iw dev $INTERFACE link + gateway_ping_test --interface=$INTERFACE + STATUS=$? + # We reconnect the Ethernet connection if any (lp:1471663) + WIRED=$(nmcli -f UUID,TYPE c | grep -oP ".*(?=\s+.*ethernet)") + if [[ ! -z $WIRED ]]; then + nmcli c up uuid $WIRED + fi + exit $STATUS + else + exit 1 + fi +_description: + Tests that the systems wireless hardware can connect to a router using no + security and the 802.11n protocol after the system has been suspended. + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/wireless_connection_after_suspend_wpa_ac +depends: suspend/suspend_advanced +estimated_duration: 1.2 +requires: + device.category == 'WIRELESS' + environment.ROUTERS == 'multiple' + IEEE_80211.ac == 'supported' +user: root +environ: WPA_AC_SSID WPA_AC_PSK +command: + trap "nmcli con delete id $WPA_AC_SSID" EXIT + if create_connection wifi $WPA_AC_SSID --security=wpa --key=$WPA_AC_PSK; then + connect_wireless # lp:1471663 + INTERFACE=`nmcli dev status | awk '/802-11-wireless|wifi/ {print $1}'` + iw dev $INTERFACE link + gateway_ping_test --interface=$INTERFACE + STATUS=$? + # We reconnect the Ethernet connection if any (lp:1471663) + WIRED=$(nmcli -f UUID,TYPE c | grep -oP ".*(?=\s+.*ethernet)") + if [[ ! -z $WIRED ]]; then + nmcli c up uuid $WIRED + fi + exit $STATUS + else + exit 1 + fi +_description: + Tests that the systems wireless hardware can connect to a router using WPA + security and the 802.11ac protocol after the system has been suspended. + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/wireless_connection_after_suspend_open_ac +depends: suspend/suspend_advanced +estimated_duration: 1.2 +requires: + device.category == 'WIRELESS' + environment.ROUTERS == 'multiple' + IEEE_80211.ac == 'supported' +user: root +environ: OPEN_AC_SSID +command: + trap "nmcli con delete id $OPEN_AC_SSID" EXIT + if create_connection wifi $OPEN_AC_SSID; then + connect_wireless # lp:1471663 + INTERFACE=`nmcli dev status | awk '/802-11-wireless|wifi/ {print $1}'` + iw dev $INTERFACE link + gateway_ping_test --interface=$INTERFACE + STATUS=$? + # We reconnect the Ethernet connection if any (lp:1471663) + WIRED=$(nmcli -f UUID,TYPE c | grep -oP ".*(?=\s+.*ethernet)") + if [[ ! -z $WIRED ]]; then + nmcli c up uuid $WIRED + fi + exit $STATUS + else + exit 1 + fi +_description: + Tests that the systems wireless hardware can connect to a router using no + security and the 802.11ac protocol after the system has been suspended. + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/wireless_connection_after_suspend_wpa_bg_auto +depends: suspend/suspend_advanced_auto +estimated_duration: 1.2 +requires: + device.category == 'WIRELESS' + environment.ROUTERS == 'multiple' +user: root +environ: WPA_BG_SSID WPA_BG_PSK +command: + trap "nmcli con delete id $WPA_BG_SSID" EXIT + if create_connection wifi $WPA_BG_SSID --security=wpa --key=$WPA_BG_PSK; then + connect_wireless # lp:1471663 + INTERFACE=`nmcli dev status | awk '/802-11-wireless|wifi/ {print $1}'` + iw dev $INTERFACE link + gateway_ping_test --interface=$INTERFACE + STATUS=$? + # We reconnect the Ethernet connection if any (lp:1471663) + WIRED=$(nmcli -f UUID,TYPE c | grep -oP ".*(?=\s+.*ethernet)") + if [[ ! -z $WIRED ]]; then + nmcli c up uuid $WIRED + fi + exit $STATUS + else + exit 1 + fi +_description: + Tests that the systems wireless hardware can connect to a router using WPA + security and the 802.11b/g protocols after the system has been suspended. + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/wireless_connection_after_suspend_open_bg_auto +depends: suspend/suspend_advanced_auto +estimated_duration: 1.2 +requires: + device.category == 'WIRELESS' + environment.ROUTERS == 'multiple' +user: root +environ: OPEN_BG_SSID +command: + trap "nmcli con delete id $OPEN_BG_SSID" EXIT + if create_connection wifi $OPEN_BG_SSID; then + connect_wireless # lp:1471663 + INTERFACE=`nmcli dev status | awk '/802-11-wireless|wifi/ {print $1}'` + iw dev $INTERFACE link + gateway_ping_test --interface=$INTERFACE + STATUS=$? + # We reconnect the Ethernet connection if any (lp:1471663) + WIRED=$(nmcli -f UUID,TYPE c | grep -oP ".*(?=\s+.*ethernet)") + if [[ ! -z $WIRED ]]; then + nmcli c up uuid $WIRED + fi + exit $STATUS + else + exit 1 + fi +_description: + Tests that the systems wireless hardware can connect to a router using no + security and the 802.11b/g protocols after the system has been suspended. + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/wireless_connection_after_suspend_wpa_n_auto +depends: suspend/suspend_advanced_auto +estimated_duration: 1.2 +requires: + device.category == 'WIRELESS' + environment.ROUTERS == 'multiple' +user: root +environ: WPA_N_SSID WPA_N_PSK +command: + trap "nmcli con delete id $WPA_N_SSID" EXIT + if create_connection wifi $WPA_N_SSID --security=wpa --key=$WPA_N_PSK; then + connect_wireless # lp:1471663 + INTERFACE=`nmcli dev status | awk '/802-11-wireless|wifi/ {print $1}'` + iw dev $INTERFACE link + gateway_ping_test --interface=$INTERFACE + STATUS=$? + # We reconnect the Ethernet connection if any (lp:1471663) + WIRED=$(nmcli -f UUID,TYPE c | grep -oP ".*(?=\s+.*ethernet)") + if [[ ! -z $WIRED ]]; then + nmcli c up uuid $WIRED + fi + exit $STATUS + else + exit 1 + fi +_description: + Tests that the systems wireless hardware can connect to a router using WPA + security and the 802.11n protocol after the system has been suspended. + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/wireless_connection_after_suspend_open_n_auto +depends: suspend/suspend_advanced_auto +estimated_duration: 1.2 +requires: + device.category == 'WIRELESS' + environment.ROUTERS == 'multiple' +user: root +environ: OPEN_N_SSID +command: + trap "nmcli con delete id $OPEN_N_SSID" EXIT + if create_connection wifi $OPEN_N_SSID; then + connect_wireless # lp:1471663 + INTERFACE=`nmcli dev status | awk '/802-11-wireless|wifi/ {print $1}'` + iw dev $INTERFACE link + gateway_ping_test --interface=$INTERFACE + STATUS=$? + # We reconnect the Ethernet connection if any (lp:1471663) + WIRED=$(nmcli -f UUID,TYPE c | grep -oP ".*(?=\s+.*ethernet)") + if [[ ! -z $WIRED ]]; then + nmcli c up uuid $WIRED + fi + exit $STATUS + else + exit 1 + fi +_description: + Tests that the systems wireless hardware can connect to a router using no + security and the 802.11n protocol after the system has been suspended. + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/wireless_connection_after_suspend_wpa_ac_auto +depends: suspend/suspend_advanced_auto +estimated_duration: 1.2 +requires: + device.category == 'WIRELESS' + environment.ROUTERS == 'multiple' + IEEE_80211.ac == 'supported' +user: root +environ: WPA_AC_SSID WPA_AC_PSK +command: + trap "nmcli con delete id $WPA_AC_SSID" EXIT + if create_connection wifi $WPA_AC_SSID --security=wpa --key=$WPA_AC_PSK; then + connect_wireless # lp:1471663 + INTERFACE=`nmcli dev status | awk '/802-11-wireless|wifi/ {print $1}'` + iw dev $INTERFACE link + gateway_ping_test --interface=$INTERFACE + STATUS=$? + # We reconnect the Ethernet connection if any (lp:1471663) + WIRED=$(nmcli -f UUID,TYPE c | grep -oP ".*(?=\s+.*ethernet)") + if [[ ! -z $WIRED ]]; then + nmcli c up uuid $WIRED + fi + exit $STATUS + else + exit 1 + fi +_description: + Tests that the systems wireless hardware can connect to a router using WPA + security and the 802.11ac protocol after the system has been suspended. + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/wireless_connection_after_suspend_open_ac_auto +depends: suspend/suspend_advanced_auto +estimated_duration: 1.2 +requires: + device.category == 'WIRELESS' + environment.ROUTERS == 'multiple' + IEEE_80211.ac == 'supported' +user: root +environ: OPEN_AC_SSID +command: + trap "nmcli con delete id $OPEN_AC_SSID" EXIT + if create_connection wifi $OPEN_AC_SSID; then + connect_wireless # lp:1471663 + INTERFACE=`nmcli dev status | awk '/802-11-wireless|wifi/ {print $1}'` + iw dev $INTERFACE link + gateway_ping_test --interface=$INTERFACE + STATUS=$? + # We reconnect the Ethernet connection if any (lp:1471663) + WIRED=$(nmcli -f UUID,TYPE c | grep -oP ".*(?=\s+.*ethernet)") + if [[ ! -z $WIRED ]]; then + nmcli c up uuid $WIRED + fi + exit $STATUS + else + exit 1 + fi +_description: + Tests that the systems wireless hardware can connect to a router using no + security and the 802.11ac protocol after the system has been suspended. + +unit: template +template-resource: device +template-filter: device.category == 'NETWORK' +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/iperf_after_suspend_ether_auto_device{__index__}_{interface} +depends: suspend/suspend_advanced +estimated_duration: 30.0 +requires: + package.name == 'iperf' +user: root +environ: TEST_TARGET_FTP TEST_TARGET_IPERF TEST_USER TEST_PASS +command: network -i {interface} -t iperf +_description: + This test executes iperf connection performance/stability against device {__index__} ({interface}) after suspend. + +unit: template +template-resource: device +template-filter: device.category == 'WIRELESS' +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/iperf_after_suspend_wifi_auto_device{__index__}_{interface} +depends: suspend/suspend_advanced +estimated_duration: 30.0 +requires: + package.name == 'iperf' +user: root +environ: TEST_TARGET_FTP TEST_TARGET_IPERF TEST_USER TEST_PASS +command: network -i {interface} -t iperf +_description: + This test executes iperf connection performance/stability against device {__index__} ({interface}) after suspend. + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/iperf_after_suspend_mobilebroadband_gsm_auto +depends: suspend/suspend_advanced_auto +estimated_duration: 20.0 +user: root +environ: TEST_TARGET_FTP TEST_TARGET_IPERF TEST_USER TEST_PASS +command: + INTERFACE=`(nmcli -t -f GENERAL -m tabular dev list 2>/dev/null || nmcli -t -f GENERAL -m tabular dev show) |grep gsm |cut -d ":" -f 13` + [ -z $INTERFACE ] && exit 1 + network test -i $INTERFACE -t iperf +_description: + This test executes iperf connection performance/stability against the broadband device found on the system after suspend. + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/iperf_after_suspend_mobilebroadband_cdma_auto +depends: suspend/suspend_advanced_auto +estimated_duration: 20.0 +user: root +environ: TEST_TARGET_FTP TEST_TARGET_IPERF TEST_USER TEST_PASS +command: + INTERFACE=`(nmcli -t -f GENERAL -m tabular dev list 2>/dev/null || nmcli -t -f GENERAL -m tabular dev show) |grep cdma |cut -d ":" -f 13` + [ -z $INTERFACE ] && exit 1 + network test -i $INTERFACE -t iperf +_description: + This test executes iperf connection performance/stability against the broadband device found on the system after suspend. + +plugin: user-interact-verify +category_id: com.canonical.plainbox::suspend +id: suspend/wireless_connection_after_suspend_wpa_bg_manual +depends: suspend/suspend_advanced +estimated_duration: 120.0 +requires: + device.category == 'WIRELESS' + environment.ROUTERS == 'single' +user: root +environ: ROUTER_SSID ROUTER_PSK +command: + trap "nmcli con delete id $ROUTER_SSID" EXIT + if create_connection wifi $ROUTER_SSID --security=wpa --key=$ROUTER_PSK; then + connect_wireless # lp:1471663 + INTERFACE=`nmcli dev status | awk '/802-11-wireless|wifi/ {print $1}'` + iw dev $INTERFACE link + gateway_ping_test --interface=$INTERFACE + STATUS=$? + # We reconnect the Ethernet connection if any (lp:1471663) + WIRED=$(nmcli -f UUID,TYPE c | grep -oP ".*(?=\s+.*ethernet)") + if [[ ! -z $WIRED ]]; then + nmcli c up uuid $WIRED + fi + exit $STATUS + else + exit 1 + fi +_description: + PURPOSE: + Tests that the systems wireless hardware can connect to a router using WPA + security and the 802.11b/g protocols. + STEPS: + 1. Open your routers configuration tool + 2. Change the settings to only accept connections on the B and G wireless bands + 3. Make sure the SSID is set to ROUTER_SSID + 4. Change the security settings to use WPA2 and ensure the PSK matches that set in ROUTER_PSK + 5. Commence the test to create a connection to the router and test the connection + VERIFICATION: + Verification is automated, do not change the automatically selected result. + +plugin: user-interact-verify +category_id: com.canonical.plainbox::suspend +id: suspend/wireless_connection_after_suspend_open_bg_manual +depends: suspend/suspend_advanced +estimated_duration: 120.0 +requires: + device.category == 'WIRELESS' + environment.ROUTERS == 'single' +user: root +environ: ROUTER_SSID +command: + trap "nmcli con delete id $ROUTER_SSID" EXIT + if create_connection wifi $ROUTER_SSID; then + connect_wireless # lp:1471663 + INTERFACE=`nmcli dev status | awk '/802-11-wireless|wifi/ {print $1}'` + iw dev $INTERFACE link + gateway_ping_test --interface=$INTERFACE + STATUS=$? + # We reconnect the Ethernet connection if any (lp:1471663) + WIRED=$(nmcli -f UUID,TYPE c | grep -oP ".*(?=\s+.*ethernet)") + if [[ ! -z $WIRED ]]; then + nmcli c up uuid $WIRED + fi + exit $STATUS + else + exit 1 + fi +_description: + PURPOSE: + Tests that the systems wireless hardware can connect to a router using + no security and the 802.11b/g protocols. + STEPS: + 1. Open your routers configuration tool + 2. Change the settings to only accept connections on the B and G wireless bands + 3. Make sure the SSID is set to ROUTER_SSID + 4. Change the security settings to use no security + 5. Commence the test to create a connection to the router and test the connection + VERIFICATION: + Verification is automated, do not change the automatically selected result. + +plugin: user-interact-verify +category_id: com.canonical.plainbox::suspend +id: suspend/wireless_connection_after_suspend_wpa_n_manual +depends: suspend/suspend_advanced +estimated_duration: 120.0 +requires: + device.category == 'WIRELESS' + environment.ROUTERS == 'single' +user: root +environ: ROUTER_SSID ROUTER_PSK +command: + trap "nmcli con delete id $ROUTER_SSID" EXIT + if create_connection wifi $ROUTER_SSID --security=wpa --key=$ROUTER_PSK; then + connect_wireless # lp:1471663 + INTERFACE=`nmcli dev status | awk '/802-11-wireless|wifi/ {print $1}'` + iw dev $INTERFACE link + gateway_ping_test --interface=$INTERFACE + STATUS=$? + # We reconnect the Ethernet connection if any (lp:1471663) + WIRED=$(nmcli -f UUID,TYPE c | grep -oP ".*(?=\s+.*ethernet)") + if [[ ! -z $WIRED ]]; then + nmcli c up uuid $WIRED + fi + exit $STATUS + else + exit 1 + fi +_description: + PURPOSE: + Tests that the systems wireless hardware can connect to a router using WPA + security and the 802.11n protocols. + STEPS: + 1. Open your routers configuration tool + 2. Change the settings to only accept connections on the N wireless band + 3. Make sure the SSID is set to ROUTER_SSID + 4. Change the security settings to use WPA2 and ensure the PSK matches that set in ROUTER_PSK + 5. Commence the test to create a connection to the router and test the connection + VERIFICATION: + Verification is automated, do not change the automatically selected result. + +plugin: user-interact-verify +category_id: com.canonical.plainbox::suspend +id: suspend/wireless_connection_after_suspend_open_n_manual +depends: suspend/suspend_advanced +estimated_duration: 120.0 +requires: + device.category == 'WIRELESS' + environment.ROUTERS == 'single' +user: root +environ: ROUTER_SSID +command: + trap "nmcli con delete id $ROUTER_SSID" EXIT + if create_connection wifi $ROUTER_SSID; then + connect_wireless # lp:1471663 + INTERFACE=`nmcli dev status | awk '/802-11-wireless|wifi/ {print $1}'` + iw dev $INTERFACE link + gateway_ping_test --interface=$INTERFACE + STATUS=$? + # We reconnect the Ethernet connection if any (lp:1471663) + WIRED=$(nmcli -f UUID,TYPE c | grep -oP ".*(?=\s+.*ethernet)") + if [[ ! -z $WIRED ]]; then + nmcli c up uuid $WIRED + fi + exit $STATUS + else + exit 1 + fi +_description: + PURPOSE: + Tests that the systems wireless hardware can connect to a router using + no security and the 802.11n protocol. + STEPS: + 1. Open your routers configuration tool + 2. Change the settings to only accept connections on the N wireless band + 3. Make sure the SSID is set to ROUTER_SSID + 4. Change the security settings to use no security + 5. Commence the test to create a connection to the router and test the connection + VERIFICATION: + Verification is automated, do not change the automatically selected result. + +plugin: user-interact-verify +category_id: com.canonical.plainbox::suspend +id: suspend/wireless_connection_after_suspend_wpa_ac_manual +depends: suspend/suspend_advanced +estimated_duration: 120.0 +requires: + device.category == 'WIRELESS' + environment.ROUTERS == 'single' +user: root +environ: ROUTER_SSID ROUTER_PSK +command: + trap "nmcli con delete id $ROUTER_SSID" EXIT + if create_connection wifi $ROUTER_SSID --security=wpa --key=$ROUTER_PSK; then + connect_wireless # lp:1471663 + INTERFACE=`nmcli dev status | awk '/802-11-wireless|wifi/ {print $1}'` + iw dev $INTERFACE link + gateway_ping_test --interface=$INTERFACE + STATUS=$? + # We reconnect the Ethernet connection if any (lp:1471663) + WIRED=$(nmcli -f UUID,TYPE c | grep -oP ".*(?=\s+.*ethernet)") + if [[ ! -z $WIRED ]]; then + nmcli c up uuid $WIRED + fi + exit $STATUS + else + exit 1 + fi +_description: + PURPOSE: + Tests that the systems wireless hardware can connect to a router using WPA + security and the 802.11ac protocol. + STEPS: + 1. Open your routers configuration tool + 2. Change the settings to only accept connections on the 802.11ac protocol. + 3. Make sure the SSID is set to ROUTER_SSID + 4. Change the security settings to use WPA2 and ensure the PSK matches that set in ROUTER_PSK + 5. Commence the test to create a connection to the router and test the connection + VERIFICATION: + Verification is automated, do not change the automatically selected result. + +plugin: user-interact-verify +category_id: com.canonical.plainbox::suspend +id: suspend/wireless_connection_after_suspend_open_ac_manual +depends: suspend/suspend_advanced +estimated_duration: 120.0 +requires: + device.category == 'WIRELESS' + environment.ROUTERS == 'single' +user: root +environ: ROUTER_SSID +command: + trap "nmcli con delete id $ROUTER_SSID" EXIT + if create_connection wifi $ROUTER_SSID; then + connect_wireless # lp:1471663 + INTERFACE=`nmcli dev status | awk '/802-11-wireless|wifi/ {print $1}'` + iw dev $INTERFACE link + gateway_ping_test --interface=$INTERFACE + STATUS=$? + # We reconnect the Ethernet connection if any (lp:1471663) + WIRED=$(nmcli -f UUID,TYPE c | grep -oP ".*(?=\s+.*ethernet)") + if [[ ! -z $WIRED ]]; then + nmcli c up uuid $WIRED + fi + exit $STATUS + else + exit 1 + fi +_description: + PURPOSE: + Tests that the systems wireless hardware can connect to a router using + no security and the 802.11ac protocol. + STEPS: + 1. Open your routers configuration tool + 2. Change the settings to only accept connections on the 802.11ac protocol. + 3. Make sure the SSID is set to ROUTER_SSID + 4. Change the security settings to use no security + 5. Commence the test to create a connection to the router and test the connection + VERIFICATION: + Verification is automated, do not change the automatically selected result. + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/bluetooth_detect_after_suspend +depends: suspend/suspend_advanced bluetooth/detect-output +estimated_duration: 1.2 +requires: + package.name == 'bluez' + device.category == 'BLUETOOTH' +command: + if rfkill list bluetooth | grep -q 'Hard blocked: yes'; then + echo "rfkill shows BT is hard blocked" + fi + if rfkill list bluetooth | grep -q 'Soft blocked: yes'; then + echo "rfkill shows BT is soft blocked, removing before testing" + rfkill unblock bluetooth + sleep 3 + fi + output=$(hcitool dev | tail -n+2 | awk '{print $2}') + echo $output | diff $PLAINBOX_SESSION_SHARE/bluetooth_address - + if [ -z "$output" ]; then + echo "BT hardware not available" + exit 1 + fi +_description: + This test grabs the hardware address of the bluetooth adapter after suspend and compares it to the address grabbed before suspend. + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/bluetooth_obex_send_after_suspend +depends: suspend/suspend_advanced +estimated_duration: 10.0 +requires: + package.name == 'bluez' + package.name == 'obexftp' + device.category == 'BLUETOOTH' +command: + if [ -z "$BTDEVADDR" ] + then + echo "btdevaddr option not set to device address of Bluetooth target in checkbox.ini" + exit 1 + fi + if rfkill list bluetooth | grep -q 'Hard blocked: yes' + then + echo "rfkill list shows BT is hard blocked" + fi + if rfkill list bluetooth | grep -q 'Soft blocked: yes' + then + echo "rfkill list shows BT is soft blocked, removing before testing" + rfkill unblock bluetooth + sleep 3 + fi + set -o pipefail; bluetooth_test $PLAINBOX_PROVIDER_DATA/images/JPEG_Color_Image_Ubuntu.jpg $BTDEVADDR send 2>&1 | ansi_parser +_description: + This is an automated Bluetooth file transfer test. It sends an image to the device specified by the BTDEVADDR environment variable. + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/bluetooth_obex_send_after_suspend_auto +depends: suspend/suspend_advanced_auto +estimated_duration: 10.0 +requires: + package.name == 'bluez' + package.name == 'obexftp' + device.category == 'BLUETOOTH' +command: + if [ -z "$BTDEVADDR" ] + then + echo "btdevaddr option not set to device address of Bluetooth target in checkbox.ini" + exit 1 + fi + if rfkill list bluetooth | grep -q 'Hard blocked: yes' + then + echo "rfkill list shows BT is hard blocked" + fi + if rfkill list bluetooth | grep -q 'Soft blocked: yes' + then + echo "rfkill list shows BT is soft blocked, removing before testing" + rfkill unblock bluetooth + sleep 3 + fi + set -o pipefail; bluetooth_test $PLAINBOX_PROVIDER_DATA/images/JPEG_Color_Image_Ubuntu.jpg $BTDEVADDR send 2>&1 | ansi_parser +_description: + This is an automated Bluetooth file transfer test. It sends an image to the device specified by the BTDEVADDR environment variable. + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/bluetooth_obex_browse_after_suspend +depends: suspend/suspend_advanced +estimated_duration: 10.0 +requires: + package.name == 'bluez' + package.name == 'obexftp' + device.category == 'BLUETOOTH' +command: + if [ -z "$BTDEVADDR" ] + then + echo "btdevaddr option not set to device address of Bluetooth target in checkbox.ini" + exit 1 + fi + if rfkill list bluetooth | grep -q 'Hard blocked: yes' + then + echo "rfkill list shows BT is hard blocked" + fi + if rfkill list bluetooth | grep -q 'Soft blocked: yes' + then + echo "rfkill list shows BT is soft blocked, removing before testing" + rfkill unblock bluetooth + sleep 3 + fi + set -o pipefail; bluetooth_test $PLAINBOX_PROVIDER_DATA/images/JPEG_Color_Image_Ubuntu.jpg $BTDEVADDR browse 2>&1 | ansi_parser +_description: + This is an automated Bluetooth test. It emulates browsing on a remote device specified by the BTDEVADDR environment variable. + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/bluetooth_obex_browse_after_suspend_auto +depends: suspend/suspend_advanced_auto +estimated_duration: 20.0 +requires: + package.name == 'bluez' + package.name == 'obexftp' + device.category == 'BLUETOOTH' +command: + if [ -z "$BTDEVADDR" ] + then + echo "btdevaddr option not set to device address of Bluetooth target in checkbox.ini" + exit 1 + fi + if rfkill list bluetooth | grep -q 'Hard blocked: yes' + then + echo "rfkill list shows BT is hard blocked" + fi + if rfkill list bluetooth | grep -q 'Soft blocked: yes' + then + echo "rfkill list shows BT is soft blocked, removing before testing" + rfkill unblock bluetooth + sleep 3 + fi + set -o pipefail; bluetooth_test $PLAINBOX_PROVIDER_DATA/images/JPEG_Color_Image_Ubuntu.jpg $BTDEVADDR browse 2>&1 | ansi_parser +_description: + This is an automated Bluetooth test. It emulates browsing on a remote device specified by the BTDEVADDR environment variable. + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/bluetooth_obex_get_after_suspend +estimated_duration: 20.0 +depends: suspend/suspend_advanced +requires: + package.name == 'bluez' + package.name == 'obexftp' + device.category == 'BLUETOOTH' +command: + if [ -z "$BTDEVADDR" ] + then + echo "btdevaddr option not set to device address of Bluetooth target in checkbox.ini" + exit 1 + fi + if rfkill list bluetooth | grep -q 'Hard blocked: yes' + then + echo "rfkill list shows BT is hard blocked" + fi + if rfkill list bluetooth | grep -q 'Soft blocked: yes' + then + echo "rfkill list shows BT is soft blocked, removing before testing" + rfkill unblock bluetooth + sleep 3 + fi + set -o pipefail; bluetooth_test $PLAINBOX_PROVIDER_DATA/images/JPEG_Color_Image_Ubuntu.jpg $BTDEVADDR get 2>&1 | ansi_parser +_description: + This is an automated Bluetooth test. It receives the given file from a remote host specified by the BTDEVADDR environment variable + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/bluetooth_obex_get_after_suspend_auto +depends: suspend/suspend_advanced_auto +estimated_duration: 20.0 +requires: + package.name == 'bluez' + package.name == 'obexftp' + device.category == 'BLUETOOTH' +command: + if [ -z "$BTDEVADDR" ] + then + echo "btdevaddr option not set to device address of Bluetooth target in checkbox.ini" + exit 1 + fi + if rfkill list bluetooth | grep -q 'Hard blocked: yes' + then + echo "rfkill list shows BT is hard blocked" + fi + if rfkill list bluetooth | grep -q 'Soft blocked: yes' + then + echo "rfkill list shows BT is soft blocked, removing before testing" + rfkill unblock bluetooth + sleep 3 + fi + set -o pipefail; bluetooth_test $PLAINBOX_PROVIDER_DATA/images/JPEG_Color_Image_Ubuntu.jpg $BTDEVADDR get 2>&1 | ansi_parser +_description: + This is an automated Bluetooth test. It receives the given file from a remote host specified by the BTDEVADDR environment variable + +plugin: user-interact-verify +category_id: com.canonical.plainbox::suspend +id: suspend/cycle_resolutions_after_suspend +estimated_duration: 120.0 +requires: package.name == 'xorg' +depends: suspend/suspend_advanced +command: xrandr_cycle --keyword=after_suspend --screenshot-dir $PLAINBOX_SESSION_SHARE +_description: + PURPOSE: + This test will cycle through the detected display modes + STEPS: + 1. Click "Test" and the display will cycle trough the display modes + VERIFICATION: + Did your display look fine in the detected mode? + +unit: template +template-resource: graphics_card +template-filter: graphics_card.prime_gpu_offload == 'Off' +plugin: user-interact-verify +category_id: com.canonical.plainbox::suspend +id: suspend/{index}_cycle_resolutions_after_suspend_{product_slug} +requires: package.name == 'xorg' +depends: suspend/{index}_suspend_after_switch_to_card_{product_slug} +estimated_duration: 120.0 +command: xrandr_cycle --keyword={index}_after_suspend --screenshot-dir $PLAINBOX_SESSION_SHARE +_description: + PURPOSE: + This test will cycle through the detected display modes + STEPS: + 1. Click "Test" and the display will cycle trough the display modes + VERIFICATION: + Did your display look fine in the detected mode? + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/cycle_resolutions_after_suspend_auto +estimated_duration: 1.2 +requires: package.name == 'xorg' +depends: suspend/suspend_advanced_auto +_description: + This test will check to make sure supported video modes work after a suspend and resume. + This is done automatically by taking screenshots and uploading them as an attachment. +command: xrandr_cycle --keyword=after_suspend --screenshot-dir $PLAINBOX_SESSION_SHARE + +plugin: attachment +category_id: com.canonical.plainbox::suspend +id: suspend/xrandr_screens_after_suspend.tar.gz +depends: suspend/cycle_resolutions_after_suspend +command: [ -f $PLAINBOX_SESSION_SHARE/xrandr_screens_after_suspend.tgz ] && base64 $PLAINBOX_SESSION_SHARE/xrandr_screens_after_suspend.tgz +_description: This attaches screenshots from the suspend/cycle_resolutions_after_suspend test to the results submission. + +unit: template +template-resource: graphics_card +template-filter: graphics_card.prime_gpu_offload == 'Off' +plugin: attachment +category_id: com.canonical.plainbox::suspend +id: suspend/{index}_xrandr_screens_after_suspend.tar.gz +depends: suspend/{index}_cycle_resolutions_after_suspend_{product_slug} +command: [ -f $PLAINBOX_SESSION_SHARE/{index}_xrandr_screens_after_suspend.tgz ] && base64 $PLAINBOX_SESSION_SHARE/{index}_xrandr_screens_after_suspend.tgz +_description: This attaches screenshots from the suspend/cycle_resolutions_after_suspend test to the results submission. + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/record_playback_after_suspend +estimated_duration: 10.0 +depends: suspend/suspend_advanced +requires: + package.name == 'python3-gi' + package.name == 'gir1.2-gstreamer-1.0' + package.name == 'libgstreamer1.0-0' + package.name == 'gstreamer1.0-plugins-good' + package.name == 'gstreamer1.0-pulseaudio' + package.name == 'alsa-base' + device.category == 'AUDIO' +command: audio_test +_description: + This will check to make sure that your audio device works properly after a suspend and resume. This may work fine with speakers and onboard microphone, however, it works best if used with a cable connecting the audio-out jack to the audio-in jack. + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/record_playback_after_suspend_auto +estimated_duration: 10.0 +depends: suspend/suspend_advanced_auto +requires: + package.name == 'python3-gi' + package.name == 'gir1.2-gstreamer-1.0' + package.name == 'libgstreamer1.0-0' + package.name == 'gstreamer1.0-plugins-good' + package.name == 'gstreamer1.0-pulseaudio' + package.name == 'alsa-base' + device.category == 'AUDIO' +command: audio_test +_description: + This will check to make sure that your audio device works properly after a suspend and resume. This may work fine with speakers and onboard microphone, however, it works best if used with a cable connecting the audio-out jack to the audio-in jack. + +plugin: attachment +category_id: com.canonical.plainbox::suspend +id: suspend/suspend-auto-single-log-attach +command: [ -e $PLAINBOX_SESSION_SHARE/suspend_auto_single_log ] && xz -c $PLAINBOX_SESSION_SHARE/suspend_auto_single_log | base64 +_description: + Attaches the log from the single suspend/resume test to the results + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/screenshot_after_suspend +estimated_duration: 10.0 +depends: suspend/suspend_advanced_auto +requires: package.name == 'fswebcam' +command: set -o pipefail; camera_test still --device=/dev/external_webcam -f ${PLAINBOX_SESSION_SHARE}/screenshot_after_suspend.jpg -q 2>&1 | ansi_parser +_description: + PURPOSE: + Take a screengrab of the current screen after suspend (logged on Unity desktop) + STEPS: + 1. Take picture using USB webcam + VERIFICATION: + Review attachment manually later + +plugin: attachment +category_id: com.canonical.plainbox::suspend +id: screenshot_after_suspend.jpg +depends: suspend/screenshot_after_suspend +command: base64 ${PLAINBOX_SESSION_SHARE}/screenshot_after_suspend.jpg +_description: Attaches the screenshot captured in graphics/screenshot. + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/gpu_lockup_after_suspend +estimated_duration: 10.0 +requires: + package.name == 'wmctrl' + package.name == 'mesa-utils' + package.name == 'firefox' +depends: suspend/suspend_advanced_auto +command: gpu_test +_description: + PURPOSE: + Do some challenging operations then check for lockup on the GPU + STEPS: + 1. Create 2 glxgears windows and move them quickly + 2. Switch workspaces with wmctrl + 3. Launch an HTML5 video playback in firefox + VERIFICATION: + After a 60s workload, check kern.log for reported GPU errors + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/wifi_resume_time +depends: suspend/suspend_advanced +requires: + device.category == 'WIRELESS' +command: network_reconnect_resume_test -t 90 -d wifi +estimated_duration: 0.530 +_description: + Checks the length of time it takes to reconnect an existing wifi connection + after a suspend/resume cycle. + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/wifi_resume_time_auto +estimated_duration: 1.2 +depends: suspend/suspend_advanced_auto +requires: + device.category == 'WIRELESS' +command: network_reconnect_resume_test -t 90 -d wifi +_description: + Checks the length of time it takes to reconnect an existing wifi connection + after a suspend/resume cycle. + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/network_resume_time +depends: suspend/suspend_advanced +estimated_duration: 1.2 +requires: + device.category == 'NETWORK' +command: network_reconnect_resume_test -t 10 -d wired +_description: + Checks the length of time it takes to reconnect an existing wired connection + after a suspend/resume cycle. + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/network_resume_time_auto +depends: suspend/suspend_advanced_auto +estimated_duration: 1.2 +requires: + device.category == 'NETWORK' +command: network_reconnect_resume_test -t 10 -d wired +_description: + Checks the length of time it takes to reconnect an existing wired connection + after a suspend/resume cycle. + +plugin: manual +category_id: com.canonical.plainbox::suspend +id: suspend/led_after_suspend/power +depends: suspend/suspend_advanced +estimated_duration: 120.0 +_description: + PURPOSE: + Validate that the power LED operated the same after resuming from suspend + STEPS: + 1. Power LED should be on while device is switched on + VERIFICATION: + Does the power LED remain on after resuming from suspend? + +plugin: manual +category_id: com.canonical.plainbox::suspend +id: suspend/led_after_suspend/battery-charging +estimated_duration: 120.0 +depends: suspend/suspend_advanced +_description: + PURPOSE: + Validate that the battery light shows charging status after resuming from suspend + STEPS: + 1. Let system run on battery for a while + 2. Plug in AC plug + VERIFICATION: + Did the battery indicator LED still turn orange after resuming from suspend? + +plugin: manual +category_id: com.canonical.plainbox::suspend +id: suspend/led_after_suspend/battery-charged +depends: suspend/suspend_advanced +estimated_duration: 120.0 +_description: + PURPOSE: + Validate that the battery LED properly displays charged status after resuming from suspend + STEPS: + 1. Let system run on battery for a short time + 2. Plug in AC + 3. Let system run on AC + VERIFICATION: + Does the orange battery LED still shut off when system is fully charged after resuming from suspend? + +plugin: manual +category_id: com.canonical.plainbox::suspend +id: suspend/led_after_suspend/battery-low +depends: suspend/suspend_advanced +estimated_duration: 120.0 +_description: + PURPOSE: + Validate that the battery LED indicated low power after resuming from suspend + STEPS: + 1. Let system run on battery for several hours + 2. Monitor battery LED carefully + VERIFICATION: + Does the LED light orange when battery is low after resuming from suspend? + +plugin: user-interact-verify +category_id: com.canonical.plainbox::suspend +id: suspend/led_after_suspend/hdd +depends: suspend/suspend_advanced +estimated_duration: 120.0 +command: led_hdd_test +_description: + PURPOSE: + Validate that the HDD LED still operates as expected after resuming from suspend + STEPS: + 1. Select "Test" to write and read a temporary file for a few seconds + 2. HDD LED should blink when writing to/reading from HDD + VERIFICATION: + Did the HDD LED still blink with HDD activity after resuming from suspend? + +plugin: user-interact-verify +category_id: com.canonical.plainbox::suspend +id: suspend/led_after_suspend/numeric-keypad +estimated_duration: 120.0 +depends: suspend/suspend_advanced +command: keyboard_test +_description: + PURPOSE: + Validate that the numeric keypad LED operates the same before and after resuming from suspend + STEPS: + 1. Press "Block Num" key to toggle numeric keypad LED + 2. Click on the "Test" button to open a window to verify your typing + 3. Type using the numeric keypad both when the LED is on and off + VERIFICATION: + 1. Numeric keypad LED status should toggle everytime the "Block Num" key is pressed + 2. Numbers should only be entered in the keyboard verification window when the LED is on + +plugin: manual +category_id: com.canonical.plainbox::suspend +id: suspend/led_after_suspend/caps-lock +depends: suspend/suspend_advanced +estimated_duration: 120.0 +_description: + PURPOSE: + Validate that the Caps Lock key operates the same before and after resuming from suspend + STEPS: + 1. Press "Block Cap Keys" to activate/deactivate cap keys blocking + 2. Cap Keys LED should be switched on/off every time the key is pressed + VERIFICATION: + Did the Cap Keys LED light as expected after resuming from suspend? + +plugin: manual +category_id: com.canonical.plainbox::suspend +id: suspend/led_after_suspend/wlan +depends: suspend/suspend_advanced +estimated_duration: 120.0 +_description: + PURPOSE: + WLAN LED verification after resuming from suspend + STEPS: + 1. Make sure WLAN connection is established + 2. WLAN LED should light + VERIFICATION: + Did the WLAN LED light as expected after resuming from suspend? + +plugin: manual +category_id: com.canonical.plainbox::suspend +id: suspend/led_after_suspend/wlan-disabled +depends: suspend/suspend_advanced +estimated_duration: 120.0 +_description: + PURPOSE: + Validate that WLAN LED shuts off when disabled after resuming from suspend + STEPS: + 1. Connect to AP + 2. Use Physical switch to disable WLAN + 3. Re-enable + 4. Use Network-Manager to disable WLAN + VERIFICATION: + Did the LED turn off then WLAN is disabled after resuming from suspend? + +plugin: manual +category_id: com.canonical.plainbox::suspend +id: suspend/led_after_suspend/bluetooth +depends: suspend/suspend_advanced +estimated_duration: 120.0 +_description: + PURPOSE: + Validate that the Bluetooth LED turns on and off when BT is enabled/disabled after resuming from suspend + STEPS: + 1. Switch bluetooth off from a hardware switch (if present) + 2. Switch bluetooth back on + 3. Switch bluetooth off from the panel applet + 4. Switch bluetooth back on + VERIFICATION: + Did the bluetooth LED turn off and on twice after resuming from suspend? + +plugin: user-interact-verify +category_id: com.canonical.plainbox::suspend +id: suspend/led_after_suspend/camera +estimated_duration: 120.0 +depends: + camera/detect + suspend/suspend_advanced +command: camera_test led +_description: + PURPOSE: + Validate that the camera LED still works as expected after resuming from suspend + STEPS: + 1. Select Test to activate camera + 2. Camera LED should light for a few seconds + VERIFICATION: + Did the camera LED still turn on and off after resuming from suspend? + +plugin: manual +category_id: com.canonical.plainbox::suspend +id: suspend/led_after_suspend/touchpad +depends: suspend/suspend_advanced +estimated_duration: 120.0 +_description: + PURPOSE: + Touchpad LED verification after resuming from suspend + STEPS: + 1. Click on the touchpad button or press key combination to enable/disable touchpad button + 2. Slide your finger on the touchpad + VERIFICATION: + 1. Touchpad LED status should toggle everytime the button is clicked or the key combination is pressed + 2. When the LED is on, the mouse pointer should move on touchpad usage + 3. When the LED is off, the mouse pointer should not move on touchpad usage + +plugin: manual +category_id: com.canonical.plainbox::suspend +id: suspend/led_after_suspend/wireless +depends: suspend/suspend_advanced +estimated_duration: 120.0 +_description: + PURPOSE: + Validate Wireless (WLAN + Bluetooth) LED operated the same after resuming from suspend + STEPS: + 1. Make sure WLAN connection is established and Bluetooth is enabled. + 2. WLAN/Bluetooth LED should light + 3. Switch WLAN and Bluetooth off from a hardware switch (if present) + 4. Switch them back on + 5. Switch WLAN and Bluetooth off from the panel applet + 6. Switch them back on + VERIFICATION: + Did the WLAN/Bluetooth LED light as expected after resuming from suspend? + +plugin: manual +category_id: com.canonical.plainbox::suspend +id: suspend/keys_after_suspend/brightness +depends: suspend/suspend_advanced +estimated_duration: 120.0 +requires: dmi.product in ['Notebook','Laptop','Portable','All In One','All-In-One','AIO'] +_description: + PURPOSE: + This test will test the brightness key after resuming from suspend + STEPS: + 1. Press the brightness buttons on the keyboard + VERIFICATION: + Did the brightness change following to your key presses after resuming from suspend? + +plugin: user-interact-verify +category_id: com.canonical.plainbox::suspend +id: suspend/keys_after_suspend/volume +depends: suspend/suspend_advanced +estimated_duration: 120.0 +requires: + device.category == 'KEYBOARD' +user: root +command: key_test -s '0xe02e,0xe0ae:Volume Up' '0xe030,0xe0b0:Volume Down' +_description: + PURPOSE: + This test will test the volume keys of your keyboard after resuming from suspend + STEPS: + Skip this test if your computer has no volume keys. + 1. Click test to open a window on which to test the volume keys. + 2. If all the keys work, the test will be marked as passed. + VERIFICATION: + Did the volume change following to your key presses? + +plugin: user-interact-verify +category_id: com.canonical.plainbox::suspend +id: suspend/keys_after_suspend/mute +depends: suspend/suspend_advanced +estimated_duration: 120.0 +requires: + device.category == 'KEYBOARD' +user: root +command: key_test -s '0xe020,0xe0a0:Mute' +_description: + PURPOSE: + This test will test the mute key of your keyboard after resuming from suspend + STEPS: + 1. Click test to open a window on which to test the mute key. + 2. If the key works, the test will pass and the window will close. + VERIFICATION: + Did the volume mute following your key presses? + +plugin: manual +category_id: com.canonical.plainbox::suspend +id: suspend/keys_after_suspend/sleep +depends: suspend/suspend_advanced +estimated_duration: 120.0 +requires: + device.category == 'KEYBOARD' +_description: + PURPOSE: + This test will test the sleep key after resuming from suspend + STEPS: + 1. Press the sleep key on the keyboard + 2. Wake your system up by pressing the power button + VERIFICATION: + Did the system go to sleep after pressing the sleep key after resuming from suspend? + +plugin: user-interact-verify +category_id: com.canonical.plainbox::suspend +id: suspend/keys_after_suspend/battery-info +depends: suspend/suspend_advanced +estimated_duration: 120.0 +requires: dmi.product in ['Notebook','Laptop','Portable'] +user: root +command: key_test -s '0xe071,0xef1:Battery Info' +_description: + PURPOSE: + This test will test the battery information key after resuming from suspend + STEPS: + Skip this test if you do not have a Battery Button. + 1. Click Test to begin + 2. Press the Battery Info button (or combo like Fn+F3) + 3: Close the Power Statistics tool if it opens + VERIFICATION: + Did the Battery Info key work as expected after resuming from suspend? + +plugin: manual +category_id: com.canonical.plainbox::suspend +id: suspend/keys_after_suspend/wireless +depends: suspend/suspend_advanced +estimated_duration: 120.0 +requires: dmi.product in ['Notebook','Laptop','Portable'] +_description: + PURPOSE: + This test will test the wireless key after resuming from suspend + STEPS: + 1. Press the wireless key on the keyboard + 2. Press the same key again + VERIFICATION: + Did the wireless go off on the first press and on again on the second after resuming from suspend? + +plugin: user-interact-verify +category_id: com.canonical.plainbox::suspend +id: suspend/keys_after_suspend/media-control +estimated_duration: 120.0 +depends: suspend/suspend_advanced +requires: + device.category == 'KEYBOARD' +user: root +command: key_test -s 0xe010,0xe090:Previous 0xe024,0xe0a4:Stop 0xe019,0xe099:Next 0xe022,0xe0a2:Play +_description: + PURPOSE: + This test will test the media keys of your keyboard after resuming from suspend + STEPS: + Skip this test if your computer has no media keys. + 1. Click test to open a window on which to test the media keys. + 2. If all the keys work, the test will be marked as passed. + VERIFICATION: + Do the keys work as expected after resuming from suspend? + +plugin: user-interact-verify +category_id: com.canonical.plainbox::suspend +id: suspend/keys_after_suspend/super +depends: suspend/suspend_advanced +estimated_duration: 120.0 +requires: + device.category == 'KEYBOARD' +user: root +command: key_test -s '0xe05b,0xe0db:Left Super Key' +_description: + PURPOSE: + This test will test the super key of your keyboard after resuming from suspend + STEPS: + 1. Click test to open a window on which to test the super key. + 2. If the key works, the test will pass and the window will close. + VERIFICATION: + Does the super key work as expected after resuming from suspend? + +plugin: manual +category_id: com.canonical.plainbox::suspend +id: suspend/keys_after_suspend/video-out +depends: suspend/suspend_advanced +estimated_duration: 120.0 +requires: dmi.product in ['Notebook','Laptop','Portable'] +_description: + PURPOSE: + Validate that the External Video hot key is working as expected after resuming from suspend + STEPS: + 1. Plug in an external monitor + 2. Press the display hot key to change the monitors configuration + VERIFICATION: + Check that the video signal can be mirrored, extended, displayed on external or onboard only, after resuming from suspend. + +plugin: manual +category_id: com.canonical.plainbox::suspend +id: suspend/keys_after_suspend/touchpad +depends: suspend/suspend_advanced +estimated_duration: 120.0 +requires: dmi.product in ['Notebook','Laptop','Portable'] +_description: + PURPOSE: + Verify touchpad hotkey toggles touchpad functionality on and off after resuming from suspend + STEPS: + 1. Verify the touchpad is functional + 2. Tap the touchpad toggle hotkey + 3. Tap the touchpad toggle hotkey again + VERIFICATION: + Verify the touchpad has been disabled and re-enabled. + +plugin: user-interact +category_id: com.canonical.plainbox::suspend +id: suspend/usb_insert_after_suspend +estimated_duration: 30.0 +depends: + suspend/suspend_advanced +command: removable_storage_watcher insert usb +_description: + PURPOSE: + This test will check that the system correctly detects the insertion of + a USB storage device after suspend and resume. + STEPS: + 1. Click "Test" and insert a USB storage device (pen-drive/HDD). + (Note: this test will time-out after 20 seconds.) + 2. Do not unplug the device after the test. + VERIFICATION: + The verification of this test is automated. Do not change the + automatically selected result. + +plugin: user-interact +category_id: com.canonical.plainbox::suspend +id: suspend/usb3_insert_after_suspend +estimated_duration: 30.0 +requires: + usb.usb3 == 'supported' +depends: + suspend/suspend_advanced +command: removable_storage_watcher -m 500000000 insert usb +_description: + PURPOSE: + This test will check that the system correctly detects the insertion of + a USB 3.0 storage device after suspend and resume. + STEPS: + 1. Click "Test" and insert a USB 3.0 storage device (pen-drive/HDD) in + a USB 3.0 port. (Note: this test will time-out after 20 seconds.) + 2. Do not unplug the device after the test. + VERIFICATION: + The verification of this test is automated. Do not change the + automatically selected result. + +plugin: user-interact +category_id: com.canonical.plainbox::suspend +id: suspend/usb_remove_after_suspend +estimated_duration: 30.0 +depends: + suspend/usb_insert_after_suspend +command: removable_storage_watcher remove usb +_description: + PURPOSE: + This test will check that the system correctly detects the removal of + a USB storage device after suspend. + STEPS: + 1. Click "Test" and remove the USB device. + (Note: this test will time-out after 20 seconds.) + VERIFICATION: + The verification of this test is automated. Do not change the + automatically selected result. + +plugin: user-interact +category_id: com.canonical.plainbox::suspend +id: suspend/usb3_remove_after_suspend +estimated_duration: 30.0 +depends: + suspend/usb3_insert_after_suspend +requires: + usb.usb3 == 'supported' +command: removable_storage_watcher -m 500000000 remove usb +_description: + PURPOSE: + This test will check that the system correctly detects the removal of + a USB 3.0 storage device after suspend + STEPS: + 1. Click "Test" and remove the USB 3.0 device. + (Note: this test will time-out after 20 seconds.) + VERIFICATION: + The verification of this test is automated. Do not change the + automatically selected result. + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/usb_storage_automated_after_suspend +estimated_duration: 1.2 +depends: suspend/usb_insert_after_suspend +user: root +command: removable_storage_test -s 268400000 usb +_description: + This test is automated and executes after the suspend/usb_insert_after_suspend + test is run. + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/usb3_storage_automated_after_suspend +estimated_duration: 1.2 +requires: + usb.usb3 == 'supported' +depends: suspend/usb3_insert_after_suspend +user: root +command: removable_storage_test -s 268400000 -m 500000000 usb --driver xhci_hcd +_description: + This test is automated and executes after the suspend/usb3_insert_after_suspend + test is run. + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/usb_storage_preinserted_after_suspend +estimated_duration: 1.2 +user: root +depends: suspend/suspend_advanced_auto +command: removable_storage_test -l usb && removable_storage_test -s 268400000 usb +_description: + This is an automated version of usb/storage-automated and assumes that the + server has usb storage devices plugged in prior to checkbox execution. It + is intended for servers and SRU automated testing. + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/usb3_storage_preinserted_after_suspend +estimated_duration: 1.2 +user: root +requires: + usb.usb3 == 'supported' +depends: suspend/suspend_advanced_auto +command: removable_storage_test -l usb && removable_storage_test -s 268400000 -m 500000000 usb --driver xhci_hcd +_description: + This is an automated version of usb3/storage-automated and assumes that the + server has usb 3.0 storage devices plugged in prior to checkbox execution. It + is intended for servers and SRU automated testing. + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/usb_performance_after_suspend +depends: suspend/usb_insert_after_suspend +user: root +estimated_duration: 45.00 +command: removable_storage_test -s 268400000 -p 15 usb +_description: + This test will check that your USB 2.0 port transfers data at a + minimum expected speed. + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/usb3_superspeed_performance_after_suspend +requires: + usb.usb3 == 'supported' +depends: suspend/usb3_insert_after_suspend +user: root +estimated_duration: 45.00 +command: removable_storage_test -s 268400000 -m 500000000 usb --driver xhci_hcd +_description: + This test will check that your USB 3.0 port could be recognized + as SuperSpeed USB device using xhci_hcd driver and transfers data correctly. + +plugin: user-interact +category_id: com.canonical.plainbox::suspend +id: suspend/mmc-insert-after-suspend +estimated_duration: 30.0 +depends: suspend/suspend_advanced +imports: from com.canonical.plainbox import manifest +requires: + package.name == 'udisks2' or snap.name == 'udisks2' + manifest.has_card_reader == 'True' +command: removable_storage_watcher --memorycard insert sdio usb scsi +_description: + PURPOSE: + This test will check that the systems media card reader can + detect the insertion of an MMC card after the system has been suspended + STEPS: + 1. Click "Test" and insert an MMC card into the reader. + If a file browser opens up, you can safely close it. + (Note: this test will time-out after 20 seconds.) + 2. Do not remove the device after this test. + VERIFICATION: + The verification of this test is automated. Do not change the + automatically selected result. + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/mmc-storage-after-suspend +depends: suspend/mmc-insert-after-suspend +estimated_duration: 10.0 +user: root +command: removable_storage_test -s 67120000 --memorycard sdio usb scsi +_description: + This test is automated and executes after the mediacard/mmc-insert-after-suspend test + is run. It tests reading and writing to the MMC card after the system has been suspended. + +plugin: user-interact +category_id: com.canonical.plainbox::suspend +id: suspend/mmc-remove-after-suspend +depends: suspend/mmc-insert-after-suspend +estimated_duration: 30.0 +command: removable_storage_watcher --memorycard remove sdio usb scsi +_description: + PURPOSE: + This test will check that the system correctly detects the removal + of an MMC card from the systems card reader after the system has been suspended. + STEPS: + 1. Click "Test" and remove the MMC card from the reader. + (Note: this test will time-out after 20 seconds.) + VERIFICATION: + The verification of this test is automated. Do not change the + automatically selected result. + +plugin: user-interact +category_id: com.canonical.plainbox::suspend +id: suspend/sd-insert-after-suspend +estimated_duration: 30.0 +depends: suspend/suspend_advanced +imports: from com.canonical.plainbox import manifest +requires: + package.name == 'udisks2' or snap.name == 'udisks2' + manifest.has_card_reader == 'True' +command: removable_storage_watcher --memorycard insert sdio usb scsi +_description: + PURPOSE: + This test will check that the systems media card reader can + detect the insertion of an UNLOCKED SD card after the system + has been suspended + STEPS: + 1. Click "Test" and insert an UNLOCKED SD card into the reader. + If a file browser opens up, you can safely close it. + (Note: this test will time-out after 20 seconds.) + 2. Do not remove the device after this test. + VERIFICATION: + The verification of this test is automated. Do not change the + automatically selected result. + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/sd-storage-after-suspend +estimated_duration: 10.0 +depends: suspend/sd-insert-after-suspend +user: root +command: removable_storage_test -s 268400000 --memorycard sdio usb scsi +_description: + This test is automated and executes after the mediacard/sd-insert-after-suspend test + is run. It tests reading and writing to the SD card after the system has been suspended. + +plugin: user-interact +category_id: com.canonical.plainbox::suspend +id: suspend/sd-remove-after-suspend +estimated_duration: 30.0 +depends: suspend/sd-insert-after-suspend +command: removable_storage_watcher --memorycard remove sdio usb scsi +_description: + PURPOSE: + This test will check that the system correctly detects + the removal of an SD card from the systems card reader + after the system has been suspended. + STEPS: + 1. Click "Test" and remove the SD card from the reader. + (Note: this test will time-out after 20 seconds.) + VERIFICATION: + The verification of this test is automated. Do not change the + automatically selected result. + +plugin: user-interact +category_id: com.canonical.plainbox::suspend +id: suspend/sdhc-insert-after-suspend +estimated_duration: 30.0 +depends: suspend/suspend_advanced +imports: from com.canonical.plainbox import manifest +requires: + package.name == 'udisks2' or snap.name == 'udisks2' + manifest.has_card_reader == 'True' +command: removable_storage_watcher --memorycard insert sdio usb scsi +_description: + PURPOSE: + This test will check that the systems media card reader can + detect the insertion of an UNLOCKED SDHC media card after the + system has been suspended + STEPS: + 1. Click "Test" and insert an UNLOCKED SDHC card into the reader. + If a file browser opens up, you can safely close it. + (Note: this test will time-out after 20 seconds.) + 2. Do not remove the device after this test. + VERIFICATION: + The verification of this test is automated. Do not change the + automatically selected result. + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/sdhc-storage-after-suspend +estimated_duration: 10.0 +depends: suspend/sdhc-insert-after-suspend +user: root +command: removable_storage_test -s 268400000 --memorycard sdio usb scsi +_description: + This test is automated and executes after the mediacard/sdhc-insert-after-suspend test + is run. It tests reading and writing to the SDHC card after the system has been suspended. + +plugin: user-interact +category_id: com.canonical.plainbox::suspend +id: suspend/sdhc-remove-after-suspend +estimated_duration: 30.0 +depends: suspend/sdhc-insert-after-suspend +command: removable_storage_watcher --memorycard remove sdio usb scsi +_description: + PURPOSE: + This test will check that the system correctly detects the removal + of an SDHC card from the systems card reader after the system has been suspended. + STEPS: + 1. Click "Test" and remove the SDHC card from the reader. + (Note: this test will time-out after 20 seconds.) + VERIFICATION: + The verification of this test is automated. Do not change the + automatically selected result. + +plugin: user-interact +category_id: com.canonical.plainbox::suspend +id: mediacard/cf-insert-after-suspend +estimated_duration: 30.0 +depends: suspend/suspend_advanced +imports: from com.canonical.plainbox import manifest +requires: + package.name == 'udisks2' or snap.name == 'udisks2' + manifest.has_card_reader == 'True' +command: removable_storage_watcher --memorycard insert sdio usb scsi +_description: + PURPOSE: + This test will check that the systems media card reader can + detect the insertion of a CF card after the system has been suspended + STEPS: + 1. Click "Test" and insert a CF card into the reader. + If a file browser opens up, you can safely close it. + (Note: this test will time-out after 20 seconds.) + 2. Do not remove the device after this test. + VERIFICATION: + The verification of this test is automated. Do not change the + automatically selected result. + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: mediacard/cf-storage-after-suspend +estimated_duration: 10.0 +depends: mediacard/cf-insert-after-suspend +user: root +command: removable_storage_test -s 268400000 --memorycard sdio usb scsi +_description: + This test is automated and executes after the mediacard/cf-insert-after-suspend test + is run. It tests reading and writing to the CF card after the system has been suspended. + +plugin: user-interact +category_id: com.canonical.plainbox::suspend +id: mediacard/cf-remove-after-suspend +estimated_duration: 30.0 +depends: mediacard/cf-insert-after-suspend +command: removable_storage_watcher --memorycard remove sdio usb scsi +_description: + PURPOSE: + This test will check that the system correctly detects the removal + of a CF card from the systems card reader after the system has been suspended. + STEPS: + 1. Click "Test" and remove the CF card from the reader. + (Note: this test will time-out after 20 seconds.) + VERIFICATION: + The verification of this test is automated. Do not change the + automatically selected result. + +plugin: user-interact +category_id: com.canonical.plainbox::suspend +id: mediacard/sdxc-insert-after-suspend +estimated_duration: 30.0 +depends: suspend/suspend_advanced +imports: from com.canonical.plainbox import manifest +requires: + package.name == 'udisks2' or snap.name == 'udisks2' + manifest.has_card_reader == 'True' +command: removable_storage_watcher --memorycard insert sdio usb scsi +_description: + PURPOSE: + This test will check that the systems media card reader can + detect the insertion of a SDXC card after the system has been suspended + STEPS: + 1. Click "Test" and insert a SDXC card into the reader. + If a file browser opens up, you can safely close it. + (Note: this test will time-out after 20 seconds.) + 2. Do not remove the device after this test. + VERIFICATION: + The verification of this test is automated. Do not change the + automatically selected result. + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: mediacard/sdxc-storage-after-suspend +estimated_duration: 10.0 +depends: mediacard/sdxc-insert-after-suspend +user: root +command: removable_storage_test -s 268400000 --memorycard sdio usb scsi +_description: + This test is automated and executes after the mediacard/sdxc-insert-after-suspend test + is run. It tests reading and writing to the SDXC card after the system has been suspended. + +plugin: user-interact +category_id: com.canonical.plainbox::suspend +id: mediacard/sdxc-remove-after-suspend +depends: mediacard/sdxc-insert-after-suspend +estimated_duration: 30.0 +command: removable_storage_watcher --memorycard remove sdio usb scsi +_description: + PURPOSE: + This test will check that the system correctly detects the removal + of a SDXC card from the systems card reader after the system has been suspended. + STEPS: + 1. Click "Test" and remove the SDXC card from the reader. + (Note: this test will time-out after 20 seconds.) + VERIFICATION: + The verification of this test is automated. Do not change the + automatically selected result. + +plugin: user-interact +category_id: com.canonical.plainbox::suspend +id: mediacard/ms-insert-after-suspend +estimated_duration: 30.0 +depends: suspend/suspend_advanced +imports: from com.canonical.plainbox import manifest +requires: + package.name == 'udisks2' or snap.name == 'udisks2' + manifest.has_card_reader == 'True' +command: removable_storage_watcher --memorycard insert sdio usb scsi +_description: + PURPOSE: + This test will check that the systems media card reader can + detect the insertion of a MS card after the system has been suspended + STEPS: + 1. Click "Test" and insert a MS card into the reader. + If a file browser opens up, you can safely close it. + (Note: this test will time-out after 20 seconds.) + 2. Do not remove the device after this test. + VERIFICATION: + The verification of this test is automated. Do not change the + automatically selected result. + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: mediacard/ms-storage-after-suspend +estimated_duration: 10.0 +depends: mediacard/ms-insert-after-suspend +user: root +command: removable_storage_test -s 268400000 --memorycard sdio usb scsi +_description: + This test is automated and executes after the mediacard/ms-insert-after-suspend test + is run. It tests reading and writing to the MS card after the system has been suspended. + +plugin: user-interact +category_id: com.canonical.plainbox::suspend +id: mediacard/ms-remove-after-suspend +estimated_duration: 30.0 +depends: mediacard/ms-insert-after-suspend +command: removable_storage_watcher --memorycard remove sdio usb scsi +_description: + PURPOSE: + This test will check that the system correctly detects the removal + of a MS card from the systems card reader after the system has been suspended. + STEPS: + 1. Click "Test" and remove the MS card from the reader. + (Note: this test will time-out after 20 seconds.) + VERIFICATION: + The verification of this test is automated. Do not change the + automatically selected result. + +plugin: user-interact +category_id: com.canonical.plainbox::suspend +id: mediacard/msp-insert-after-suspend +estimated_duration: 30.0 +depends: suspend/suspend_advanced +imports: from com.canonical.plainbox import manifest +requires: + package.name == 'udisks2' or snap.name == 'udisks2' + manifest.has_card_reader == 'True' +command: removable_storage_watcher --memorycard insert sdio usb scsi +_description: + PURPOSE: + This test will check that the systems media card reader can + detect the insertion of a MSP card after the system has been suspended + STEPS: + 1. Click "Test" and insert a MSP card into the reader. + If a file browser opens up, you can safely close it. + (Note: this test will time-out after 20 seconds.) + 2. Do not remove the device after this test. + VERIFICATION: + The verification of this test is automated. Do not change the + automatically selected result. + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: mediacard/msp-storage-after-suspend +estimated_duration: 10.0 +depends: mediacard/msp-insert-after-suspend +user: root +command: removable_storage_test -s 268400000 --memorycard sdio usb scsi +_description: + This test is automated and executes after the mediacard/msp-insert-after-suspend test + is run. It tests reading and writing to the MSP card after the system has been suspended. + +plugin: user-interact +category_id: com.canonical.plainbox::suspend +id: mediacard/msp-remove-after-suspend +estimated_duration: 30.0 +depends: mediacard/msp-insert-after-suspend +command: removable_storage_watcher --memorycard remove sdio usb scsi +_description: + PURPOSE: + This test will check that the system correctly detects the removal + of a MSP card from the systems card reader after the system has been suspended. + STEPS: + 1. Click "Test" and remove the MSP card from the reader. + (Note: this test will time-out after 20 seconds.) + VERIFICATION: + The verification of this test is automated. Do not change the + automatically selected result. + +plugin: user-interact +category_id: com.canonical.plainbox::suspend +id: mediacard/xd-insert-after-suspend +estimated_duration: 30.0 +depends: suspend/suspend_advanced +imports: from com.canonical.plainbox import manifest +requires: + package.name == 'udisks2' or snap.name == 'udisks2' + manifest.has_card_reader == 'True' +command: removable_storage_watcher --memorycard insert sdio usb scsi +_description: + PURPOSE: + This test will check that the systems media card reader can + detect the insertion of a xD card after the system has been suspended + STEPS: + 1. Click "Test" and insert a xD card into the reader. + If a file browser opens up, you can safely close it. + (Note: this test will time-out after 20 seconds.) + 2. Do not remove the device after this test. + VERIFICATION: + The verification of this test is automated. Do not change the + automatically selected result. + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: mediacard/xd-storage-after-suspend +estimated_duration: 10.0 +depends: mediacard/xd-insert-after-suspend +user: root +command: removable_storage_test -s 268400000 --memorycard sdio usb scsi +_description: + This test is automated and executes after the mediacard/xd-insert-after-suspend test + is run. It tests reading and writing to the xD card after the system has been suspended. + +plugin: user-interact +category_id: com.canonical.plainbox::suspend +id: mediacard/xd-remove-after-suspend +estimated_duration: 30.0 +depends: mediacard/xd-insert-after-suspend +command: removable_storage_watcher --memorycard remove sdio usb scsi +_description: + PURPOSE: + This test will check that the system correctly detects the removal + of a xD card from the systems card reader after the system has been suspended. + STEPS: + 1. Click "Test" and remove the xD card from the reader. + (Note: this test will time-out after 20 seconds.) + VERIFICATION: + The verification of this test is automated. Do not change the + automatically selected result. + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: touchpad/touchpad_after_suspend +depends: suspend/suspend_advanced_auto +requires: + dmi.product in ['Notebook','Laptop','Portable'] + xinput.device_class == 'XITouchClass' and xinput.touch_mode != 'dependent' +command: true +estimated_duration: 1.2 +_description: + Determine if the touchpad is still functioning after suspend/resume. + +plugin: manual +category_id: com.canonical.plainbox::suspend +id: touchscreen/touchscreen_after_suspend +imports: from com.canonical.plainbox import manifest +requires: manifest.has_touchscreen == 'True' +depends: suspend/suspend_advanced_auto +estimated_duration: 30.0 +_description: + PURPOSE: + Check touchscreen tap recognition + STEPS: + 1. Tap an object on the screen with finger. The cursor should jump to location tapped and object should highlight + VERIFICATION: + Does tap recognition work? + +plugin: user-interact-verify +category_id: com.canonical.plainbox::suspend +id: camera/display_after_suspend +estimated_duration: 120.0 +depends: suspend/suspend_advanced +requires: + device.category == 'CAPTURE' +command: camera_test display +_description: + PURPOSE: + This test will check that the built-in camera works after suspend + STEPS: + 1. Click on Test to display a video capture from the camera for ten seconds. + VERIFICATION: + Did you see the video capture? + +plugin: user-interact-verify +category_id: com.canonical.plainbox::suspend +id: camera/still_after_suspend +estimated_duration: 120.0 +depends: camera/display_after_suspend +requires: + package.name == 'gir1.2-gst-plugins-base-0.10' or package.name == 'gir1.2-gst-plugins-base-1.0' + package.name == 'eog' + package.name == 'fswebcam' or package.name == 'gir1.2-gst-plugins-base-0.10' or package.name == 'gir1.2-gst-plugins-base-1.0' + device.category == 'CAPTURE' +command: camera_test still +_description: + PURPOSE: + This test will check that the built-in camera works after suspend + STEPS: + 1. Click on Test to display a still image from the camera for ten seconds. + VERIFICATION: + Did you see the image? + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: camera/multiple_resolution_after_suspend +estimated_duration: 1.2 +depends: camera/still_after_suspend +requires: + package.name == 'fswebcam' or package.name == 'gir1.2-gst-plugins-base-0.10' or package.name == 'gir1.2-gst-plugins-base-1.0' + device.category == 'CAPTURE' +command: camera_test resolutions +_description: + Takes multiple pictures based on the resolutions supported by the camera and + validates their size and that they are of a valid format after suspend + +unit: template +template-resource: device +template-filter: device.category == 'MOUSE' or device.category == 'TOUCHPAD' or device.category == 'TOUCHSCREEN' +plugin: manual +category_id: com.canonical.plainbox::suspend +id: suspend/pointing-after-suspend_{product_slug}_{category}_{__index__} +depends: suspend/suspend_advanced +_description: + PURPOSE: + This will test your {product} device after suspend. + STEPS: + 1. Move the cursor with your {product}. + VERIFICATION: + Did the cursor move? + +unit: template +template-resource: device +template-filter: device.category == 'MOUSE' or device.category == 'TOUCHPAD' +plugin: manual +category_id: com.canonical.plainbox::suspend +estimated_duration: 30.0 +id: suspend/clicking-after-suspend_{product_slug}_{category}_{__index__} +depends: suspend/suspend_advanced +_summary: Check post suspend button functionality for {product} +_description: + PURPOSE: + This will test the buttons of your {product} device after suspend + STEPS: + 1. Click the left button with your {product}. + 2. Click the right button with your {product}. + 3. Click the middle button with your {product} (if available). + 4. Double-click the left button with your {product}. + VERIFICATION: + Did these buttons work as expected? + +unit: template +template-resource: graphics_card +template-filter: graphics_card.prime_gpu_offload == 'Off' +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/{index}_compiz_check_after_suspend_{product_slug} +depends: suspend/{index}_suspend_after_switch_to_card_{product_slug} +requires: package.name == 'nux-tools' +command: ! /usr/lib/nux/unity_support_test -c -p 2>&1 | ansi_parser | grep -e ":\(\s\+\)no$" -ie "error" +estimated_duration: 0.130 +_description: Check that {product} hardware is able to run compiz after suspend +_summary: Test Compiz support for {product} after suspend + +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/compiz_check_after_suspend +depends: suspend/suspend_advanced +requires: package.name == 'nux-tools' +command: ! /usr/lib/nux/unity_support_test -c -p 2>&1 | ansi_parser | grep -e ":\(\s\+\)no$" -ie "error" +estimated_duration: 0.130 +_description: Check that the hardware is able to run compiz after suspend +_summary: Test Compiz support after suspend + +unit: template +template-resource: graphics_card +plugin: user-interact-verify +category_id: com.canonical.plainbox::suspend +id: suspend/{index}_glxgears_after_suspend_{product_slug} +depends: suspend/{index}_suspend_after_switch_to_card_{product_slug} +requires: package.name == 'mesa-utils' +command: glxgears; true +_summary: Test that glxgears works for {vendor} {product} after suspend +_description: + PURPOSE: + This test tests the basic 3D capabilities of your {vendor} {product} video card after suspend + STEPS: + 1. Click "Test" to execute an OpenGL demo. Press ESC at any time to close. + 2. Verify that the animation is not jerky or slow. + VERIFICATION: + 1. Did the 3d animation appear? + 2. Was the animation free from slowness/jerkiness? + +plugin: user-interact-verify +category_id: com.canonical.plainbox::suspend +id: suspend/glxgears_after_suspend +depends: suspend/suspend_advanced +requires: + package.name == 'mesa-utils' + device.category == 'VIDEO' +command: glxgears; true +_summary: Test that glxgears works after suspend +_description: + PURPOSE: + This test tests the basic 3D capabilities of your video card after suspend + STEPS: + 1. Click "Test" to execute an OpenGL demo. Press ESC at any time to close. + 2. Verify that the animation is not jerky or slow. + VERIFICATION: + 1. Did the 3d animation appear? + 2. Was the animation free from slowness/jerkiness? + +unit: template +template-resource: graphics_card +id: suspend/{index}_video_after_suspend_{product_slug} +depends: suspend/{index}_suspend_after_switch_to_card_{product_slug} +plugin: user-interact-verify +category_id: com.canonical.plainbox::suspend +requires: + package.name == 'xorg' + package.name == 'gir1.2-gst-plugins-base-0.10' or package.name == 'gir1.2-gst-plugins-base-1.0' +command: gst_pipeline_test -t 2 'videotestsrc ! videoconvert ! autovideosink' || gst_pipeline_test -t 2 'videotestsrc ! ffmpegcolorspace ! autovideosink' +_summary: Test that video can be displayed after suspend with {vendor} {product} +_description: + PURPOSE: + This test will test the default display after suspend with a sample video + STEPS: + 1. Click "Test" to display a video test. + VERIFICATION: + Do you see color bars and static? + +id: suspend/video_after_suspend +plugin: user-interact-verify +category_id: com.canonical.plainbox::suspend +depends: suspend/suspend_advanced +requires: + device.category == 'VIDEO' + package.name == 'xorg' + package.name == 'gir1.2-gst-plugins-base-0.10' or package.name == 'gir1.2-gst-plugins-base-1.0' +command: gst_pipeline_test -t 2 'videotestsrc ! videoconvert ! autovideosink' || gst_pipeline_test -t 2 'videotestsrc ! ffmpegcolorspace ! autovideosink' +_summary: Test that video can be displayed after suspend +_description: + PURPOSE: + This test will test the default display after suspend with a sample video + STEPS: + 1. Click "Test" to display a video test. + VERIFICATION: + Do you see color bars and static? + +unit: template +template-resource: graphics_card +template-filter: graphics_card.prime_gpu_offload == 'Off' +plugin: shell +category_id: com.canonical.plainbox::suspend +id: suspend/{index}_driver_version_after_suspend_{product_slug} +depends: suspend/{index}_suspend_after_switch_to_card_{product_slug} +command: graphics_driver +estimated_duration: 0.500 +_description: Parses Xorg.0.Log and discovers the running X driver and version after suspend for the {vendor} {product} graphics card +_summary: Test X driver/version after suspend for {vendor} {product} + +id: suspend/driver_version_after_suspend +requires: device.category == 'VIDEO' +depends: suspend/suspend_advanced +plugin: shell +category_id: com.canonical.plainbox::suspend +command: graphics_driver +estimated_duration: 0.500 +_description: Parses Xorg.0.Log and discovers the running X driver and version after suspend +_summary: Test X driver/version after suspend + +id: suspend/oops_after_suspend +depends: suspend/suspend_advanced +plugin:shell +estimated_duration: 10.0 +requires: + package.name == 'fwts' +user: root +_description: + Run Firmware Test Suite (fwts) oops tests after suspend. +environ: PLAINBOX_SESSION_SHARE +command: + if [[ -v SNAP ]]; then + export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$SNAP/usr/lib/fwts" + fi + checkbox-support-fwts_test -l $PLAINBOX_SESSION_SHARE/fwts_oops_results_after_s3.log -t oops + +id: suspend/oops_results_after_suspend.log +plugin: attachment +category_id: com.canonical.plainbox::suspend +estimated_duration: 0.5 +command: + [ -e ${PLAINBOX_SESSION_SHARE}/fwts_oops_results_after_s3.log ] && xz -c ${PLAINBOX_SESSION_SHARE}/fwts_oops_results_after_s3.log | base64 +_description: Attaches the FWTS oops results log to the submission after suspend + diff --git a/units/suspend/test-plan.pxu b/units/suspend/test-plan.pxu new file mode 100644 index 0000000..57124f5 --- /dev/null +++ b/units/suspend/test-plan.pxu @@ -0,0 +1,86 @@ +id: before-suspend-reference-cert-full +unit: test plan +_name: Before suspend reference tests +_description: Before suspend reference tests +include: +nested_part: + before-suspend-reference-cert-automated + +id: after-suspend-reference-cert-full +unit: test plan +_name: After suspend reference tests +_description: After suspend reference tests +include: +nested_part: + after-suspend-reference-cert-automated + +id: suspend-key-led-oops-check-cert +unit: test plan +_name: After suspend LED and oops tests (using special sleep key) +_description: After suspend LED and oops tests (using special sleep key) +include: + keys/sleep certification-status=blocker + suspend/oops_after_suspend certification-status=blocker + suspend/oops_results_after_suspend.log + led/power-blink-suspend + led/suspend + +id: before-suspend-reference-cert-automated +unit: test plan +_name: Before suspend reference tests (automated) +_description: Before suspend reference tests (automated) +include: + suspend/network_before_suspend certification-status=blocker + suspend/audio_before_suspend certification-status=blocker + suspend/cpu_before_suspend certification-status=blocker + suspend/memory_before_suspend certification-status=blocker + suspend/bluetooth_obex_send_before_suspend + suspend/bluetooth_obex_browse_before_suspend + suspend/bluetooth_obex_get_before_suspend + +id: after-suspend-reference-cert-automated +unit: test plan +_name: After suspend reference tests (automated) +_description: After suspend reference tests (automated) +include: + suspend/network_after_suspend certification-status=blocker + suspend/audio_after_suspend certification-status=blocker + suspend/cpu_after_suspend certification-status=blocker + suspend/memory_after_suspend certification-status=blocker + suspend/bluetooth_detect_after_suspend certification-status=blocker + suspend/bluetooth_obex_send_after_suspend + suspend/bluetooth_obex_browse_after_suspend + suspend/bluetooth_obex_get_after_suspend + +id: before-suspend-reference-cert-blockers +unit: test plan +_name: Before suspend reference tests (certification blockers only) +_description: Before suspend reference tests (certification blockers only) +include: + suspend/network_before_suspend certification-status=blocker + suspend/audio_before_suspend certification-status=blocker + suspend/cpu_before_suspend certification-status=blocker + suspend/memory_before_suspend certification-status=blocker + +id: after-suspend-reference-cert-blockers +unit: test plan +_name: After suspend reference tests (certification blockers only) +_description: After suspend reference tests (certification blockers only) +include: + suspend/network_after_suspend certification-status=blocker + suspend/audio_after_suspend certification-status=blocker + suspend/cpu_after_suspend certification-status=blocker + suspend/memory_after_suspend certification-status=blocker + suspend/bluetooth_detect_after_suspend certification-status=blocker + +id: suspend-key-led-oops-check-cert-blockers +unit: test plan +_name: + After suspend LED and oops tests (using special sleep key, cert. blockers only) +_description: + After suspend LED and oops tests (using special sleep key) + (certification blockers only) +include: + keys/sleep certification-status=blocker + suspend/oops_after_suspend certification-status=blocker + suspend/oops_results_after_suspend.log diff --git a/units/thunderbolt/jobs.pxu b/units/thunderbolt/jobs.pxu new file mode 100644 index 0000000..188b2f6 --- /dev/null +++ b/units/thunderbolt/jobs.pxu @@ -0,0 +1,152 @@ +plugin: user-interact +category_id: com.canonical.plainbox::disk +id: thunderbolt/insert +imports: from com.canonical.plainbox import manifest +requires: manifest.has_thunderbolt == 'True' +estimated_duration: 20.0 +command: removable_storage_watcher insert --timeout 40 scsi +_summary: Storage insert detection on Thunderbolt +_description: + PURPOSE: + This test will check if the insertion of a Thunderbolt HDD could be detected + STEPS: + 1. Click 'Test' to begin the test. This test will + timeout and fail if the insertion has not been detected within 40 seconds. + 2. Plug a Thunderbolt HDD into an available Thunderbolt port, if it's not + mounted automatically, please click the HDD icon to mount it. + VERIFICATION: + The verification of this test is automated. Do not change the automatically + selected result + +plugin: shell +category_id: com.canonical.plainbox::disk +id: thunderbolt/storage-test +user: root +imports: from com.canonical.plainbox import manifest +requires: manifest.has_thunderbolt == 'True' +depends: thunderbolt/insert +estimated_duration: 45.0 +command: removable_storage_test -s 268400000 scsi +_summary: Storage test on Thunderbolt +_description: + This is an automated test which performs read/write operations on an attached + Thunderbolt HDD + +plugin: user-interact +category_id: com.canonical.plainbox::disk +id: thunderbolt/remove +imports: from com.canonical.plainbox import manifest +requires: manifest.has_thunderbolt == 'True' +depends: thunderbolt/insert +estimated_duration: 10.0 +command: removable_storage_watcher remove scsi +_summary: Storage removal detection on Thunderbolt +_description: + PURPOSE: + This test will check the system can detect the removal of a Thunderbolt HDD + STEPS: + 1. Click 'Test' to begin the test. This test will timeout and fail if + the removal has not been detected within 20 seconds. + 2. Remove the previously attached Thunderbolt HDD from the Thunderbolt port. + VERIFICATION: + The verification of this test is automated. Do not change the automatically + selected result + +plugin: user-interact-verify +category_id: com.canonical.plainbox::disk +id: thunderbolt/daisy-chain +imports: from com.canonical.plainbox import manifest +requires: manifest.has_thunderbolt == 'True' +estimated_duration: 45.0 +command: removable_storage_test -s 268400000 scsi +_summary: Daisy-chain testing for Thunderbolt storage and display device +_description: + PURPOSE: + This test will check if your system can support daisy-chaining of a storage + and a monitor over Thunderbolt port + STEPS: + 1. Connect your Thunderbolt monitor to your systems + 2. Connect and mount your Thunderbolt HDD to another Thunderbolt port of the + monitor (you can do this with HDD first as well) + 3. Click 'Test' to perform the storage test on the Thunderbolt HDD + VERIFICATION: + 1. The verification for storage is automated, please select the result combine + with the result for the display. + 2. Was the desktop displayed correctly on the Thunderbolt-connected screen? + +plugin: user-interact +category_id: com.canonical.plainbox::disk +id: thunderbolt3/insert +imports: from com.canonical.plainbox import manifest +requires: manifest.has_thunderbolt3 == 'True' +estimated_duration: 20.0 +command: removable_storage_watcher insert --timeout 40 scsi +_summary: Storage insert detection on Thunderbolt 3 port +_description: + PURPOSE: + This test will check if the insertion of a Thunderbolt HDD could be detected + STEPS: + 1. Click 'Test' to begin the test. This test will + timeout and fail if the insertion has not been detected within 40 seconds. + 2. Plug a Thunderbolt HDD into an available Thunderbolt 3 port, if it's not + mounted automatically, please click the HDD icon to mount it. + VERIFICATION: + The verification of this test is automated. Do not change the automatically + selected result + +plugin: shell +category_id: com.canonical.plainbox::disk +id: thunderbolt3/storage-test +user: root +imports: from com.canonical.plainbox import manifest +requires: manifest.has_thunderbolt3 == 'True' +depends: thunderbolt3/insert +estimated_duration: 45.0 +command: removable_storage_test -s 268400000 scsi +_summary: Storage test on Thunderbolt 3 +_description: + This is an automated test which performs read/write operations on an attached + Thunderbolt HDD + +plugin: user-interact +category_id: com.canonical.plainbox::disk +id: thunderbolt3/remove +imports: from com.canonical.plainbox import manifest +requires: manifest.has_thunderbolt3 == 'True' +depends: thunderbolt3/insert +estimated_duration: 10.0 +command: removable_storage_watcher remove scsi +_summary: Storage removal detection on Thunderbolt 3 port +_description: + PURPOSE: + This test will check the system can detect the removal of a Thunderbolt HDD + STEPS: + 1. Click 'Test' to begin the test. This test will timeout and fail if + the removal has not been detected within 20 seconds. + 2. Remove the previously attached Thunderbolt HDD from the Thunderbolt port. + VERIFICATION: + The verification of this test is automated. Do not change the automatically + selected result + +plugin: user-interact-verify +category_id: com.canonical.plainbox::disk +id: thunderbolt3/daisy-chain +imports: from com.canonical.plainbox import manifest +requires: manifest.has_thunderbolt3 == 'True' +estimated_duration: 45.0 +command: removable_storage_test -s 268400000 scsi +_summary: Daisy-chain testing for Thunderbolt 3 storage and display device +_description: + PURPOSE: + This test will check if your system can support daisy-chaining of a storage + and a monitor over Thunderbolt 3 port + STEPS: + 1. Connect your Thunderbolt monitor to your systems + 2. Connect and mount your Thunderbolt HDD to another Thunderbolt 3 port of the + monitor (you can do this with HDD first as well) + 3. Click 'Test' to perform the storage test on the Thunderbolt HDD + VERIFICATION: + 1. The verification for storage is automated, please select the result combine + with the result for the display. + 2. Was the desktop displayed correctly on the Thunderbolt-connected screen? + diff --git a/units/thunderbolt/manifest.pxu b/units/thunderbolt/manifest.pxu new file mode 100644 index 0000000..f8926f9 --- /dev/null +++ b/units/thunderbolt/manifest.pxu @@ -0,0 +1,9 @@ +unit: manifest entry +id: has_thunderbolt +_name: Thunderbolt Support +value-type: bool + +unit: manifest entry +id: has_thunderbolt3 +_name: Thunderbolt 3 Support +value-type: bool diff --git a/units/thunderbolt/test-plan.pxu b/units/thunderbolt/test-plan.pxu new file mode 100644 index 0000000..054bb26 --- /dev/null +++ b/units/thunderbolt/test-plan.pxu @@ -0,0 +1,20 @@ +id: thunderbolt-cert-full +unit: test plan +_name: Thunderbolt tests +_description: Thunderbolt tests +include: + thunderbolt/insert certification-status=blocker + thunderbolt/storage-test certification-status=blocker + thunderbolt/remove certification-status=blocker + thunderbolt3/insert certification-status=non-blocker + thunderbolt3/storage-test certification-status=non-blocker + thunderbolt3/remove certification-status=non-blocker + +id: thunderbolt-cert-blockers +unit: test plan +_name: Thunderbolt tests (certification blockers only) +_description: Thunderbolt tests (certification blockers only) +include: + thunderbolt/insert certification-status=blocker + thunderbolt/storage-test certification-status=blocker + thunderbolt/remove certification-status=blocker diff --git a/units/touchpad/jobs.pxu b/units/touchpad/jobs.pxu new file mode 100644 index 0000000..ed23ad2 --- /dev/null +++ b/units/touchpad/jobs.pxu @@ -0,0 +1,272 @@ +plugin: manual +category_id: com.canonical.plainbox::touchpad +id: touchpad/basic +requires: dmi.product in ['Notebook','Laptop','Portable','Convertible'] +estimated_duration: 120.0 +_purpose: + Touchpad manual verification +_steps: + 1. Make sure that touchpad is enabled. + 2. Move cursor using the touchpad. +_verification: + Did the cursor move? +_siblings: + [{ "id": "touchpad/basic-after-suspend", + "depends": "suspend/suspend_advanced touchpad/basic" }] + +plugin: user-interact +category_id: com.canonical.plainbox::touchpad +id: touchpad/horizontal +requires: + dmi.product in ['Notebook','Laptop','Portable','Convertible'] + 'Button Horiz Wheel Left' in xinput.button_labels and 'Button Horiz Wheel Right' in xinput.button_labels +command: touchpad_test right left --edge-scroll +estimated_duration: 120.0 +_purpose: + Touchpad horizontal scroll verification +_steps: + 1. Select "Test" when ready and place your cursor within the borders of the displayed test window. + 2. Verify that you can move the horizontal slider by moving your finger right and left in the lower part of the touchpad. +_verification: + Could you scroll right and left? +_siblings: + [{ "id": "touchpad/horizontal-after-suspend", + "depends": "suspend/suspend_advanced touchpad/horizontal" }] + +plugin: user-interact +category_id: com.canonical.plainbox::touchpad +id: touchpad/vertical +requires: + dmi.product in ['Notebook','Laptop','Portable','Convertible'] + 'Button Wheel Up' in xinput.button_labels and 'Button Wheel Down' in xinput.button_labels +command: touchpad_test up down --edge-scroll +estimated_duration: 120.0 +_purpose: + Touchpad vertical scroll verification +_steps: + 1. Select "Test" when ready and place your cursor within the borders of the displayed test window. + 2. Verify that you can move the vertical slider by moving your finger up and down in the right part of the touchpad. +_verification: + Could you scroll up and down? +_siblings: + [{ "id": "touchpad/vertical-after-suspend", + "depends": "suspend/suspend_advanced touchpad/vertical" }] + +plugin: shell +category_id: com.canonical.plainbox::touchpad +id: touchpad/singletouch-automated +requires: + dmi.product in ['Notebook','Laptop','Portable','Convertible'] + xinput.device_class == 'XITouchClass' and xinput.touch_mode != 'dependent' +command: true +estimated_duration: 1.2 +_purpose: + Determine whether the touchpad is detected as a singletouch device automatically. + +plugin: shell +category_id: com.canonical.plainbox::touchpad +id: touchpad/multitouch-automated +estimated_duration: 1.2 +requires: + dmi.product in ['Notebook','Laptop','Portable','Convertible'] + xinput.device_class == 'XITouchClass' and xinput.touch_mode == 'dependent' +command: true +_purpose: + Determine whether the touchpad is detected as a multitouch device automatically. + +plugin: manual +category_id: com.canonical.plainbox::touchpad +id: touchpad/multitouch-manual +requires: dmi.product in ['Notebook','Laptop','Portable','Convertible'] +depends: touchpad/singletouch-automated +estimated_duration: 120.0 +_purpose: + Touchpad manual detection of multitouch. +_steps: + 1. Look at the specifications for your system. +_verification: + Is the touchpad supposed to be multitouch? +_siblings: + [{ "id": "touchpad/multitouch-manual-after-suspend", + "depends": "suspend/suspend_advanced touchpad/multitouch-manual" }] + +plugin: manual +category_id: com.canonical.plainbox::touchpad +id: touchpad/singletouch-corner +requires: dmi.product in ['Notebook','Laptop','Portable','Convertible'] +estimated_duration: 120.0 +_purpose: + Determine that the hot corner feature is working as expected +_steps: + 1. Launch a browser. + 2. Go to a website, and hover the cursor over a link. + 3. Tap the upper right corner on the touchpad. +_verification: + Did the tap trigger a new tab to be opened? + +plugin: manual +category_id: com.canonical.plainbox::touchpad +id: touchpad/singletouch-selection +requires: dmi.product in ['Notebook','Laptop','Portable','Convertible'] +estimated_duration: 120.0 +_purpose: + Determine that the selection window function is working as expected. +_steps: + 1. Open a file folder + 2. Double tap and drag the cursor across several file. +_verification: + Did a selection window open and were several files selected? +_siblings: + [{ "id": "touchpad/singletouch-selection-after-suspend", + "depends": "suspend/suspend_advanced touchpad/singletouch-selection" }] + +plugin: manual +category_id: com.canonical.plainbox::touchpad +id: touchpad/multitouch-rightclick +requires: dmi.product in ['Notebook','Laptop','Portable','Convertible'] +depends: touchpad/multitouch-automated +estimated_duration: 120.0 +_purpose: + Determine that the right click function is working as expected. +_steps: + 1. Open a file folder + 2. Hover cursor over file in folder + 3. 2-touch tap. +_verification: + Did the right click pop up menu appear? +_siblings: + [{ "id": "touchpad/multitouch-rightclick-after-suspend", + "depends": "suspend/suspend_advanced touchpad/multitouch-rightclick" }] + +plugin: user-interact +category_id: com.canonical.plainbox::touchpad +id: touchpad/multitouch-horizontal +requires: dmi.product in ['Notebook','Laptop','Portable','Convertible'] +command: touchpad_test right left +estimated_duration: 120.0 +_purpose: + Touchpad 2-touch horizontal scroll verification +_steps: + 1. Select "Test" when ready and place your cursor within the borders of the displayed test window. + 2. Verify that you can move the horizontal slider by moving 2 fingers right and left along the touchpad. +_verification: + Could you scroll right and left? +_siblings: + [{ "id": "touchpad/multitouch-horizontal-after-suspend", + "depends": "suspend/suspend_advanced touchpad/multitouch-horizontal" }] + +plugin: user-interact +category_id: com.canonical.plainbox::touchpad +id: touchpad/multitouch-vertical +requires: dmi.product in ['Notebook','Laptop','Portable','Convertible'] +command: touchpad_test up down +estimated_duration: 120.0 +_purpose: + Touchpad 2-touch vertical scroll verification +_steps: + 1. Select "Test" when ready and place your cursor within the borders of the displayed test window. + 2. Verify that you can move the vertical slider by moving 2 fingers up and down along the touchpad. +_verification: + Could you scroll up and down? +_siblings: + [{ "id": "touchpad/multitouch-vertical-after-suspend", + "depends": "suspend/suspend_advanced touchpad/multitouch-vertical" }] + +plugin: manual +category_id: com.canonical.plainbox::touchpad +id: touchpad/drag-and-drop +requires: dmi.product in ['Notebook','Laptop','Portable','Convertible'] +estimated_duration: 120.0 +_purpose: + Determine that the drag and drop function is working as expected. +_steps: + 1. Browse to the examples folder in the current user's home directory + 2. Double tap and hold to select the "Ubuntu_Free_Culture_Showcase" folder + 3. Drag the selected folder to the desktop and remove finger from touchpad. +_verification: + Did a selected folder move to the desktop? +_siblings: + [{ "id": "touchpad/drag-and-drop-after-suspend", + "depends": "suspend/suspend_advanced touchpad/drag-and-drop" }] + +plugin: manual +category_id: com.canonical.plainbox::touchpad +id: touchpad/multitouch-zoom +requires: dmi.product in ['Notebook','Laptop','Portable','Convertible'] +depends: touchpad/multitouch-automated +estimated_duration: 120.0 +_summary: Check touchpad pinch-to-zoom gesture +_purpose: + Check touchpad pinch gesture for zoom +_steps: + 1. Open gallery-app with an image + 2. Place two fingers on the touchpad and pinch them together + 3. Place two fingers on the touchpad and move them apart +_verification: + Does the image zoom in and out? + +plugin: manual +category_id: com.canonical.plainbox::touchpad +id: touchpad/multitouch-dash +requires: dmi.product in ['Notebook','Laptop','Portable','Convertible'] +depends: touchpad/multitouch-automated +estimated_duration: 120.0 +_summary: Check 4-finger tap gesture +_purpose: + Validate that 4-touch tap is operating as expected +_steps: + 1. 4-touch tap (tap with 4 fingers) anywhere on the touchpad +_verification: + Did the tap open the Dash? +_siblings: + [{ "id": "touchpad/multitouch-dash-after-suspend", + "depends": "suspend/suspend_advanced touchpad/multitouch-dash", + "_summary": "Check 4-finger tap gesture after suspend (S3)" }] + +plugin: shell +category_id: com.canonical.plainbox::touchpad +id: touchpad/detected-as-mouse +requires: dmi.product in ['Notebook','Laptop','Portable','Convertible'] +estimated_duration: 1.2 +command: + info=`touchpad_driver_info` + if [ $? -eq 0 ]; then + echo -e "touchpad detected with:\n$info" + else + echo "$info" + exit 1 + fi +_purpose: + This test will check if your touchpad was detected as a mouse. +_siblings: + [{ "id": "touchpad/detected-as-mouse-after-suspend", + "depends": "suspend/suspend_advanced touchpad/detected-as-mouse" }] + +plugin: user-interact +category_id: com.canonical.plainbox::touchpad +id: touchpad/continuous-move +requires: dmi.product in ['Notebook','Laptop','Portable','Convertible'] +estimated_duration: 12 +command: qmlscene -qt5 --fullscreen $PLAINBOX_PROVIDER_DATA/touch_continuous_move_test.qml 2>&1 | grep -o PASS +_purpose: + Touchpad continuous move verification +_steps: + 1. Select "Test" when ready and continuously move your cursor within the borders of the displayed test window. + You'll need to keep moving your finger on the touchpad for 10 seconds. +_verification: + Did the mouse cursor move without interruption? +_siblings: + [{ "id": "touchpad/continuous-move-after-suspend", + "depends": "suspend/suspend_advanced touchpad/continuous-move" }] + +plugin: qml +category_id: com.canonical.plainbox::touchpad +id: touchpad/palm-rejection +requires: dmi.product in ['Notebook','Laptop','Portable','Convertible'] +estimated_duration: 20 +_purpose: + This test checks if touchpad ignores palm touches +qml_file: palm_rejection.qml +_siblings: + [{ "id": "touchpad/palm-rejection-after-suspend", + "depends": "suspend/suspend_advanced touchpad/palm-rejection" }] diff --git a/units/touchpad/test-plan.pxu b/units/touchpad/test-plan.pxu new file mode 100644 index 0000000..7514991 --- /dev/null +++ b/units/touchpad/test-plan.pxu @@ -0,0 +1,79 @@ +id: touchpad-cert-full +unit: test plan +_name: Touchpad tests +_description: Touchpad tests +include: + touchpad/basic certification-status=blocker + touchpad/detected-as-mouse certification-status=blocker + touchpad/palm-rejection certification-status=non-blocker + touchpad/continuous-move certification-status=blocker + touchpad/horizontal certification-status=blocker + touchpad/vertical certification-status=blocker + touchpad/singletouch-automated certification-status=blocker + touchpad/singletouch-selection certification-status=blocker + touchpad/drag-and-drop certification-status=blocker + touchpad/multitouch-automated certification-status=blocker + touchpad/multitouch-manual certification-status=blocker + touchpad/multitouch-rightclick certification-status=blocker + touchpad/multitouch-horizontal certification-status=blocker + touchpad/multitouch-vertical certification-status=blocker + touchpad/multitouch-dash certification-status=non-blocker + +id: after-suspend-touchpad-cert-full +unit: test plan +_name: Touchpad tests +_description: Touchpad tests +include: + touchpad/basic-after-suspend certification-status=blocker + touchpad/detected-as-mouse-after-suspend certification-status=blocker + touchpad/palm-rejection-after-suspend certification-status=non-blocker + touchpad/continuous-move-after-suspend certification-status=blocker + touchpad/horizontal-after-suspend certification-status=blocker + touchpad/vertical-after-suspend certification-status=blocker + touchpad/singletouch-automated-after-suspend certification-status=blocker + touchpad/singletouch-selection-after-suspend certification-status=blocker + touchpad/drag-and-drop-after-suspend certification-status=blocker + touchpad/multitouch-automated-after-suspend certification-status=blocker + touchpad/multitouch-manual-after-suspend certification-status=blocker + touchpad/multitouch-rightclick-after-suspend certification-status=blocker + touchpad/multitouch-horizontal-after-suspend certification-status=blocker + touchpad/multitouch-vertical-after-suspend certification-status=blocker + touchpad/multitouch-dash-after-suspend certification-status=non-blocker + +id: touchpad-cert-blockers +unit: test plan +_name: Touchpad tests (certification blockers only) +_description: Touchpad tests (certification blockers only) +include: + touchpad/basic certification-status=blocker + touchpad/detected-as-mouse certification-status=blocker + touchpad/continuous-move certification-status=blocker + touchpad/horizontal certification-status=blocker + touchpad/vertical certification-status=blocker + touchpad/singletouch-automated certification-status=blocker + touchpad/singletouch-selection certification-status=blocker + touchpad/drag-and-drop certification-status=blocker + touchpad/multitouch-automated certification-status=blocker + touchpad/multitouch-manual certification-status=blocker + touchpad/multitouch-rightclick certification-status=blocker + touchpad/multitouch-horizontal certification-status=blocker + touchpad/multitouch-vertical certification-status=blocker + +id: after-suspend-touchpad-cert-blockers +unit: test plan +_name: Touchpad tests (after suspend, certification blockers only) +_description: Touchpad tests (after suspend, certification blockers only) +include: + touchpad/basic-after-suspend certification-status=blocker + touchpad/detected-as-mouse-after-suspend certification-status=blocker + touchpad/continuous-move-after-suspend certification-status=blocker + touchpad/horizontal-after-suspend certification-status=blocker + touchpad/vertical-after-suspend certification-status=blocker + touchpad/singletouch-automated-after-suspend certification-status=blocker + touchpad/singletouch-selection-after-suspend certification-status=blocker + touchpad/drag-and-drop-after-suspend certification-status=blocker + touchpad/multitouch-automated-after-suspend certification-status=blocker + touchpad/multitouch-manual-after-suspend certification-status=blocker + touchpad/multitouch-rightclick-after-suspend certification-status=blocker + touchpad/multitouch-horizontal-after-suspend certification-status=blocker + touchpad/multitouch-vertical-after-suspend certification-status=blocker diff --git a/units/touchscreen/jobs.pxu b/units/touchscreen/jobs.pxu new file mode 100644 index 0000000..51b4a4e --- /dev/null +++ b/units/touchscreen/jobs.pxu @@ -0,0 +1,170 @@ +plugin: shell +category_id: com.canonical.plainbox::touchscreen +id: touchscreen/nontouch-automated +requires: + xinput.device_class == 'XITouchClass' and xinput.touch_mode != 'direct' +command: true +estimated_duration: 1.2 +_description: + Determine whether the screen is detected as a non-touch device automatically. + +plugin: shell +category_id: com.canonical.plainbox::touchscreen +id: touchscreen/multitouch-automated +requires: + xinput.device_class == 'XITouchClass' and xinput.touch_mode == 'direct' +command: true +estimated_duration: 1.2 +_description: + Determine whether the screen is detected as a multitouch device automatically. + +plugin: manual +category_id: com.canonical.plainbox::touchscreen +id: touchscreen/multitouch-manual +depends: touchscreen/nontouch-automated +estimated_duration: 120.0 +_description: + PURPOSE: + Touchscreen capability manual detection. + STEPS: + 1. Look at the specifications for your system. + VERIFICATION: + Your screen was detected as a non touch screen. Select PASS if this is correct. + +plugin: manual +category_id: com.canonical.plainbox::touchscreen +id: touchscreen/tap-detect +imports: from com.canonical.plainbox import manifest +requires: manifest.has_touchscreen == 'True' +estimated_duration: 120.0 +_description: + PURPOSE: + Check touchscreen tap recognition + STEPS: + 1. Tap an object on the screen with finger. The cursor should jump to location tapped and object should highlight + VERIFICATION: + Does tap recognition work? + +plugin: manual +category_id: com.canonical.plainbox::touchscreen +id: touchscreen/drag-n-drop +imports: from com.canonical.plainbox import manifest +requires: manifest.has_touchscreen == 'True' +estimated_duration: 120.0 +_description: + PURPOSE: + Check touchscreen drag & drop + STEPS: + 1. Tap and hold an object on the desktop + 2. Drag and drop the object in a different location + VERIFICATION: + Does drag and drop work? + +plugin: user-interact-verify +category_id: com.canonical.plainbox::touchscreen +id: touchscreen/multitouch-zoom +_summary: Check touchscreen pinch gesture for zoom +imports: from com.canonical.plainbox import manifest +requires: manifest.has_touchscreen == 'True' +command: qmlscene -qt5 $PLAINBOX_PROVIDER_DATA/touch_zoom_test.qml 2>&1 |grep -o PASS +estimated_duration: 120.0 +_description: + PURPOSE: + Check touchscreen pinch gesture for zoom + STEPS: + 1. Commence the test + 2. Using 2 fingers, resize the blue square until it turns green, then release it. + VERIFICATION: + Did the blue square change size following the gesture? + +plugin: user-interact-verify +category_id: com.canonical.plainbox::touchscreen +id: touchscreen/multitouch-rotate +_summary: Check touchscreen pinch gesture for rotate +imports: from com.canonical.plainbox import manifest +requires: manifest.has_touchscreen == 'True' +command: qmlscene -qt5 $PLAINBOX_PROVIDER_DATA/touch_rotate_test.qml 2>&1 |grep -o PASS +estimated_duration: 120.0 +_description: + PURPOSE: + Check touchscreen pinch gesture for rotate + STEPS: + 1. Commence the test + 2. Using 2 fingers, rotate the blue square until it turns green, then release it. + VERIFICATION: + Did the blue square rotate following the gesture? + +plugin: manual +category_id: com.canonical.plainbox::touchscreen +id: touchscreen/multitouch-window-move +imports: from com.canonical.plainbox import manifest +requires: manifest.has_touchscreen == 'True' +estimated_duration: 120.0 +_description: + PURPOSE: + Validate that 3-touch drag is operating as expected + STEPS: + 1. Open a windows and bring it to the foreground + 2. 3-touch the window and drag + VERIFICATION: + Did the window move along with the drag? + +plugin: manual +category_id: com.canonical.plainbox::touchscreen +id: touchscreen/multitouch-dash +imports: from com.canonical.plainbox import manifest +requires: manifest.has_touchscreen == 'True' +estimated_duration: 120.0 +_description: + PURPOSE: + Validate that 4-touch tap is operating as expected + STEPS: + 1. 4-touch tap anywhere on the touchscreen + VERIFICATION: + Did the tap open the Dash? + +plugin: user-interact-verify +category_id: com.canonical.plainbox::touchscreen +id: touchscreen/3-touch-tap +imports: from com.canonical.plainbox import manifest +requires: manifest.has_touchscreen == 'True' +estimated_duration: 15.0 +_description: + PURPOSE: + Validate that 3-touch tap is operating as expected + STEPS: + 1. Commence the test + 2. Tap the screen within the test area with 3 fingers simultaneously. + 3. Once 3 fingers are on the screen you should see the indicator they are recognized. + VERIFICATION: + Did you see the green circles around the three fingers? +command: + manage_compiz_plugin unityshell disable + qmlscene -qt5 $PLAINBOX_PROVIDER_DATA/touch_tap_test.qml --touchpoints=3 2>&1 | grep -o PASS + EXIT=$? + sleep 5 + manage_compiz_plugin unityshell enable + exit $EXIT + +plugin: user-interact-verify +category_id: com.canonical.plainbox::touchscreen +id: touchscreen/4-touch-tap +imports: from com.canonical.plainbox import manifest +requires: manifest.has_touchscreen == 'True' +estimated_duration: 15.0 +_description: + PURPOSE: + Validate that 4-touch tap is operating as expected + STEPS: + 1. Commence the test + 2. Tap the screen within the test area with 4 fingers simultaneously. + 3. Once 4 fingers are on the screen you should see the indicator they are recognized. + VERIFICATION: + Did you see the green circles around the four fingers? +command: + manage_compiz_plugin unityshell disable + qmlscene -qt5 $PLAINBOX_PROVIDER_DATA/touch_tap_test.qml --touchpoints=4 2>&1 | grep -o PASS + EXIT=$? + sleep 5 + manage_compiz_plugin unityshell enable + exit $EXIT diff --git a/units/touchscreen/manifest.pxu b/units/touchscreen/manifest.pxu new file mode 100644 index 0000000..733f01e --- /dev/null +++ b/units/touchscreen/manifest.pxu @@ -0,0 +1,4 @@ +unit: manifest entry +id: has_touchscreen +_name: Touchscreen +value-type: bool diff --git a/units/touchscreen/test-plan.pxu b/units/touchscreen/test-plan.pxu new file mode 100644 index 0000000..e984af9 --- /dev/null +++ b/units/touchscreen/test-plan.pxu @@ -0,0 +1,21 @@ +id: touchscreen-cert-full +unit: test plan +_name: Touchscreen tests +_description: Touchscreen tests +include: + touchscreen/drag-n-drop certification-status=blocker + touchscreen/multitouch-zoom certification-status=blocker + touchscreen/multitouch-rotate + touchscreen/3-touch-tap certification-status=blocker + touchscreen/4-touch-tap certification-status=blocker + touchscreen/multitouch-dash certification-status=non-blocker + +id: touchscreen-cert-blockers +unit: test plan +_name: Touchscreen tests (certification blockers only) +_description: Touchscreen tests (certification blockers only) +include: + touchscreen/drag-n-drop certification-status=blocker + touchscreen/multitouch-zoom certification-status=blocker + touchscreen/3-touch-tap certification-status=blocker + touchscreen/4-touch-tap certification-status=blocker diff --git a/units/usb/manifest.pxu b/units/usb/manifest.pxu new file mode 100644 index 0000000..914e892 --- /dev/null +++ b/units/usb/manifest.pxu @@ -0,0 +1,4 @@ +unit: manifest entry +id: has_usb_type_c +_name: USB Type-C Support +value-type: bool diff --git a/units/usb/test-plan.pxu b/units/usb/test-plan.pxu new file mode 100644 index 0000000..96ece6d --- /dev/null +++ b/units/usb/test-plan.pxu @@ -0,0 +1,89 @@ +id: usb-cert-full +unit: test plan +_name: USB tests +_description: USB tests +include: + usb/detect certification-status=blocker + usb/HID certification-status=blocker + usb/insert certification-status=blocker + usb/storage-automated certification-status=blocker + usb/remove certification-status=blocker + +id: usb3-cert-full +unit: test plan +_name: USB3 tests +_description: USB3 tests +include: + usb3/insert certification-status=blocker + usb3/storage-automated certification-status=blocker + usb3/remove certification-status=blocker + +id: usb-c-cert-full +unit: test plan +_name: USB Type-C tests +_description: USB Type-C tests +include: + usb-c/c-to-a-adapter/hid certification-status=non-blocker + usb-c/c-to-a-adapter/insert certification-status=non-blocker + usb-c/c-to-a-adapter/storage-automated certification-status=non-blocker + usb-c/c-to-a-adapter/remove certification-status=non-blocker + usb-c/hid certification-status=non-blocker + usb-c/insert certification-status=non-blocker + usb-c/storage-automated certification-status=non-blocker + usb-c/remove certification-status=non-blocker + +id: after-suspend-usb-cert-full +unit: test plan +_name: USB tests (after suspend) +_description: USB tests (after suspend) +include: + suspend/usb_insert_after_suspend certification-status=blocker + suspend/usb_storage_automated_after_suspend certification-status=blocker + suspend/usb_remove_after_suspend certification-status=blocker + +id: after-suspend-usb3-cert-full +unit: test plan +_name: USB3 tests (after suspend) +_description: USB3 tests (after suspend) +include: + suspend/usb3_insert_after_suspend certification-status=blocker + suspend/usb3_storage_automated_after_suspend certification-status=blocker + suspend/usb3_remove_after_suspend certification-status=blocker + +id: usb-cert-blockers +unit: test plan +_name: USB tests (certification blockers only) +_description: USB tests (certification blockers only) +include: + usb/detect certification-status=blocker + usb/HID certification-status=blocker + usb/insert certification-status=blocker + usb/storage-automated certification-status=blocker + usb/remove certification-status=blocker + +id: usb3-cert-blockers +unit: test plan +_name: USB3 tests (certification blockers only) +_description: USB3 tests (certification blockers only) +include: + usb3/insert certification-status=blocker + usb3/storage-automated certification-status=blocker + usb3/remove certification-status=blocker + +id: after-suspend-usb-cert-blockers +unit: test plan +_name: USB tests (after suspend, certification blockers only) +_description: USB tests (after suspend, certification blockers only) +include: + suspend/usb_insert_after_suspend certification-status=blocker + suspend/usb_storage_automated_after_suspend certification-status=blocker + suspend/usb_remove_after_suspend certification-status=blocker + +id: after-suspend-usb3-cert-blockers +unit: test plan +_name: USB3 tests (after suspend, certification blockers only) +_description: USB3 tests (after suspend, certification blockers only) +include: + suspend/usb3_insert_after_suspend certification-status=blocker + suspend/usb3_storage_automated_after_suspend certification-status=blocker + suspend/usb3_remove_after_suspend certification-status=blocker diff --git a/units/usb/usb-c.pxu b/units/usb/usb-c.pxu new file mode 100644 index 0000000..9a77664 --- /dev/null +++ b/units/usb/usb-c.pxu @@ -0,0 +1,189 @@ +id: usb-c/c-to-a-adapter/hid +_summary: USB HID work on USB Type-C port using a "USB Type-C to Type-A" adapter +_purpose: + This test will check that you can use a USB HID device plugged in a USB + Type-C port using a "USB Type-C to Type-A" adapter +_steps: + 1. Enable either a USB mouse or keyboard by plugging it in the USB Type-C port using a "USB Type-C to Type-A" adapter + 2. For mice, perform actions such as moving the pointer, right and left button clicks and double clicks + 3. For keyboards, switch to another tty and type some text +_verification: + Did the device work as expected? +plugin: manual +flags: also-after-suspend +category_id: com.canonical.plainbox::usb +imports: from com.canonical.plainbox import manifest +requires: manifest.has_usb_type_c == 'True' +estimated_duration: 60 + +id: usb-c/c-to-a-adapter/insert +_purpose: + This test will check that the system correctly detects the insertion of + a USB 3 storage device in a USB Type-C connector using a "Type-C to Type-A" + adapter +_steps: + 1. Commence the test + 2. Connect a USB 3 storage device to a USB Type-C port using a + "Type-C to Type-A" adapter + 3. Do not unplug the device after the test. +_verification: + The verification of this test is automated. Do not change the + automatically selected result. +plugin: user-interact +flags: also-after-suspend +user: root +command: + if [[ -v SNAP ]]; then + checkbox-support-run_watcher insertion usb3 + else + removable_storage_watcher -m 500000000 insert usb + fi +category_id: com.canonical.plainbox::usb +imports: from com.canonical.plainbox import manifest +requires: + usb.usb3 == 'supported' + manifest.has_usb_type_c == 'True' +estimated_duration: 30 + +id: usb-c/c-to-a-adapter/storage-automated +_description: + This test is automated and executes after the usb-c/c-to-a-adapter/insert test + is run. +plugin: shell +flags: also-after-suspend +user: root +command: + if [[ -v SNAP ]]; then + checkbox-support-usb_read_write + else + removable_storage_test -s 268400000 -m 500000000 usb --driver xhci_hcd + fi +category_id: com.canonical.plainbox::usb +imports: from com.canonical.plainbox import manifest +requires: + manifest.has_usb_type_c == 'True' +depends: usb-c/c-to-a-adapter/insert +estimated_duration: 60 + +id: usb-c/c-to-a-adapter/remove +_purpose: + This test will check that the system correctly detects the removal of + a USB 3 storage device connected to a USB Type-C port using a "USB Type-C + to Type-A" adapter. +_steps: + 1. Commence the test + 2. Disconnect a USB 3 storage device to a USB Type-C port using a + "Type-C to Type-A" adapter +_verification: + The verification of this test is automated. Do not change the + automatically selected result. +plugin: user-interact +flags: also-after-suspend +user: root +command: + if [[ -v SNAP ]]; then + checkbox-support-run_watcher removal usb3 + else + removable_storage_watcher -m 500000000 remove usb + fi +category_id: com.canonical.plainbox::usb +depends: usb-c/c-to-a-adapter/insert +imports: from com.canonical.plainbox import manifest +requires: + manifest.has_usb_type_c == 'True' +estimated_duration: 30 + +id: usb-c/hid +_summary: USB HID work on USB Type-C port +_purpose: + This test will check that you can use a USB HID device plugged in a USB + Type-C port +_steps: + 1. Enable either a USB mouse or keyboard by plugging it in the USB Type-C port + 2. For mice, perform actions such as moving the pointer, right and left button clicks and double clicks + 3. For keyboards, switch to another tty and type some text +_verification: + Did the device work as expected? +plugin: manual +flags: also-after-suspend +category_id: com.canonical.plainbox::usb +imports: from com.canonical.plainbox import manifest +requires: manifest.has_usb_type_c == 'True' +estimated_duration: 60 + +id: usb-c/insert +_summary: USB 3.0 storage device insertion detected on USB Type-C port +_purpose: + This test will check that the system correctly detects the insertion of + a USB 3 storage device in a USB Type-C connector +_steps: + 1. Commence the test + 2. Connect a USB 3 storage device to a USB Type-C port + 3. Do not unplug the device after the test. +_verification: + The verification of this test is automated. Do not change the + automatically selected result. +plugin: user-interact +flags: also-after-suspend +user: root +command: + if [[ -v SNAP ]]; then + checkbox-support-run_watcher insertion usb3 + else + removable_storage_watcher -m 500000000 insert usb + fi +category_id: com.canonical.plainbox::usb +imports: from com.canonical.plainbox import manifest +requires: + usb.usb3 == 'supported' + manifest.has_usb_type_c == 'True' +estimated_duration: 30 + +id: usb-c/storage-automated +_summary: USB 3.0 storage device read & write works on USB Type-C port +_description: + This test is automated and executes after the usb-c/insert test + is run. +plugin: shell +flags: also-after-suspend +user: root +command: + if [[ -v SNAP ]]; then + checkbox-support-usb_read_write + else + removable_storage_test -s 268400000 -m 500000000 usb --driver xhci_hcd + fi +category_id: com.canonical.plainbox::usb +imports: from com.canonical.plainbox import manifest +requires: + manifest.has_usb_type_c == 'True' +depends: usb-c/insert +estimated_duration: 60 + +id: usb-c/remove +_summary: USB 3.0 storage device removal detected on USB Type-C port +_purpose: + This test will check that the system correctly detects the removal of + a USB 3 storage device connected to a USB Type-C port. +_steps: + 1. Commence the test + 2. Disconnect a USB 3 storage device to a USB Type-C port +_verification: + The verification of this test is automated. Do not change the + automatically selected result. +plugin: user-interact +flags: also-after-suspend +user: root +command: + if [[ -v SNAP ]]; then + checkbox-support-run_watcher removal usb3 + else + removable_storage_watcher -m 500000000 remove usb + fi +category_id: com.canonical.plainbox::usb +depends: usb-c/insert +imports: from com.canonical.plainbox import manifest +requires: + manifest.has_usb_type_c == 'True' +estimated_duration: 30 + diff --git a/units/usb/usb.pxu b/units/usb/usb.pxu new file mode 100644 index 0000000..6615401 --- /dev/null +++ b/units/usb/usb.pxu @@ -0,0 +1,281 @@ +plugin: shell +category_id: com.canonical.plainbox::usb +id: usb/detect +requires: + cpuinfo.platform != 's390x' +estimated_duration: 1.0 +command: + set -o pipefail + if [[ -v SNAP ]]; then + lsusb.py -f $SNAP/var/lib/usbutils/usb.ids 2>/dev/null | sed 's/.*\(ID .*\)/\1/' | head -n 4 || echo "No USB devices were detected" >&2 + else + lsusb 2>/dev/null | sort || echo "No USB devices were detected" >&2 + fi +_summary: Display USB devices attached to SUT +_description: Detects and shows USB devices attached to this system. + +plugin: user-interact-verify +category_id: com.canonical.plainbox::usb +id: usb/disk_detect +depends: usb/detect +estimated_duration: 1.0 +command: removable_storage_test -l usb +_description: + PURPOSE: + This test will check that your system detects USB storage devices. + STEPS: + 1. Plug in one or more USB keys or hard drives. + 2. Click on "Test". + VERIFICATION: + Were the drives detected? + +plugin: user-interact-verify +category_id: com.canonical.plainbox::usb +id: usb/HID +depends: usb/detect +estimated_duration: 1.0 +command: keyboard_test +_description: + PURPOSE: + This test will check that you can use a USB HID device + STEPS: + 1. Enable either a USB mouse or keyboard + 2. For mice, perform actions such as moving the pointer, right and left button clicks and double clicks + 3. For keyboards, commence the test to launch a small tool. Type some text and close the tool. + VERIFICATION: + Did the device work as expected? + +id: usb/insert +_summary: USB 2.0 storage device insertion detected +_purpose: + Check system can detect USB 2.0 storage when inserted +_steps: + 1. Press continue + 2. Connect USB 2.0 storage device +_verification: + The verification of this test is automated. + Do not change the automatically selected result. +plugin: user-interact +flags: also-after-suspend +user: root +command: + if [[ -v SNAP ]]; then + checkbox-support-run_watcher insertion usb2 + else + removable_storage_watcher insert usb + fi +category_id: com.canonical.plainbox::usb +estimated_duration: 120 + +id: usb3/insert +requires: + usb.usb3 == 'supported' +_summary: USB 3.0 storage device insertion detected +_purpose: + Check system can detect insertion of a USB 3.0 storage device +_steps: + 1. Press continue + 2. Connect USB 3.0 storage device +_verification: + The verification of this test is automated. + Do not change the automatically selected result. +plugin: user-interact +user: root +command: + if [[ -v SNAP ]]; then + checkbox-support-run_watcher insertion usb3 + else + removable_storage_watcher -m 500000000 insert usb + fi +category_id: com.canonical.plainbox::usb +estimated_duration: 120 + +id: usb/remove +_summary: USB 2.0 storage device removal detected +_purpose: + Check system can detect removal of a USB 2.0 storage device +_steps: + 1. Press continue + 2. Disconnect USB 2.0 storage device +_verification: + The verification of this test is automated. + Do not change the automatically selected result. +plugin: user-interact +depends: usb/insert +flags: also-after-suspend +user: root +command: + if [[ -v SNAP ]]; then + checkbox-support-run_watcher removal usb2 + else + removable_storage_watcher remove usb + fi +category_id: com.canonical.plainbox::usb +estimated_duration: 120 + +id: usb3/remove +_summary: USB 3.0 storage device removal detected +_purpose: + Check system can detect removal of a USB 3.0 storage device +_steps: + 1. Press continue + 2. Disconnect USB 3.0 storage device +_verification: + The verification of this test is automated. + Do not change the automatically selected result. +plugin: user-interact +flags: also-after-suspend +depends: usb3/insert +user: root +command: + if [[ -v SNAP ]]; then + checkbox-support-run_watcher removal usb3 + else + removable_storage_watcher -m 500000000 remove usb + fi +category_id: com.canonical.plainbox::usb +estimated_duration: 120 + +plugin: user-interact-verify +category_id: com.canonical.plainbox::usb +id: usb/storage-transfer +depends: usb/insert +user: root +estimated_duration: 45.0 +command: removable_storage_test -s 268400000 usb +_description: + PURPOSE: + This test will check your USB connection. + STEPS: + 1. Plug a USB HDD or thumbdrive into the computer. + 2. An icon should appear on the Launcher. + 3. Click "Test" to begin the test. + VERIFICATION: + The verification of this test is automated. Do not change the + automatically selected result. + +plugin: user-interact-verify +category_id: com.canonical.plainbox::usb +id: usb3/storage-transfer +requires: + usb.usb3 == 'supported' +depends: usb3/insert +user: root +estimated_duration: 45.0 +command: removable_storage_test -s 268400000 -m 500000000 usb +_description: + PURPOSE: + This test will check your USB 3.0 connection. + STEPS: + 1. Plug a USB 3.0 HDD or thumbdrive into a USB 3.0 port in the computer. + 2. An icon should appear on the Launcher. + 3. Click "Test" to begin the test. + VERIFICATION: + The verification of this test is automated. Do not change the + automatically selected result. + +id: usb/storage-automated +_summary: USB 2.0 storage device read & write works +_purpose: + Check system can read/write to USB 2.0 storage correctly +_steps: + 1. This task is fully automatic and need USB 2.0 insertion test was applied first. +_verification: + This task is fully automatic and will verify the result for you. +plugin: shell +depends: usb/insert +flags: also-after-suspend +user: root +command: + if [[ -v SNAP ]]; then + checkbox-support-usb_read_write + else + removable_storage_test -s 268400000 usb + fi +category_id: com.canonical.plainbox::usb +estimated_duration: 300 + +id: usb3/storage-automated +_summary: USB 3.0 storage device read & write works +_purpose: + Check system can read/write to USB 3.0 storage devices correctly +_steps: + 1. This task is fully automatic and need USB 3.0 insertion test was applied first. +_verification: + This task is fully automatic and will verify the result for you. +plugin: shell +depends: usb3/insert +flags: also-after-suspend +user: root +command: + if [[ -v SNAP ]]; then + checkbox-support-usb_read_write + else + removable_storage_test -s 268400000 -m 500000000 usb --driver xhci_hcd + fi +category_id: com.canonical.plainbox::usb +estimated_duration: 300 + +plugin: shell +category_id: com.canonical.plainbox::usb +id: usb/storage-preinserted +user: root +estimated_duration: 45.0 +command: removable_storage_test -l usb && removable_storage_test -s 268400000 usb +flags: also-after-suspend preserve-cwd +requires: + cpuinfo.platform != 's390x' + package.name == 'udisks2' or snap.name == 'udisks2' + package.name == 'udisks2' or (snap.name == 'core' and int(snap.revision) >= 1804) +_summary: + Test USB 2.0 or 1.1 ports +_description: + Tests USB 2.0 or 1.1 ports on a system by doing write/read/compare tests on + randomly created data. It requires that a USB stick is plugged into an + available USB port before running the certification suite. + +plugin: shell +category_id: com.canonical.plainbox::usb +id: usb3/storage-preinserted +user: root +flags: also-after-suspend +requires: + cpuinfo.platform != 's390x' + usb.usb3 == 'supported' + package.name == 'udisks2' or snap.name == 'udisks2' + package.name == 'udisks2' or (snap.name == 'core' and int(snap.revision) >= 1804) +estimated_duration: 45.0 +command: removable_storage_test -l usb && removable_storage_test -s 268400000 -m 500000000 usb --driver xhci_hcd +_summary: + Test USB 3.0 or 3.1 ports +_description: + Tests USB 3.0 ports on a system by doing write/read/compare tests on + randomly created data. It requires that a USB stick is plugged into an + available USB port before running the certification suite. Additionally, it + will only work with USB sticks and ports rated for USB 3.0 speeds or faster. + +plugin: manual +category_id: com.canonical.plainbox::usb +id: usb/panels +_description: + PURPOSE: + This test will check your USB connection. + STEPS: + 1. Connect a USB storage device to an external USB slot on this computer. + 2. An icon should appear on the Launcher. + 3. Confirm that the icon appears. + 4. Eject the device. + 5. Repeat with each external USB slot. + VERIFICATION: + Do all USB slots work with the device? + +plugin: shell +category_id: com.canonical.plainbox::usb +id: usb/performance +depends: usb/insert +user: root +estimated_duration: 45.00 +command: removable_storage_test -s 268400000 -p 15 usb +_description: + This test will check that your USB 2.0 port transfers data at a + minimum expected speed. diff --git a/units/virtualization/jobs.pxu b/units/virtualization/jobs.pxu new file mode 100644 index 0000000..39f37e9 --- /dev/null +++ b/units/virtualization/jobs.pxu @@ -0,0 +1,31 @@ +plugin: shell +category_id: com.canonical.plainbox::virtualization +id: virtualization/kvm_check_vm +user: root +environ: KVM_TIMEOUT KVM_IMAGE http_proxy https_proxy +estimated_duration: 300.0 +requires: + package.name == 'qemu-system' + package.name == 'qemu-utils' + virtualization.kvm == 'supported' +command: virtualization --debug kvm --log-file=$PLAINBOX_SESSION_SHARE/virt_debug +_description: + Verifies that a KVM guest can be created and booted using an Ubuntu Server + cloud image. +_summary: + Verify KVM guest boots + +plugin: shell +category_id: com.canonical.plainbox::virtualization +id: virtualization/verify_lxd +user: root +environ: LXD_TEMPLATE LXD_ROOTFS +estimated_duration: 30.0 +requires: + package.name == 'lxd-client' or executable.name == 'lxc' + package.name == 'lxd' or snap.name == 'lxd' +command: virtualization --debug lxd +_description: + Verifies that an LXD container can be created and launched +_summary: + Verify LXD container launches diff --git a/units/wireless/jobs.pxu b/units/wireless/jobs.pxu new file mode 100644 index 0000000..cf13a3c --- /dev/null +++ b/units/wireless/jobs.pxu @@ -0,0 +1,693 @@ +plugin: shell +category_id: com.canonical.plainbox::wireless +id: wireless/wireless_scanning +requires: + package.name == 'network-manager' + device.category == 'WIRELESS' +command: + rfkill unblock wlan wifi + if rfkill list wlan wifi | grep -q 'Hard blocked: yes'; then + echo "Hard block is applied to WiFi device. Please remove and retest." + exit 1 + fi + wireless_networks=`(nmcli -f SSID dev wifi list 2>/dev/null || nmcli -f SSID dev wifi)` + if [ `echo "$wireless_networks" | wc -l` -gt 1 ]; then + echo "Wireless networks discovered: " + echo "$wireless_networks" + exit 0 + fi + echo "No wireless networks discovered." + exit 1 +estimated_duration: 0.645 +_description: Wireless scanning test. It scans and reports on discovered APs. + +plugin: shell +category_id: com.canonical.plainbox::wireless +id: wireless/info_automated +requires: + package.name == 'network-manager' + device.category == 'WIRELESS' +command: udev_resource -f WIRELESS | awk "/interface: / { print \$2 }" | xargs -n 1 network_info +estimated_duration: 1.2 +_description: + This is an automated test to gather some info on the current state of your wireless devices. If no devices are found, the test will exit with an error. + +plugin: user-interact-verify +category_id: com.canonical.plainbox::wireless +id: wireless/wireless_connection +command: network_check +estimated_duration: 120.0 +requires: device.category == 'WIRELESS' +_description: + PURPOSE: + This test will check your wireless connection. + STEPS: + 1. Click on the Network icon in the panel. + 2. Select a network below the 'Wireless networks' section. + 3. Click "Test" to verify that it's possible to establish an HTTP connection. + VERIFICATION: + Did a notification show and was the connection correctly established? + +plugin: shell +category_id: com.canonical.plainbox::wireless +id: wireless/wireless_connection_wpa_bg +requires: + device.category == 'WIRELESS' + environment.ROUTERS == 'multiple' +user: root +environ: WPA_BG_SSID WPA_BG_PSK +command: + trap "nmcli con delete id $WPA_BG_SSID" EXIT + if create_connection wifi $WPA_BG_SSID --security=wpa --key=$WPA_BG_PSK; then + connect_wireless # lp:1471663 + INTERFACE=`nmcli dev status | awk '/802-11-wireless|wifi/ {print $1}'` + iw dev $INTERFACE link + gateway_ping_test --interface=$INTERFACE + STATUS=$? + # We reconnect the Ethernet connection if any (lp:1471663) + WIRED=$(nmcli -f UUID,TYPE c | grep -oP ".*(?=\s+.*ethernet)") + if [[ ! -z $WIRED ]]; then + nmcli c up uuid $WIRED + fi + exit $STATUS + else + exit 1 + fi +estimated_duration: 30.0 +_description: + Tests that the systems wireless hardware can connect to a router using WPA + security and the 802.11b/g protocols. + +plugin: shell +category_id: com.canonical.plainbox::wireless +id: wireless/wireless_connection_open_bg +requires: + device.category == 'WIRELESS' + environment.ROUTERS == 'multiple' +user: root +environ: OPEN_BG_SSID +command: + trap "nmcli con delete id $OPEN_BG_SSID" EXIT + if create_connection wifi $OPEN_BG_SSID; then + connect_wireless # lp:1471663 + INTERFACE=`nmcli dev status | awk '/802-11-wireless|wifi/ {print $1}'` + iw dev $INTERFACE link + gateway_ping_test --interface=$INTERFACE + STATUS=$? + # We reconnect the Ethernet connection if any (lp:1471663) + WIRED=$(nmcli -f UUID,TYPE c | grep -oP ".*(?=\s+.*ethernet)") + if [[ ! -z $WIRED ]]; then + nmcli c up uuid $WIRED + fi + exit $STATUS + else + exit 1 + fi +estimated_duration: 30.0 +_description: + Tests that the systems wireless hardware can connect to a router using no + security and the 802.11b/g protocols. + +plugin: shell +category_id: com.canonical.plainbox::wireless +id: wireless/wireless_connection_wpa_n +requires: + device.category == 'WIRELESS' + environment.ROUTERS == 'multiple' +user: root +environ: WPA_N_SSID WPA_N_PSK +command: + trap "nmcli con delete id $WPA_N_SSID" EXIT + if create_connection wifi $WPA_N_SSID --security=wpa --key=$WPA_N_PSK; then + connect_wireless # lp:1471663 + INTERFACE=`nmcli dev status | awk '/802-11-wireless|wifi/ {print $1}'` + iw dev $INTERFACE link + gateway_ping_test --interface=$INTERFACE + STATUS=$? + # We reconnect the Ethernet connection if any (lp:1471663) + WIRED=$(nmcli -f UUID,TYPE c | grep -oP ".*(?=\s+.*ethernet)") + if [[ ! -z $WIRED ]]; then + nmcli c up uuid $WIRED + fi + exit $STATUS + else + exit 1 + fi +estimated_duration: 30.0 +_description: + Tests that the systems wireless hardware can connect to a router using WPA + security and the 802.11n protocol. + +plugin: shell +category_id: com.canonical.plainbox::wireless +id: wireless/wireless_connection_open_n +requires: + device.category == 'WIRELESS' + environment.ROUTERS == 'multiple' +user: root +environ: OPEN_N_SSID +command: + trap "nmcli con delete id $OPEN_N_SSID" EXIT + if create_connection wifi $OPEN_N_SSID; then + connect_wireless # lp:1471663 + INTERFACE=`nmcli dev status | awk '/802-11-wireless|wifi/ {print $1}'` + iw dev $INTERFACE link + gateway_ping_test --interface=$INTERFACE + STATUS=$? + # We reconnect the Ethernet connection if any (lp:1471663) + WIRED=$(nmcli -f UUID,TYPE c | grep -oP ".*(?=\s+.*ethernet)") + if [[ ! -z $WIRED ]]; then + nmcli c up uuid $WIRED + fi + exit $STATUS + else + exit 1 + fi +estimated_duration: 30.0 +_description: + Tests that the systems wireless hardware can connect to a router using no + security and the 802.11n protocol. + +plugin: shell +category_id: com.canonical.plainbox::wireless +id: wireless/wireless_connection_wpa_ac +requires: + device.category == 'WIRELESS' + environment.ROUTERS == 'multiple' + IEEE_80211.ac == 'supported' +user: root +environ: WPA_AC_SSID WPA_AC_PSK +command: + trap "nmcli con delete id $WPA_AC_SSID" EXIT + if create_connection wifi $WPA_AC_SSID --security=wpa --key=$WPA_AC_PSK; then + connect_wireless # lp:1471663 + INTERFACE=`nmcli dev status | awk '/802-11-wireless|wifi/ {print $1}'` + iw dev $INTERFACE link + gateway_ping_test --interface=$INTERFACE + STATUS=$? + # We reconnect the Ethernet connection if any (lp:1471663) + WIRED=$(nmcli -f UUID,TYPE c | grep -oP ".*(?=\s+.*ethernet)") + if [[ ! -z $WIRED ]]; then + nmcli c up uuid $WIRED + fi + exit $STATUS + else + exit 1 + fi +estimated_duration: 30.0 +_description: + Tests that the systems wireless hardware can connect to a router using WPA + security and the 802.11ac protocol. + +plugin: shell +category_id: com.canonical.plainbox::wireless +id: wireless/wireless_connection_open_ac +requires: + device.category == 'WIRELESS' + environment.ROUTERS == 'multiple' + IEEE_80211.ac == 'supported' +user: root +environ: OPEN_AC_SSID +command: + trap "nmcli con delete id $OPEN_AC_SSID" EXIT + if create_connection wifi $OPEN_AC_SSID; then + connect_wireless # lp:1471663 + INTERFACE=`nmcli dev status | awk '/802-11-wireless|wifi/ {print $1}'` + iw dev $INTERFACE link + gateway_ping_test --interface=$INTERFACE + STATUS=$? + # We reconnect the Ethernet connection if any (lp:1471663) + WIRED=$(nmcli -f UUID,TYPE c | grep -oP ".*(?=\s+.*ethernet)") + if [[ ! -z $WIRED ]]; then + nmcli c up uuid $WIRED + fi + exit $STATUS + else + exit 1 + fi +estimated_duration: 30.0 +_description: + Tests that the systems wireless hardware can connect to a router using no + security and the 802.11ac protocol. + +plugin: user-interact-verify +category_id: com.canonical.plainbox::wireless +id: wireless/wireless_connection_wpa_bg_manual +requires: + device.category == 'WIRELESS' + environment.ROUTERS == 'single' +user: root +environ: ROUTER_SSID ROUTER_PSK +command: + trap "nmcli con delete id $ROUTER_SSID" EXIT + if create_connection wifi $ROUTER_SSID --security=wpa --key=$ROUTER_PSK; then + connect_wireless # lp:1471663 + INTERFACE=`nmcli dev status | awk '/802-11-wireless|wifi/ {print $1}'` + iw dev $INTERFACE link + gateway_ping_test --interface=$INTERFACE + STATUS=$? + # We reconnect the Ethernet connection if any (lp:1471663) + WIRED=$(nmcli -f UUID,TYPE c | grep -oP ".*(?=\s+.*ethernet)") + if [[ ! -z $WIRED ]]; then + nmcli c up uuid $WIRED + fi + exit $STATUS + else + exit 1 + fi +estimated_duration: 120.0 +_description: + PURPOSE: + Tests that the systems wireless hardware can connect to a router using WPA + security and the 802.11b/g protocols. + STEPS: + 1. Open your router's configuration tool + 2. Change the settings to only accept connections on the B and G wireless bands + 3. Make sure the SSID is set to the contents of the ROUTER_SSID environment variable + 4. Change the security settings to use WPA2 and ensure the PSK matches what is set in the ROUTER_PSK environment variable + 5. Commence the test to create a connection to the router and test the connection + VERIFICATION: + Verification is automated, do not change the automatically selected result. + +plugin: user-interact-verify +category_id: com.canonical.plainbox::wireless +id: wireless/wireless_connection_open_bg_manual +requires: + device.category == 'WIRELESS' + environment.ROUTERS == 'single' +user: root +environ: ROUTER_SSID +command: + trap "nmcli con delete id $ROUTER_SSID" EXIT + if create_connection wifi $ROUTER_SSID; then + connect_wireless # lp:1471663 + INTERFACE=`nmcli dev status | awk '/802-11-wireless|wifi/ {print $1}'` + iw dev $INTERFACE link + gateway_ping_test --interface=$INTERFACE + STATUS=$? + # We reconnect the Ethernet connection if any (lp:1471663) + WIRED=$(nmcli -f UUID,TYPE c | grep -oP ".*(?=\s+.*ethernet)") + if [[ ! -z $WIRED ]]; then + nmcli c up uuid $WIRED + fi + exit $STATUS + else + exit 1 + fi +estimated_duration: 120.0 +_description: + PURPOSE: + Tests that the systems wireless hardware can connect to a router using + no security and the 802.11b/g protocols. + STEPS: + 1. Open your router's configuration tool + 2. Change the settings to only accept connections on the B and G wireless bands + 3. Make sure the SSID is set to the contents of the ROUTER_SSID environment variable + 4. Change the security settings to use no security + 5. Commence the test to create a connection to the router and test the connection + VERIFICATION: + Verification is automated, do not change the automatically selected result. + +plugin: user-interact-verify +category_id: com.canonical.plainbox::wireless +id: wireless/wireless_connection_wpa_n_manual +requires: + device.category == 'WIRELESS' + environment.ROUTERS == 'single' + IEEE_80211.band_5GHz == 'supported' +user: root +environ: ROUTER_SSID ROUTER_PSK +command: + trap "nmcli con delete id $ROUTER_SSID" EXIT + if create_connection wifi $ROUTER_SSID --security=wpa --key=$ROUTER_PSK; then + connect_wireless # lp:1471663 + INTERFACE=`nmcli dev status | awk '/802-11-wireless|wifi/ {print $1}'` + iw dev $INTERFACE link + gateway_ping_test --interface=$INTERFACE + STATUS=$? + # We reconnect the Ethernet connection if any (lp:1471663) + WIRED=$(nmcli -f UUID,TYPE c | grep -oP ".*(?=\s+.*ethernet)") + if [[ ! -z $WIRED ]]; then + nmcli c up uuid $WIRED + fi + exit $STATUS + else + exit 1 + fi +estimated_duration: 120.0 +_description: + PURPOSE: + Tests that the systems wireless hardware can connect to a router using WPA + security and the 802.11n protocols. + STEPS: + 1. Open your router's configuration tool + 2. Change the settings to only accept connections on the N wireless band + 3. Make sure the SSID is set to the contents of the ROUTER_SSID environment variable + 4. Change the security settings to use WPA2 and ensure the PSK matches what is set in the ROUTER_PSK environment variable + 5. Commence the test to create a connection to the router and test the connection + VERIFICATION: + Verification is automated, do not change the automatically selected result. + +plugin: user-interact-verify +category_id: com.canonical.plainbox::wireless +id: wireless/wireless_connection_open_n_manual +requires: + device.category == 'WIRELESS' + environment.ROUTERS == 'single' + IEEE_80211.band_5GHz == 'supported' +user: root +environ: ROUTER_SSID +command: + trap "nmcli con delete id $ROUTER_SSID" EXIT + if create_connection wifi $ROUTER_SSID; then + connect_wireless # lp:1471663 + INTERFACE=`nmcli dev status | awk '/802-11-wireless|wifi/ {print $1}'` + iw dev $INTERFACE link + gateway_ping_test --interface=$INTERFACE + STATUS=$? + # We reconnect the Ethernet connection if any (lp:1471663) + WIRED=$(nmcli -f UUID,TYPE c | grep -oP ".*(?=\s+.*ethernet)") + if [[ ! -z $WIRED ]]; then + nmcli c up uuid $WIRED + fi + exit $STATUS + else + exit 1 + fi +estimated_duration: 120.0 +_description: + PURPOSE: + Tests that the systems wireless hardware can connect to a router using + no security and the 802.11n protocol. + STEPS: + 1. Open your router's configuration tool + 2. Change the settings to only accept connections on the N wireless band + 3. Make sure the SSID is set to the contents of the ROUTER_SSID environment variable + 4. Change the security settings to use no security + 5. Commence the test to create a connection to the router and test the connection + VERIFICATION: + Verification is automated, do not change the automatically selected result. + +plugin: user-interact-verify +category_id: com.canonical.plainbox::wireless +id: wireless/wireless_connection_wpa_ac_manual +requires: + device.category == 'WIRELESS' + environment.ROUTERS == 'single' + IEEE_80211.ac == 'supported' +user: root +environ: ROUTER_SSID ROUTER_PSK +command: + trap "nmcli con delete id $ROUTER_SSID" EXIT + if create_connection wifi $ROUTER_SSID --security=wpa --key=$ROUTER_PSK; then + connect_wireless # lp:1471663 + INTERFACE=`nmcli dev status | awk '/802-11-wireless|wifi/ {print $1}'` + iw dev $INTERFACE link + gateway_ping_test --interface=$INTERFACE + STATUS=$? + # We reconnect the Ethernet connection if any (lp:1471663) + WIRED=$(nmcli -f UUID,TYPE c | grep -oP ".*(?=\s+.*ethernet)") + if [[ ! -z $WIRED ]]; then + nmcli c up uuid $WIRED + fi + exit $STATUS + else + exit 1 + fi +estimated_duration: 120.0 +_description: + PURPOSE: + Tests that the systems wireless hardware can connect to a router using WPA + security and the 802.11ac protocol. + STEPS: + 1. Open your router's configuration tool + 2. Change the settings to only accept connections with the 802.11ac protocol. + 3. Make sure the SSID is set to the contents of the ROUTER_SSID environment variable + 4. Change the security settings to use WPA2 and ensure the PSK matches what is set in the ROUTER_PSK environment variable + 5. Commence the test to create a connection to the router and test the connection + VERIFICATION: + Verification is automated, do not change the automatically selected result. + +plugin: user-interact-verify +category_id: com.canonical.plainbox::wireless +id: wireless/wireless_connection_open_ac_manual +requires: + device.category == 'WIRELESS' + environment.ROUTERS == 'single' + IEEE_80211.ac == 'supported' +user: root +environ: ROUTER_SSID +command: + trap "nmcli con delete id $ROUTER_SSID" EXIT + if create_connection wifi $ROUTER_SSID; then + connect_wireless # lp:1471663 + INTERFACE=`nmcli dev status | awk '/802-11-wireless|wifi/ {print $1}'` + iw dev $INTERFACE link + gateway_ping_test --interface=$INTERFACE + STATUS=$? + # We reconnect the Ethernet connection if any (lp:1471663) + WIRED=$(nmcli -f UUID,TYPE c | grep -oP ".*(?=\s+.*ethernet)") + if [[ ! -z $WIRED ]]; then + nmcli c up uuid $WIRED + fi + exit $STATUS + else + exit 1 + fi +estimated_duration: 120.0 +_description: + PURPOSE: + Tests that the systems wireless hardware can connect to a router using + no security and the 802.11ac protocol. + STEPS: + 1. Open your router's configuration tool + 2. Change the settings to only accept connections with the 802.11ac protocol. + 3. Make sure the SSID is set to the contents of the ROUTER_SSID environment variable + 4. Change the security settings to use no security + 5. Commence the test to create a connection to the router and test the connection + VERIFICATION: + Verification is automated, do not change the automatically selected result. + +plugin: shell +category_id: com.canonical.plainbox::wireless +id: wireless/monitor_wireless_connection +requires: + package.name == 'iperf' + device.category == 'WIRELESS' +user: root +environ: WPA_BG_SSID WPA_BG_PSK SERVER_IPERF +command: + trap "nmcli con delete id $WPA_BG_SSID" EXIT + if create_connection wifi $WPA_BG_SSID --security=wpa --key=$WPA_BG_PSK; then + connect_wireless # lp:1471663 + INTERFACE=`nmcli dev status | awk '/802-11-wireless|wifi/ {print $1}'` + iw dev $INTERFACE link + iperf -c $SERVER_IPERF -t 300 -i 30 + STATUS=$? + # We reconnect the Ethernet connection if any (lp:1471663) + WIRED=$(nmcli -f UUID,TYPE c | grep -oP ".*(?=\s+.*ethernet)") + if [[ ! -z $WIRED ]]; then + nmcli c up uuid $WIRED + fi + exit $STATUS + else + exit 1 + fi +estimated_duration: 330.0 +_description: + Tests the performance of a system's wireless connection through the iperf tool. + +plugin: shell +category_id: com.canonical.plainbox::wireless +id: wireless/monitor_wireless_connection_udp +requires: + package.name == 'iperf' + device.category == 'WIRELESS' +user: root +environ: WPA_BG_SSID WPA_BG_PSK SERVER_IPERF +command: + trap "nmcli con delete id $WPA_BG_SSID" EXIT + if create_connection wifi $WPA_BG_SSID --security=wpa --key=$WPA_BG_PSK; then + connect_wireless # lp:1471663 + INTERFACE=`nmcli dev status | awk '/802-11-wireless|wifi/ {print $1}'` + iw dev $INTERFACE link + iperf -c $SERVER_IPERF -t 300 -i 30 -u -b 100m -p 5050 + STATUS=$? + # We reconnect the Ethernet connection if any (lp:1471663) + WIRED=$(nmcli -f UUID,TYPE c | grep -oP ".*(?=\s+.*ethernet)") + if [[ ! -z $WIRED ]]; then + nmcli c up uuid $WIRED + fi + exit $STATUS + else + exit 1 + fi +estimated_duration: 330.0 +_description: + Tests the performance of a system's wireless connection through the iperf tool, using UDP packets. + +plugin: shell +category_id: com.canonical.plainbox::wireless +id: wireless/wireless_connection_open_a +requires: + device.category == 'WIRELESS' + IEEE_80211.band_5GHz == 'supported' +user: root +environ: OPEN_A_SSID +command: + trap "nmcli con delete id $OPEN_A_SSID" EXIT + if create_connection wifi $OPEN_A_SSID; then + connect_wireless # lp:1471663 + INTERFACE=`nmcli dev status | awk '/802-11-wireless|wifi/ {print $1}'` + iw dev $INTERFACE link + gateway_ping_test --interface=$INTERFACE + STATUS=$? + # We reconnect the Ethernet connection if any (lp:1471663) + WIRED=$(nmcli -f UUID,TYPE c | grep -oP ".*(?=\s+.*ethernet)") + if [[ ! -z $WIRED ]]; then + nmcli c up uuid $WIRED + fi + exit $STATUS + else + exit 1 + fi +estimated_duration: 30.0 +_description: + Test that the system's wireless hardware can connect to a router using the + 802.11a protocol. This requires that you have a router pre-configured to only + respond to requests on the 802.11a protocol. + +plugin: shell +category_id: com.canonical.plainbox::wireless +id: wireless/wireless_connection_open_b +requires: + device.category == 'WIRELESS' +user: root +environ: OPEN_B_SSID +command: + trap "nmcli con delete id $OPEN_B_SSID" EXIT + if create_connection wifi $OPEN_B_SSID; then + connect_wireless # lp:1471663 + INTERFACE=`nmcli dev status | awk '/802-11-wireless|wifi/ {print $1}'` + iw dev $INTERFACE link + gateway_ping_test --interface=$INTERFACE + STATUS=$? + # We reconnect the Ethernet connection if any (lp:1471663) + WIRED=$(nmcli -f UUID,TYPE c | grep -oP ".*(?=\s+.*ethernet)") + if [[ ! -z $WIRED ]]; then + nmcli c up uuid $WIRED + fi + exit $STATUS + else + exit 1 + fi +estimated_duration: 30.0 +_description: + Test that the system's wireless hardware can connect to a router using the + 802.11b protocol. This requires that you have a router pre-configured to only + respond to requests on the 802.11b protocol. + +unit: template +template-resource: device +template-filter: + device.category == 'WIRELESS' +plugin: shell +category_id: com.canonical.plainbox::wireless +id: wireless/stress_performance_device{__index__}_{interface} +estimated_duration: 330.0 +requires: + package.name == 'iperf' +environ: TEST_TARGET_IPERF +user: root +command: network test -i {interface} -t stress +_description: + This test executes iperf to generate a load on the network device {__index__} ({interface}) and then performs a ping test to watch for dropped packets and very large latency periods. + +plugin: shell +category_id: com.canonical.plainbox::wireless +id: wireless/wireless_connection_open_g +requires: + device.category == 'WIRELESS' +user: root +environ: OPEN_G_SSID +command: + trap "nmcli con delete id $OPEN_G_SSID" EXIT + if create_connection wifi $OPEN_G_SSID; then + connect_wireless # lp:1471663 + INTERFACE=`nmcli dev status | awk '/802-11-wireless|wifi/ {print $1}'` + iw dev $INTERFACE link + gateway_ping_test --interface=$INTERFACE + STATUS=$? + # We reconnect the Ethernet connection if any (lp:1471663) + WIRED=$(nmcli -f UUID,TYPE c | grep -oP ".*(?=\s+.*ethernet)") + if [[ ! -z $WIRED ]]; then + nmcli c up uuid $WIRED + fi + exit $STATUS + else + exit 1 + fi +estimated_duration: 30.0 +_description: + Test that the system's wireless hardware can connect to a router using the + 802.11g protocol. This requires that you have a router pre-configured to only + respond to requests on the 802.11g protocol. + +plugin: shell +category_id: com.canonical.plainbox::wireless +id: wireless/wireless_extension +requires: device.category == 'WIRELESS' +command: wireless_ext +estimated_duration: 1.2 +_description: + Test that the MAC80211 modules are loaded and wireless extensions are working. + +unit: template +template-resource: device +template-filter: device.category == 'WIRELESS' +plugin: shell +category_id: com.canonical.plainbox::wireless +id: wireless/iwconfig_check_device{__index__}_{interface} +estimated_duration: 1.2 +command: iwconfig {interface} +_description: + This test executes iwconfig requests against wireless device {__index__} ({interface}). + +plugin: user-interact-verify +category_id: com.canonical.plainbox::wireless +id: wireless/wireless_rfkill +command: rfkill list | zenity --text-info --title rfkill-Info +estimated_duration: 120.0 +requires: device.category == 'WIRELESS' +_description: + PURPOSE: + This test will check whether or not your driver responds to rfkill commands. + STEPS: + 1. Use the hardware switch on the side of your device to switch off wireless. + 2. If you do not have a hardware switch disable wireless from the network manager icon in the panel + 3. Click "Test" to verify that the hard or soft blocks are in place. + VERIFICATION: + Did the hard or soft blocks show on in the dialog? + +unit: template +template-resource: device +template-filter: device.category == 'WIRELESS' +plugin: user-interact-verify +category_id: com.canonical.plainbox::wireless +id: wireless/maximum_bandwidth_device{__index__}_{interface} +estimated_duration: 120.0 +requires: + package.name == 'zenity' + package.name == 'iperf' +environ: TEST_TARGET_IPERF +user: root +command: network test -i {interface} -t iperf 2>&1 | cat - <(echo; echo "Verify the result and click OK to decide on the outcome") | zenity --text-info --title 'mobile broadband max bw {interface}' +_purpose: + User verification of whether the observed transfer throughput is acceptable + for the type and maximum speed of wireless device {__index__} ({interface}). +_steps: + 1. Click "Test". + 2. Read the network test summary and confirm that the throughput is acceptable. + 3. If needed, click "Test" again to repeat the transfer test. +_verification: + Was the reported throughput acceptable for the type and maximum speed of this interface? diff --git a/units/wireless/test-plan.pxu b/units/wireless/test-plan.pxu new file mode 100644 index 0000000..b4cf2f7 --- /dev/null +++ b/units/wireless/test-plan.pxu @@ -0,0 +1,66 @@ +id: wireless-cert-full +unit: test plan +_name: Wireless tests +_description: Wireless connection tests +include: +nested_part: + wireless-cert-automated + +id: after-suspend-wireless-cert-full +unit: test plan +_name: Wireless tests (after suspend) +_description: Wireless connection tests (after suspend) +include: +nested_part: + after-suspend-wireless-cert-automated + +id: wireless-cert-automated +unit: test plan +_name: Wireless tests +_description: Wireless connection tests +include: + wireless/wireless_scanning certification-status=blocker + wireless/wireless_connection_wpa_bg certification-status=blocker + wireless/wireless_connection_open_bg certification-status=blocker + wireless/wireless_connection_wpa_n certification-status=blocker + wireless/wireless_connection_open_n certification-status=blocker + wireless/wireless_connection_wpa_ac certification-status=blocker + wireless/wireless_connection_open_ac certification-status=blocker + +id: after-suspend-wireless-cert-automated +unit: test plan +_name: Wireless tests (after suspend, automated) +_description: Wireless connection tests (after suspend, automated) +include: + suspend/wireless_connection_after_suspend_wpa_bg certification-status=blocker + suspend/wireless_connection_after_suspend_open_bg certification-status=blocker + suspend/wireless_connection_after_suspend_wpa_n certification-status=blocker + suspend/wireless_connection_after_suspend_open_n certification-status=blocker + suspend/wireless_connection_after_suspend_wpa_ac certification-status=blocker + suspend/wireless_connection_after_suspend_open_ac certification-status=blocker + +id: wireless-cert-blockers +unit: test plan +_name: Wireless tests (certification blockers only) +_description: Wireless connection tests (certification blockers only) +include: + wireless/wireless_scanning certification-status=blocker + wireless/wireless_connection_wpa_bg certification-status=blocker + wireless/wireless_connection_open_bg certification-status=blocker + wireless/wireless_connection_wpa_n certification-status=blocker + wireless/wireless_connection_open_n certification-status=blocker + wireless/wireless_connection_wpa_ac certification-status=blocker + wireless/wireless_connection_open_ac certification-status=blocker + +id: after-suspend-wireless-cert-blockers +unit: test plan +_name: Wireless tests (after suspend, certification blockers only) +_description: + Wireless connection tests (after suspend, certification blockers only) +include: + suspend/wireless_connection_after_suspend_wpa_bg certification-status=blocker + suspend/wireless_connection_after_suspend_open_bg certification-status=blocker + suspend/wireless_connection_after_suspend_wpa_n certification-status=blocker + suspend/wireless_connection_after_suspend_open_n certification-status=blocker + suspend/wireless_connection_after_suspend_wpa_ac certification-status=blocker + suspend/wireless_connection_after_suspend_open_ac certification-status=blocker |