stress: import from p-p-snappy

4 files changed, 577 insertions, 0 deletions

diff --git a/units/stress/boot.pxu b/units/stress/boot.pxu
new file mode 100644
index 0000000..dcc0bf5
--- /dev/null
+++ b/units/stress/boot.pxu
@@ -0,0 +1,262 @@
+# Copyright 2015 Canonical Ltd.
+# All rights reserved.
+#
+# Written by:
+# Jonathan Cave <jonathan.cave@canonical.com>
+
+
+unit: category
+id: stress-tests
+_name: Stress Tests
+
+
+unit: category
+id: stress-tests/cold-boot
+_name: Cold-boot Stress Test
+
+
+unit: category
+id: stress-tests/warm-boot
+_name: Warm-boot Stress Test
+
+
+id: reboot-run-generator
+category_id: stress-tests
+_description:
+ Generate a set of IDs corresponding to number of iterations required in the
+ reboots tests.
+plugin: resource
+environ: STRESS_BOOT_ITERATIONS
+command:
+ ITERATIONS=${STRESS_BOOT_ITERATIONS:-2}
+ for i in $(seq 2 $ITERATIONS); do
+ echo "reboot_id: $i"
+ echo "reboot_id_previous: $((i-1))"
+ echo
+ done
+estimated_duration: 1s
+flags: preserve-locale
+
+
+id: init-boot-loop-data
+category_id: stress-tests
+_summary: Generate the baseline data set to test against
+_description: This creates baseline data sets which be considered the master
+ copies and all further tests will be compared against these.
+unit: job
+plugin: shell
+command:
+ lspci -i $SNAP/usr/share/misc/pci.ids > $PLAINBOX_SESSION_SHARE/lspci_original || true
+ nmcli -t -f active,BSSID d w l | grep -oP "(?<=^yes:).*" > $PLAINBOX_SESSION_SHARE/wifi_original || true
+ checkbox-support-lsusb -f $SNAP/checkbox-runtime/var/lib/usbutils/usb.ids -s | sort > $PLAINBOX_SESSION_SHARE/lsusb_original || true
+environ: LD_LIBRARY_PATH
+user: root
+estimated_duration: 1s
+flags: preserve-locale
+
+id: cold-boot-loop-reboot1
+category_id: stress-tests/cold-boot
+_summary: Perform cold reboot 1
+_description: This is a template that will be used to generate a stress test
+ of the system boot. Specifically this is how the device will be request a
+ reboot.
+unit: job
+plugin: shell
+environ: STRESS_BOOT_WAKEUP_DELAY
+command: rtcwake --mode off -s ${STRESS_BOOT_WAKEUP_DELAY:-120}
+user: root
+flags: preserve-locale noreturn autorestart
+estimated_duration: 180.0
+depends: init-boot-loop-data
+
+id: cold-boot-loop-reboot{reboot_id}
+category_id: stress-tests/cold-boot
+_summary: Perform cold reboot {reboot_id}
+_description: This is a template that will be used to generate a stress test
+ of the system boot. Specifically this is how the device will be request a
+ reboot.
+unit: template
+template-resource: reboot-run-generator
+template-unit: job
+plugin: shell
+environ: STRESS_BOOT_WAKEUP_DELAY STRESS_BOOT_WAIT_DELAY
+command:
+ sleep ${{STRESS_BOOT_WAIT_DELAY:-120}}
+ rtcwake --mode off -s ${{STRESS_BOOT_WAKEUP_DELAY:-120}}
+user: root
+flags: preserve-locale noreturn autorestart
+estimated_duration: 180.0
+after: cold-boot-loop-test{reboot_id_previous}
+depends: init-boot-loop-data
+
+id: cold-boot-loop-test1
+category_id: stress-tests/cold-boot
+_summary: Cold boot system configuration test 1
+_description: This is a template that will be used to generate a stress test
+ of the system boot. Specifically this is the test that will be performed
+ on each cycle to verify that all hardware is detected.
+unit: job
+plugin: shell
+environ: LD_LIBRARY_PATH
+command:
+ lspci -i $SNAP/usr/share/misc/pci.ids > $PLAINBOX_SESSION_SHARE/lspci_test
+ nmcli -t -f active,BSSID d w l | grep -oP "(?<=^yes:).*" > $PLAINBOX_SESSION_SHARE/wifi_test
+ checkbox-support-lsusb -f $SNAP/checkbox-runtime/var/lib/usbutils/usb.ids -s | sort > $PLAINBOX_SESSION_SHARE/lsusb_test
+ diff -u $PLAINBOX_SESSION_SHARE/lspci_original $PLAINBOX_SESSION_SHARE/lspci_test
+ if [ $? -ne 0 ] ; then
+ echo "lspci mismatch during cycle 1"
+ exit 1
+ fi
+ diff -u $PLAINBOX_SESSION_SHARE/wifi_original $PLAINBOX_SESSION_SHARE/wifi_test
+ if [ $? -ne 0 ] ; then
+ echo "wifi mismatch during cycle 1"
+ exit 1
+ fi
+ diff -u $PLAINBOX_SESSION_SHARE/lsusb_original $PLAINBOX_SESSION_SHARE/lsusb_test
+ if [ $? -ne 0 ] ; then
+ echo "lsusb mismatch during cycle 1"
+ exit 1
+ fi
+user: root
+flags: preserve-locale
+estimated_duration: 1.0
+depends: cold-boot-loop-reboot1
+
+id: cold-boot-loop-test{reboot_id}
+category_id: stress-tests/cold-boot
+_summary: Cold boot system configuration test {reboot_id}
+_description: This is a template that will be used to generate a stress test
+ of the system boot. Specifically this is the test that will be performed
+ on each cycle to verify that all hardware is detected.
+unit: template
+template-resource: reboot-run-generator
+template-unit: job
+plugin: shell
+environ: LD_LIBRARY_PATH
+command:
+ lspci -i $SNAP/usr/share/misc/pci.ids > $PLAINBOX_SESSION_SHARE/lspci_test
+ nmcli -t -f active,BSSID d w l | grep -oP "(?<=^yes:).*" > $PLAINBOX_SESSION_SHARE/wifi_test
+ checkbox-support-lsusb -f $SNAP/checkbox-runtime/var/lib/usbutils/usb.ids -s | sort > $PLAINBOX_SESSION_SHARE/lsusb_test
+ diff -u $PLAINBOX_SESSION_SHARE/lspci_original $PLAINBOX_SESSION_SHARE/lspci_test
+ if [ $? -ne 0 ] ; then
+ echo "lspci mismatch during cycle {reboot_id}"
+ exit 1
+ fi
+ diff -u $PLAINBOX_SESSION_SHARE/wifi_original $PLAINBOX_SESSION_SHARE/wifi_test
+ if [ $? -ne 0 ] ; then
+ echo "wifi mismatch during cycle {reboot_id}"
+ exit 1
+ fi
+ diff -u $PLAINBOX_SESSION_SHARE/lsusb_original $PLAINBOX_SESSION_SHARE/lsusb_test
+ if [ $? -ne 0 ] ; then
+ echo "lsusb mismatch during cycle {reboot_id}"
+ exit 1
+ fi
+user: root
+flags: preserve-locale
+estimated_duration: 1.0
+depends: cold-boot-loop-reboot{reboot_id}
+
+
+id: warm-boot-loop-reboot1
+category_id: stress-tests/warm-boot
+_summary: Perform warm reboot 1
+_description: This is a template that will be used to generate a stress test
+ of the system boot. Specifically this is how the device will be request a
+ reboot.
+unit: job
+plugin: shell
+command:
+ reboot
+user: root
+flags: preserve-locale noreturn autorestart
+estimated_duration: 60s
+depends: init-boot-loop-data
+
+id: warm-boot-loop-reboot{reboot_id}
+category_id: stress-tests/warm-boot
+_summary: Perform warm reboot {reboot_id}
+_description: This is a template that will be used to generate a stress test
+ of the system boot. Specifically this is how the device will be request a
+ reboot.
+unit: template
+template-resource: reboot-run-generator
+template-unit: job
+plugin: shell
+environ: STRESS_BOOT_WAIT_DELAY
+command:
+ sleep ${{STRESS_BOOT_WAIT_DELAY:-120}}
+ reboot
+user: root
+flags: preserve-locale noreturn autorestart
+estimated_duration: 60.0
+after: warm-boot-loop-test{reboot_id_previous}
+depends: init-boot-loop-data
+
+id: warm-boot-loop-test1
+category_id: stress-tests/warm-boot
+_summary: Warm boot system configuration test 1
+_description: This is a template that will be used to generate a stress test
+ of the system boot. Specifically this is the test that will be performed
+ on each cycle to verify that all hardware is detected.
+unit: job
+plugin: shell
+environ: LD_LIBRARY_PATH
+command:
+ lspci -i $SNAP/usr/share/misc/pci.ids > $PLAINBOX_SESSION_SHARE/lspci_test
+ nmcli -t -f active,BSSID d w l | grep -oP "(?<=^yes:).*" > $PLAINBOX_SESSION_SHARE/wifi_test
+ checkbox-support-lsusb -f $SNAP/checkbox-runtime/var/lib/usbutils/usb.ids -s | sort > $PLAINBOX_SESSION_SHARE/lsusb_test
+ diff -u $PLAINBOX_SESSION_SHARE/lspci_original $PLAINBOX_SESSION_SHARE/lspci_test
+ if [ $? -ne 0 ] ; then
+ echo "lspci mismatch during cycle 1"
+ exit 1
+ fi
+ diff -u $PLAINBOX_SESSION_SHARE/wifi_original $PLAINBOX_SESSION_SHARE/wifi_test
+ if [ $? -ne 0 ] ; then
+ echo "wifi mismatch during cycle 1"
+ exit 1
+ fi
+ diff -u $PLAINBOX_SESSION_SHARE/lsusb_original $PLAINBOX_SESSION_SHARE/lsusb_test
+ if [ $? -ne 0 ] ; then
+ echo "lsusb mismatch during cycle 1"
+ exit 1
+ fi
+user: root
+flags: preserve-locale
+estimated_duration: 1.0
+depends: warm-boot-loop-reboot1
+
+id: warm-boot-loop-test{reboot_id}
+category_id: stress-tests/warm-boot
+_summary: Warm boot system configuration test {reboot_id}
+_description: This is a template that will be used to generate a stress test
+ of the system boot. Specifically this is the test that will be performed
+ on each cycle to verify that all hardware is detected.
+unit: template
+template-resource: reboot-run-generator
+template-unit: job
+plugin: shell
+environ: LD_LIBRARY_PATH
+command:
+ lspci -i $SNAP/usr/share/misc/pci.ids > $PLAINBOX_SESSION_SHARE/lspci_test
+ nmcli -t -f active,BSSID d w l | grep -oP "(?<=^yes:).*" > $PLAINBOX_SESSION_SHARE/wifi_test
+ checkbox-support-lsusb -f $SNAP/checkbox-runtime/var/lib/usbutils/usb.ids -s | sort > $PLAINBOX_SESSION_SHARE/lsusb_test
+ diff -u $PLAINBOX_SESSION_SHARE/lspci_original $PLAINBOX_SESSION_SHARE/lspci_test
+ if [ $? -ne 0 ] ; then
+ echo "lspci mismatch during cycle {reboot_id}"
+ exit 1
+ fi
+ diff -u $PLAINBOX_SESSION_SHARE/wifi_original $PLAINBOX_SESSION_SHARE/wifi_test
+ if [ $? -ne 0 ] ; then
+ echo "wifi mismatch during cycle {reboot_id}"
+ exit 1
+ fi
+ diff -u $PLAINBOX_SESSION_SHARE/lsusb_original $PLAINBOX_SESSION_SHARE/lsusb_test
+ if [ $? -ne 0 ] ; then
+ echo "lsusb mismatch during cycle {reboot_id}"
+ exit 1
+ fi
+user: root
+flags: preserve-locale
+estimated_duration: 1.0
+depends: warm-boot-loop-reboot{reboot_id}
diff --git a/units/stress/s3s4.pxu b/units/stress/s3s4.pxu
new file mode 100644
index 0000000..9a62f8e
--- /dev/null
+++ b/units/stress/s3s4.pxu
@@ -0,0 +1,136 @@
+# Copyright 2016 Canonical Ltd.
+# All rights reserved.
+#
+# Written by:
+# Sylvain Pineau <sylvain.pineau@canonical.com>
+
+unit: category
+id: stress-tests/suspend
+_name: Suspend (S3) Stress Test
+
+unit: category
+id: stress-tests/hibernate
+_name: Hibernate (S4) Stress Test
+
+
+id: stress_s3_iterations
+category_id: stress-tests/suspend
+_description:
+ Resource job meant to be used in the S3 stress templates
+plugin: resource
+environ: STRESS_S3_ITERATIONS
+command:
+ echo "s3_iterations: ${STRESS_S3_ITERATIONS:-30}"
+estimated_duration: 1s
+flags: preserve-locale
+
+id: stress_s4_iterations
+category_id: stress-tests/suspend
+_description:
+ Resource job meant to be used in the S4 stress templates
+plugin: resource
+environ: STRESS_S4_ITERATIONS
+command:
+ echo "s4_iterations: ${STRESS_S4_ITERATIONS:-30}"
+estimated_duration: 1s
+flags: preserve-locale
+
+
+unit: template
+template-resource: stress_s3_iterations
+template-unit: job
+plugin: shell
+flags: preserve-locale
+category_id: stress-tests/suspend
+id: stress-tests/suspend_{s3_iterations}_cycles
+imports:
+ from com.canonical.certification import sleep
+ from com.canonical.certification import rtc
+requires:
+ sleep.mem == 'supported'
+ rtc.state == 'supported'
+estimated_duration: 2400.0
+environ: PLAINBOX_SESSION_SHARE STRESS_S3_SLEEP_DELAY STRESS_S3_WAIT_DELAY LD_LIBRARY_PATH
+user: root
+command:
+ export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$SNAP/usr/lib/fwts"
+ set -o pipefail; checkbox-support-fwts_test -l $PLAINBOX_SESSION_SHARE/suspend_{s3_iterations}_cycles -f none -s s3 --s3-device-check --s3-device-check-delay=${{STRESS_S3_WAIT_DELAY:-45}} --s3-sleep-delay=${{STRESS_S3_SLEEP_DELAY:-30}} --s3-multiple={s3_iterations} -j $SNAP/share/fwts | tee $PLAINBOX_SESSION_SHARE/suspend_{s3_iterations}_cycles_times.log
+_description:
+ PURPOSE:
+ This is an automated stress test that will force the system to suspend/resume for {s3_iterations} cycles.
+
+unit: template
+template-resource: stress_s3_iterations
+template-unit: job
+plugin: shell
+flags: preserve-locale
+category_id: stress-tests/suspend
+id: stress-tests/suspend-{s3_iterations}-cycles-log-check
+after: stress-tests/suspend_{s3_iterations}_cycles
+estimated_duration: 1.0
+command: [ -e $PLAINBOX_SESSION_SHARE/suspend_{s3_iterations}_cycles.log ] && sleep_test_log_check -v s3 $PLAINBOX_SESSION_SHARE/suspend_{s3_iterations}_cycles.log
+_description:
+ Automated check of the {s3_iterations} cycles suspend log for errors detected by fwts.
+
+unit: template
+template-resource: stress_s3_iterations
+template-unit: job
+plugin: attachment
+flags: preserve-locale
+category_id: stress-tests/suspend
+id: stress-tests/suspend-{s3_iterations}-cycles-log-attach
+estimated_duration: 1.0
+after: stress-tests/suspend_{s3_iterations}_cycles
+command: [ -e $PLAINBOX_SESSION_SHARE/suspend_{s3_iterations}_cycles.log ] && cat $PLAINBOX_SESSION_SHARE/suspend_{s3_iterations}_cycles.log
+_description:
+ Attaches the log from the {s3_iterations} cycles Suspend/Resume test if it exists
+
+
+unit: template
+template-resource: stress_s4_iterations
+template-unit: job
+plugin: shell
+flags: preserve-locale
+category_id: stress-tests/hibernate
+id: stress-tests/hibernate_{s4_iterations}_cycles
+imports:
+ from com.canonical.certification import sleep
+ from com.canonical.certification import rtc
+requires:
+ sleep.disk == 'supported'
+ rtc.state == 'supported'
+estimated_duration: 5400.00
+environ: PLAINBOX_SESSION_SHARE STRESS_S4_SLEEP_DELAY STRESS_S4_WAIT_DELAY LD_LIBRARY_PATH
+user: root
+command:
+ export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$SNAP/usr/lib/fwts"
+ checkbox-support-fwts_test -l $PLAINBOX_SESSION_SHARE/hibernate_{s4_iterations}_cycles -f none -s s4 --s4-device-check --s4-device-check-delay=${{STRESS_S4_WAIT_DELAY:-45}} --s4-sleep-delay=${{STRESS_S4_SLEEP_DELAY:-120}} --s4-multiple={s4_iterations} -j $SNAP/share/fwts
+_description:
+ PURPOSE:
+ This is an automated stress test that will force the system to hibernate/resume for {s4_iterations} cycles
+
+unit: template
+template-resource: stress_s4_iterations
+template-unit: job
+plugin: shell
+flags: preserve-locale
+category_id: stress-tests/hibernate
+id: stress-tests/hibernate-{s4_iterations}-cycles-log-check
+after: stress-tests/hibernate_{s4_iterations}_cycles
+estimated_duration: 1.0
+command: [ -e $PLAINBOX_SESSION_SHARE/hibernate_{s4_iterations}_cycles.log ] && sleep_test_log_check -v s4 $PLAINBOX_SESSION_SHARE/hibernate_{s4_iterations}_cycles.log
+_description:
+ Automated check of the {s4_iterations} cycles hibernate log for errors detected by fwts.
+
+unit: template
+template-resource: stress_s4_iterations
+template-unit: job
+plugin: attachment
+flags: preserve-locale
+category_id: stress-tests/hibernate
+id: stress-tests/hibernate-{s4_iterations}-cycles-log-attach
+estimated_duration: 1.0
+after: stress-tests/hibernate_{s4_iterations}_cycles
+command: [ -e $PLAINBOX_SESSION_SHARE/hibernate_{s4_iterations}_cycles.log ] && cat $PLAINBOX_SESSION_SHARE/hibernate_{s4_iterations}_cycles.log
+_description:
+ Attaches the log from the {s4_iterations} cycles Hibernate/Resume test if it exists
diff --git a/units/stress/stress-ng.pxu b/units/stress/stress-ng.pxu
new file mode 100644
index 0000000..9cbe615
--- /dev/null
+++ b/units/stress/stress-ng.pxu
@@ -0,0 +1,31 @@
+unit: job
+id: stress-ng-classes
+plugin: resource
+_summary: Gather list of test classes from stress-ng
+_description:
+ stress-ng divides tests in to classes. Get a list of classes so can divide
+ up tests. Complete class list: cpu, cpu-cache, device, io, interrupt,
+ filesystem, memory, network, os, pipe, scheduler, vm.
+command:
+ command -v stress-ng >/dev/null 2>&1 || { >&2 echo "stress-ng command not found"; exit 1; }
+ CLASSES="cpu cpu-cache memory os pipe scheduler vm"
+ for c in $CLASSES; do
+ echo "stress-ng-class: $c"
+ echo
+ done
+estimated_duration: 1s
+flags: preserve-locale
+
+unit: template
+template-resource: stress-ng-classes
+template-unit: job
+id: stress/stress-ng-test-for-class-{stress-ng-class}
+category_id: stress-tests
+_summary: Run the stress-ng stressors for class {stress-ng-class}
+_description:
+ Runs the stressors for the class defined. Tests run sequentially using the
+ same number of proccesses as online processors.
+plugin: shell
+estimated_duration: 1200.0
+command:
+ stress-ng --sequential 0 --class {stress-ng-class}
diff --git a/units/stress/test-plan.pxu b/units/stress/test-plan.pxu
index 8e7929a..96266e4 100644
--- a/units/stress/test-plan.pxu
+++ b/units/stress/test-plan.pxu
@@ -88,3 +88,151 @@ include:
 stress/cpu_stress_ng_test certification-status=blocker
 memory/memory_stress_ng certification-status=blocker
 
+id: stress-full
+unit: test plan
+_name: Stress tests
+_description: QA stress tests for Snappy Ubuntu Core devices
+include:
+nested_part:
+ stress-automated
+
+id: stress-automated
+unit: test plan
+_name: Automated stress tests
+_description: Automated stress tests for Snappy Ubuntu Core devices
+include:
+nested_part:
+ warm-boot-stress-test
+ cold-boot-stress-test
+ suspend-stress-test
+ hibernate-stress-test
+ stress-ng-automated
+
+unit: test plan
+id: warm-boot-stress-test
+_name: Warm boot stress test
+_description:
+ Reboots the machine a pre-defined number of times and on
+ resumption of OS performs a hardware check to ensure all
+ items are still present. Reboot is immediate.
+estimated_duration: 25h
+bootstrap_include:
+ reboot-run-generator
+include:
+ warm-boot-loop-.*
+mandatory_include:
+ com.canonical.plainbox::manifest
+ package
+ snap
+ uname
+ lsb
+ cpuinfo
+ dpkg
+ dmi_attachment
+ sysfs_attachment
+ udev_attachment
+ lspci_attachment
+ lsusb_attachment
+ dmi
+ meminfo
+ interface
+
+unit: test plan
+id: cold-boot-stress-test
+_name: Cold boot stress test
+_description:
+ Reboots the machine a pre-defined number of times and on
+ resumption of OS performs a hardware check to ensure all
+ items are still present. The reboot is delayed by 2 minutes
+ by the RTC to allow hardware to cool.
+estimated_duration: 42h
+bootstrap_include:
+ reboot-run-generator
+include:
+ cold-boot-loop-.*
+mandatory_include:
+ com.canonical.plainbox::manifest
+ package
+ snap
+ uname
+ lsb
+ cpuinfo
+ dpkg
+ dmi_attachment
+ sysfs_attachment
+ udev_attachment
+ lspci_attachment
+ lsusb_attachment
+ dmi
+ meminfo
+ interface
+
+
+unit: test plan
+id: suspend-stress-test
+_name: Suspend (S3) stress test
+_description:
+ Suspends the machine a pre-defined number of times and on
+ resume of OS performs a hardware check to ensure all
+ items are still present.
+estimated_duration: 42h
+bootstrap_include:
+ stress_s3_iterations
+include:
+ stress-tests/suspend.*
+mandatory_include:
+ com.canonical.plainbox::manifest
+ package
+ snap
+ uname
+ lsb
+ cpuinfo
+ dpkg
+ dmi_attachment
+ sysfs_attachment
+ udev_attachment
+ lspci_attachment
+ lsusb_attachment
+ dmi
+ meminfo
+ interface
+
+unit: test plan
+id: hibernate-stress-test
+_name: Hibernate (S4) stress test
+_description:
+ Hibernates the machine a pre-defined number of times and on
+ resume of OS performs a hardware check to ensure all
+ items are still present.
+estimated_duration: 42h
+bootstrap_include:
+ stress_s4_iterations
+include:
+ stress-tests/hibernate.*
+mandatory_include:
+ com.canonical.plainbox::manifest
+ package
+ snap
+ uname
+ lsb
+ cpuinfo
+ dpkg
+ dmi_attachment
+ sysfs_attachment
+ udev_attachment
+ lspci_attachment
+ lsusb_attachment
+ dmi
+ meminfo
+ interface
+
+id: stress-ng-automated
+unit: test plan
+_name: Automated stress-ng tests
+_description: Automated stress-ng tests for Snappy Ubuntu Core devices
+include:
+ stress/stress-ng-test-for-class-.*
+ disk/disk_stress_ng_.*
+bootstrap_include:
+ device
+ stress-ng-classes