5 files changed, 427 insertions, 3 deletions
diff --git a/bin/check-prerelease b/bin/check-prerelease
new file mode 100755
index 0000000..1f3d017
--- /dev/null
+++ b/bin/check-prerelease
@@ -0,0 +1,137 @@
+#!/usr/bin/env python3
+
+"""
+Script to test that the system is NOT running prerelease software
+
+Copyright (c) 2018 Canonical Ltd.
+
+Authors
+ Rod Smith <rod.smith@canonical.com>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License version 3,
+as published by the Free Software Foundation.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+The purpose of this script is to identify whether an EFI-based
+system booted from the network (test passes) or from a local disk
+(test fails).
+
+Usage:
+ check-prerelease
+"""
+
+import platform
+import shlex
+import sys
+
+from subprocess import Popen, PIPE
+
+
+def check_kernel_status():
+ """Check kernel to see if it's supported for certification
+
+ :returns:
+ True if OK, False if not
+ """
+ kernel_release = platform.release()
+
+ retval = False
+ command = "apt-cache show linux-image-{}".format(kernel_release)
+ aptinfo = []
+ aptinfo_bytes = (Popen(shlex.split(command), stdout=PIPE)
+ .communicate()[0])
+ aptinfo = (aptinfo_bytes.decode(encoding="utf-8", errors="ignore")
+ .splitlines())
+
+ """Kernel apt-cache info includes a 'Supported:' line on release
+ kernels to identify the period of support. This line is missing on
+ pre-release kernels. Thus, we return a True value only if this
+ line is present and shows a 5-year support period."""
+ if any("Supported: 5y" in s for s in aptinfo):
+ retval = True
+ if any("Supported: 9m" in s for s in aptinfo):
+ print("* Kernel is supported for 9 months; it is an interim release!")
+ if not any("Supported:" in s for s in aptinfo):
+ print("* Could not find a kernel support period; "
+ "may be a pre-release kernel!")
+
+ """We also want to exclude 'edge' kernels, which are identified via
+ the 'Source:' line in the apt-cache output...."""
+ if any("Source: linux-signed-hwe-edge" in s for s in aptinfo):
+ print("* Kernel is an 'edge' kernel!")
+ retval = False
+ if any("Source: linux-hwe-edge" in s for s in aptinfo):
+ print("* Kernel is an 'edge' kernel!")
+ retval = False
+
+ """We also want to exclude low-latency kernels, which are identified
+ via the kernel name string itself...."""
+ if "lowlatency" in kernel_release:
+ print("* Kernel is a low-latency kernel!")
+ retval = False
+
+ if (not retval):
+ print("* Kernel release is {}".format(kernel_release))
+ print("* Kernel is ineligible for certification!")
+
+ return retval
+
+
+def check_os_status():
+ """Check OS to see if it's supported for certification. Note that this
+ passes any release version (even a non-LTS version), but not pre-release
+ versions.
+
+ :returns:
+ True if OK, False if not
+ """
+ retval = True
+ command = "lsb_release -s -d"
+ lsbinfo = []
+ lsbinfo_bytes = (Popen(shlex.split(command), stdout=PIPE)
+ .communicate()[0])
+ lsbinfo = (lsbinfo_bytes.decode(encoding="utf-8", errors="ignore")
+ .rstrip())
+
+ """OS information include '(development branch)' on pre-release
+ installations. Such installations should fail this test."""
+ if "(development branch)" in lsbinfo:
+ print("* 'lsb_release -s -d' result is '{}'".format(lsbinfo))
+ print("* OS is reported as a development branch:")
+ print("* {}".format(lsbinfo))
+ retval = False
+ print("")
+
+ return retval
+
+
+def main():
+ """Check to see if the machine is running pre-release kernel or OS."""
+
+ retval = 0
+ if (not check_kernel_status()):
+ retval = 1
+ if (not check_os_status()):
+ retval += 2
+ if (retval == 0):
+ print("** All OK; production kernel and OS.")
+ elif (retval == 1):
+ print("** Test FAILS; running ineligible kernel!")
+ elif (retval == 2):
+ print("** Test FAILS; running pre-release OS!")
+ else:
+ print("** Test FAILS; running pre-release OS with ineligible kernel!")
+
+ return(retval)
+
+
+if __name__ == '__main__':
+ sys.exit(main())
diff --git a/units/ethernet/jobs.pxu b/units/ethernet/jobs.pxu
index 90e0edf..c6e679a 100644
--- a/units/ethernet/jobs.pxu
+++ b/units/ethernet/jobs.pxu
@@ -129,3 +129,275 @@ requires:
 command: network test -i {interface} -t stress
 _description:
 Automated test that tests performance of ethernet device {__index__} ({interface}).
+
+unit: template
+template-resource: device
+template-filter: device.category == 'NETWORK' and device.interface != 'UNKNOWN'
+id: ethernet/ping_{interface}
+_summary: Can ping another machine over Ethernet port {interface}
+_description: Check Ethernet works by pinging another machine
+plugin: shell
+command:
+ gateway_ping_test -v --interface {interface}
+category_id: com.canonical.plainbox::ethernet
+estimated_duration: 4.0
+flags: preserve-locale also-after-suspend
+
+unit: template
+template-resource: device
+template-filter: device.category == 'NETWORK' and device.mac != 'UNKNOWN'
+id: ethernet/wol_S5_{interface}
+_summary: Wake on LAN (WOL) test from S5 - {interface}
+_purpose:
+ Check that another system can wake up from S5 the SUT using ethernet port {interface} WOL function.
+_steps:
+ 1. Ensure there is an ethernet cable attached to port {interface}.
+ 2. Press Enter for S5 (Soft Off).
+ 3. From another computer on the same network run the following command:
+ $ wakeonlan {mac}
+ If wakeonlan tool is not installed run:
+ $ sudo apt install wakeonlan
+ 4. Resume Checkbox
+_verification:
+ Did the SUT wake up from S5?
+plugin: user-interact-verify
+command: poweroff
+user: root
+category_id: com.canonical.plainbox::ethernet
+estimated_duration: 120
+flags: preserve-locale
+
+unit: template
+template-resource: device
+template-filter: device.category == 'NETWORK' and device.mac != 'UNKNOWN'
+id: ethernet/wol_S4_{interface}
+_summary: Wake on LAN (WOL) test from S4 - {interface}
+_purpose:
+ Check that another system can wake up from S4 the SUT using ethernet port {interface} WOL function.
+_steps:
+ 1. Ensure there is an ethernet cable attached to port {interface}.
+ 2. Press Enter to hibernate the system.
+ 3. From another computer on the same network run the following command:
+ $ wakeonlan {mac}
+ If wakeonlan tool is not installed run:
+ $ sudo apt install wakeonlan
+_verification:
+ Did the SUT wake up from S4?
+plugin: user-interact-verify
+requires:
+ sleep.disk == 'supported'
+command: systemctl hibernate
+user: root
+category_id: com.canonical.plainbox::ethernet
+estimated_duration: 120
+flags: preserve-locale
+
+unit: template
+template-resource: device
+template-filter: device.category == 'NETWORK' and device.mac != 'UNKNOWN' and device.interface != 'UNKNOWN'
+id: ethernet/wol_S3_{interface}
+_summary: Wake on LAN (WOL) test from S3 - {interface}
+_purpose:
+ Check that another system can wake up from S3 the SUT using ethernet port {interface} WOL function.
+_steps:
+ 1. Ensure there is an ethernet cable attached to port {interface}.
+ 2. Press Enter to suspend the system.
+ 3. From another computer on the same network run the following command:
+ $ wakeonlan {mac}
+ If wakeonlan tool is not installed run:
+ $ sudo apt install wakeonlan
+_verification:
+ Did the SUT wake up from S3?
+plugin: user-interact-verify
+requires:
+ sleep.mem == 'supported'
+command: systemctl suspend
+user: root
+category_id: com.canonical.plainbox::ethernet
+estimated_duration: 120
+flags: preserve-locale
+
+unit: template
+template-resource: device
+template-filter: device.category == 'NETWORK' and device.interface != 'UNKNOWN'
+template-engine: jinja2
+id: ethernet/hotplug-{{ interface }}-check-disconnected
+_summary: Ensure ethernet port {{ interface }} is physically disconnected
+_purpose:
+ Check that ethernet port {{ interface }} is detected as being in the correct state
+ to begin a hotplug connection test.
+_steps:
+ 1. Ensure there is no ethernet cable attached to port {{ interface }}.
+ 2. Begin test.
+plugin: user-interact
+command:
+ if [ $(cat /sys/class/net/{{ interface }}/carrier) -ne 0 ]; then
+ # carrier indicating cable present
+ exit 1
+ fi
+ exit 0
+category_id: com.canonical.plainbox::ethernet
+estimated_duration: 1.0
+flags: preserve-locale
+after:
+ {%- if __index__ > 1 %}ethernet/hotplug-{{ __index__ - 1 }}-end-cycle{%- endif %}
+
+unit: template
+template-resource: device
+template-filter: device.category == 'NETWORK' and device.interface != 'UNKNOWN'
+id: ethernet/hotplug-{interface}-connect
+_summary: Ethernet port {interface} hotplug detection test
+_purpose:
+ Check ethernet port {interface} connects when cable inserted. Assumes an IP
+ address will be assigned by DHCP. Connection asserted by pinging the network
+ defined gateway.
+_steps:
+ 1. Begin the test.
+ 2. Insert the ethernet cable in to ethernet port {interface}.
+ 3. This test will timeout and fail if the insertion and connection
+ establishment has not been detected (10 second timeout for each check).
+plugin: user-interact
+command:
+ LOOP=0
+ CABLE_DETECT=0
+ while [ $LOOP -lt 10 ] && [ $CABLE_DETECT -eq 0 ]
+ do
+ sleep 1
+ CABLE_DETECT=$(cat /sys/class/net/{interface}/carrier)
+ LOOP=$((LOOP=LOOP+1))
+ done
+ if [ $CABLE_DETECT -eq 0 ]; then
+ echo "Didn't detect a cable insertion"
+ exit 1
+ fi
+ echo "Detected a cable insertion"
+ LOOP=0
+ OPSTATE="unknown"
+ while [ $LOOP -lt 10 ] && [ "$OPSTATE" != "up" ]
+ do
+ sleep 1
+ OPSTATE=$(cat /sys/class/net/{interface}/operstate)
+ LOOP=$((LOOP=LOOP+1))
+ done
+ if [ "$OPSTATE" != "up" ]; then
+ echo "Interface did not up"
+ exit 1
+ fi
+ echo "Interface up"
+ gateway_ping_test -v --interface {interface}
+ PING_TEST=$?
+ if [ $PING_TEST -ne 0 ]; then
+ echo "Ping test failed"
+ exit 1
+ fi
+ exit 0
+category_id: com.canonical.plainbox::ethernet
+estimated_duration: 60.0
+depends: ethernet/hotplug-{interface}-check-disconnected
+
+unit: template
+template-resource: device
+template-filter: device.category == 'NETWORK' and device.interface != 'UNKNOWN'
+id: ethernet/hotplug-{interface}-disconnect
+_summary: Ethernet port {interface} hotplug disconnect step
+_purpose:
+ Check that when cable removed from ethernet port {interface} the system
+ detects this correctly.
+_steps:
+ 1. Depends on previous hotplug connection test passing. We will now test
+ cable disconnection.
+ 2. Begin the test.
+ 3. Remove the ethernet cable from ethernet port {interface}.
+ 4. This test will timeout and fail if the removal has not been detected and
+ interface marked as down (10 second timeout for each check).
+plugin: user-interact
+command:
+ LOOP=0
+ CABLE_DETECT=1
+ while [ $LOOP -lt 10 ] && [ $CABLE_DETECT -ne 0 ]
+ do
+ sleep 1
+ CABLE_DETECT=$(cat /sys/class/net/{interface}/carrier)
+ LOOP=$((LOOP=LOOP+1))
+ done
+ if [ $CABLE_DETECT -ne 0 ]; then
+ echo "Didn't detect a cable removal"
+ exit 1
+ fi
+ echo "Detected a cable removal"
+ LOOP=0
+ OPSTATE="up"
+ while [ $LOOP -lt 10 ] && [ "$OPSTATE" == "up" ]
+ do
+ sleep 1
+ OPSTATE=$(cat /sys/class/net/{interface}/operstate)
+ LOOP=$((LOOP=LOOP+1))
+ done
+ if [ "$OPSTATE" == "up" ]; then
+ echo "Interface did not go down"
+ exit 1
+ fi
+ echo "Interface went down"
+ exit 0
+category_id: com.canonical.plainbox::ethernet
+estimated_duration: 20.0
+depends: ethernet/hotplug-{interface}-connect
+
+unit: template
+template-resource: device
+template-filter: device.category == 'NETWORK' and device.interface != 'UNKNOWN'
+id: ethernet/hotplug-{__index__}-end-cycle
+after: ethernet/hotplug-{interface}-disconnect
+command: true
+flags: simple
+
+unit: template
+template-resource: device
+template-filter: device.category == 'NETWORK' and device.interface != 'UNKNOWN'
+template-unit: job
+id: ethernet/iperf3_{interface}
+plugin: shell
+_summary: Iperf3 stress testing for {interface}
+category_id: com.canonical.plainbox::ethernet
+estimated_duration: 740.0
+user: root
+environ:
+ TEST_TARGET_IPERF
+ LD_LIBRARY_PATH
+command: checkbox-support-network test -i {interface} -t iperf --iperf3 --scan-timeout 3600 --fail-threshold 80 --cpu-load-fail-threshold 90 --runtime 90 --num_runs 4
+_description:
+ This test uses iperf3 to ensure network devices pass data at an acceptable
+ minimum percentage of advertized speed.
+
+unit: template
+template-resource: device
+template-filter: device.category == 'NETWORK' and device.interface != 'UNKNOWN'
+template-engine: jinja2
+template-unit: job
+id: ethernet/check-{{ interface }}-static-configuration
+_summary: Check that {{ interface }} has a static configuration
+_description:
+ This job is intended to be used on particular devices where an interface has
+ been identified for configuration with a static IP address. As such, this is
+ not suitable for inclusion in the ethernet nested test plans, but should be
+ specifically included where required.
+category_id: com.canonical.plainbox::ethernet
+plugin: shell
+estimated_duration: 2s
+command: check_static {{ interface }}
+requires:
+ snap.name == 'network-manager'
+ connections.slot == 'network-manager:service' and connections.plug == '{{ __system_env__["SNAP_NAME"] }}:network-manager'
+
+plugin: shell
+category_id: com.canonical.plainbox::ethernet
+id: after-suspend-ethernet/network_resume_time_auto
+depends: suspend/suspend_advanced_auto
+estimated_duration: 1.2
+requires:
+ device.category == 'NETWORK'
+command: network_reconnect_resume_test -t 10 -d wired
+_summary: Network reconnect resume test (wired)
+_description:
+ Checks the length of time it takes to reconnect an existing wired connection
+ after a suspend/resume cycle.
diff --git a/units/miscellanea/jobs.pxu b/units/miscellanea/jobs.pxu
index 30c8a16..4a9980b 100644
--- a/units/miscellanea/jobs.pxu
+++ b/units/miscellanea/jobs.pxu
@@ -161,6 +161,16 @@ command: efi-pxeboot
 
 plugin: shell
 category_id: com.canonical.plainbox::miscellanea
+estimated_duration: 0.5
+id: miscellanea/check_prerelease
+_summary: Test that system is not a pre-release version
+_description:
+ Test to verify that the system uses production, rather
+ than pre-release, versions of the kernel and the OS.
+command: check-prerelease
+
+plugin: shell
+category_id: com.canonical.plainbox::miscellanea
 id: miscellanea/bmc_info
 requires:
 package.name == 'ipmitool' or executable.name == 'ipmitool'
diff --git a/units/networking/test-plan.pxu b/units/networking/test-plan.pxu
index e9e61e2..d6a3ddb 100644
--- a/units/networking/test-plan.pxu
+++ b/units/networking/test-plan.pxu
@@ -13,6 +13,9 @@ _name: Networking tests (manual)
 _description: Networking tests (manual)
 include:
 networking/info_device.* certification-status=blocker
+ ethernet/hotplug-.* certification-status=blocker
+bootstrap_include:
+ device
 
 id: networking-cert-automated
 unit: test plan
@@ -32,3 +35,5 @@ include:
 networking/gateway_ping certification-status=blocker
 networking/info_device.* certification-status=blocker
 networking/ntp certification-status=blocker
+bootstrap_include:
+ device
diff --git a/units/suspend/suspend.pxu b/units/suspend/suspend.pxu
index c989f77..114d7d0 100644
--- a/units/suspend/suspend.pxu
+++ b/units/suspend/suspend.pxu
@@ -61,7 +61,7 @@ requires:
 package.name == 'iperf'
 user: root
 environ: TEST_TARGET_FTP TEST_TARGET_IPERF TEST_USER TEST_PASS
-command: network -i {interface} -t iperf
+command: network test -i {interface} -t iperf
 _description:
 This test executes iperf connection performance/stability against device {__index__} ({interface}) before suspend.
 
@@ -615,7 +615,7 @@ requires:
 package.name == 'iperf'
 user: root
 environ: TEST_TARGET_FTP TEST_TARGET_IPERF TEST_USER TEST_PASS
-command: network -i {interface} -t iperf
+command: network test -i {interface} -t iperf
 _description:
 This test executes iperf connection performance/stability against device {__index__} ({interface}) after suspend.
 
@@ -631,7 +631,7 @@ requires:
 package.name == 'iperf'
 user: root
 environ: TEST_TARGET_FTP TEST_TARGET_IPERF TEST_USER TEST_PASS
-command: network -i {interface} -t iperf
+command: network test -i {interface} -t iperf
 _description:
 This test executes iperf connection performance/stability against device {__index__} ({interface}) after suspend.