15 files changed, 235 insertions, 33 deletions
diff --git a/bin/roundtrip_qr.py b/bin/roundtrip_qr.py
new file mode 100755
index 00000000..e2eb5b50
--- /dev/null
+++ b/bin/roundtrip_qr.py
@@ -0,0 +1,121 @@
+#!/usr/bin/env python3
+# This file is part of Checkbox.
+#
+# Copyright 2020 Canonical Ltd.
+# Written by:
+# Jonathan Cave <jonathan.cave@canonical.com>
+#
+# Checkbox is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3,
+# as published by the Free Software Foundation.
+#
+# Checkbox is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Checkbox. If not, see <http://www.gnu.org/licenses/>.
+
+
+import os
+import random
+import string
+import time
+import subprocess as sp
+import sys
+
+import pyqrcode
+import zbar
+from PIL import Image
+
+
+def capture_rpi(name):
+ import picamera
+ file = os.path.join(os.path.expandvars(
+ '$PLAINBOX_SESSION_SHARE'), '{}_qrcapture.png'.format(name))
+ with picamera.PiCamera() as camera:
+ time.sleep(2)
+ camera.capture(file)
+ return file
+
+
+def capture_webcam(name):
+ file = os.path.join(os.path.expandvars(
+ '$PLAINBOX_SESSION_SHARE'), '{}_qrcapture.jpg'.format(name))
+ cmd = ('gst-launch-1.0 v4l2src device=/dev/{} num-buffers=1 ! jpegenc !'
+ 'filesink location={}').format(name, file)
+ try:
+ sp.check_call(cmd, shell=True)
+ except (sp.CalledProcessError, OSError) as e:
+ print(e)
+ raise SystemExit('ERROR: failed to capture image')
+ return file
+
+
+def generate_data():
+ return ''.join(random.choice(string.ascii_letters) for i in range(20))
+
+
+def generate_qr_code(data):
+ return pyqrcode.create(data)
+
+
+def display_code(qr):
+ with open('/dev/tty1', 'wb+', buffering=0) as term:
+ # clear the tty so the qr is always printed at the top of the sceen
+ term.write(str.encode('\033c'))
+ # print the qr code
+ term.write(qr.terminal(quiet_zone=5).encode())
+
+
+def decode_image(filename):
+ scanner = zbar.ImageScanner()
+ scanner.parse_config('enable')
+ pil = Image.open(filename).convert('L')
+ width, height = pil.size
+ raw = pil.tobytes()
+ image = zbar.Image(width, height, 'Y800', raw)
+ scanner.scan(image)
+ result = None
+ for code in image:
+ result = code.data
+ del(image)
+ if result is None:
+ raise SystemExit('ERROR: no qrcodes decoded')
+ return result
+
+
+def main():
+ if len(sys.argv) != 2:
+ raise SystemExit('ERROR: expected a device name')
+ name = sys.argv[1]
+ print('Testing device name: {}\n'.format(name))
+
+ test_str = generate_data()
+ print('Input string: {}'.format(test_str), flush=True)
+
+ print('Generating QR code...', flush=True)
+ qr = generate_qr_code(test_str)
+
+ print('Displaying on screen', flush=True)
+ display_code(qr)
+
+ print('Capture image of screen', flush=True)
+ if name == 'vchiq':
+ file = capture_rpi(name)
+ else:
+ file = capture_webcam(name)
+ print('Image {} captured'.format(file))
+
+ print('Decoding image file', flush=True)
+ result = decode_image(file)
+ print('Decoded data: {}'.format(result))
+
+ if result != test_str:
+ raise SystemExit('FAIL: decoded data does not match input')
+ print('PASS: decoded data and input match')
+
+
+if __name__ == '__main__':
+ main()
diff --git a/bin/stress_ng_test b/bin/stress_ng_test
index ced84553..1bc384a3 100755
--- a/bin/stress_ng_test
+++ b/bin/stress_ng_test
@@ -123,9 +123,9 @@ def stress_cpu(args):
 """Run stress-ng tests on CPUs."""
 
 retval = 0
- stressors = ['bsearch', 'context', 'cpu', 'crypt', 'hsearch', 'longjmp',
- 'lsearch', 'matrix', 'qsort', 'str', 'stream', 'tsearch',
- 'vecmath', 'wcs']
+ stressors = ['af-alg', 'bsearch', 'context', 'cpu', 'crypt', 'hsearch',
+ 'longjmp', 'lsearch', 'matrix', 'qsort', 'str', 'stream',
+ 'tsearch', 'vecmath', 'wcs']
 # Add 10% to runtime; will forcefully terminate if stress-ng
 # fails to return in that time.
 end_time = args.base_time * 11 / 10
diff --git a/bin/virtualization b/bin/virtualization
index 487fba58..2b7cd543 100755
--- a/bin/virtualization
+++ b/bin/virtualization
@@ -27,7 +27,6 @@ from argparse import ArgumentParser
 import os
 import logging
 import lsb_release
-import platform
 import requests
 import shlex
 from subprocess import (
@@ -224,7 +223,7 @@ class KVMTest(object):
 if self.qemu_config['cloudimg_type'] == CLOUD_IMAGE_TYPE_TAR:
 cloud_iso = "%s-server-cloudimg-%s.tar.gz" % (
 self.release, self.qemu_config['cloudimg_arch'])
- elif alt_pattern is "modern":
+ elif alt_pattern == "modern":
 # LP 1635345 - yakkety and beyond have a new naming scheme
 cloud_iso = "%s-server-cloudimg-%s.img" % (
 self.release, self.qemu_config['cloudimg_arch'])
@@ -253,7 +252,7 @@ class KVMTest(object):
 logging.error(" * Message: {}".format(e.with_traceback(None)))
 return False
 
- if ret.status_code is not 200:
+ if ret.status_code != 200:
 return False
 else:
 return True
@@ -561,7 +560,7 @@ class LXDTest(object):
 self.name = 'testbed'
 self.image_alias = uuid4().hex
 self.default_remote = "ubuntu:"
- self.os_version = platform.linux_distribution()[1]
+ self.os_version = lsb_release.get_distro_information()["RELEASE"]
 
 def run_command(self, cmd):
 task = RunCommand(cmd)
diff --git a/bin/xrandr_cycle b/bin/xrandr_cycle
index 34f953c9..6743f6f2 100755
--- a/bin/xrandr_cycle
+++ b/bin/xrandr_cycle
@@ -33,7 +33,8 @@ failure_messages = [] # remember which modes failed
 success_messages = [] # remember which modes succeeded
 
 # Run xrandr and ask it what devices and modes are supported
-xrandrinfo = subprocess.Popen('xrandr -q', shell=True, stdout=subprocess.PIPE)
+xrandrinfo = subprocess.Popen(
+ 'xrandr -q --verbose', shell=True, stdout=subprocess.PIPE)
 output = xrandrinfo.communicate()[0].decode().split('\n')
 
 
@@ -62,12 +63,20 @@ for line in output:
 elif device_context != '': # we've previously seen a 'connected' dev
 # mode names seem to always be of the format [horiz]x[vert]
 # (there can be non-mode information inside of a device context!)
- if foo[0].find('x') != -1:
- modes.append((device_context, foo[0]))
+ x_pos = foo[0].find('x')
+ # for a resolution there has to be at least 3 chars:
+ # a digit, an x, and a digit
+ if x_pos > 0 and x_pos < len(foo[0]) - 1:
+ if 'DoubleScan' in foo:
+ # xrandr lists DoubleScan resolutions but cannot set them
+ # so for the purposes of this test let's not use them
+ continue
+ if foo[0][x_pos-1].isdigit() and foo[0][x_pos+1].isdigit():
+ modes.append((device_context, foo[0]))
 # we also want to remember what the current mode is, which xrandr
 # marks with a '*' character, so we can set things back the way
 # we found them at the end:
- if foo[1].find('*') != -1:
+ if "*current" in foo:
 current_modes.append((device_context, foo[0]))
 # let's create a dict of aspect_ratio:largest_width for each display
 # (width, because it's easier to compare simple ints when looking for the
diff --git a/units/camera/jobs.pxu b/units/camera/jobs.pxu
index 331f2db7..c9f24517 100644
--- a/units/camera/jobs.pxu
+++ b/units/camera/jobs.pxu
@@ -131,4 +131,33 @@ command:
 _description:
 Takes multiple pictures based on the resolutions supported by the camera and
 validates their size and that they are of a valid format.
-user: root \ No newline at end of file
+user: root
+
+unit: template
+template-engine: jinja2
+template-resource: device
+template-filter: device.category in ('CAPTURE', 'MMAL') and device.name != ''
+template-unit: job
+plugin: shell
+category_id: com.canonical.plainbox::camera
+id: camera/roundtrip-qrcode_{{ name }}
+_summary: Test video output and camera {{ name }} by displaying and reading a qrcode
+estimated_duration: 5.0
+requires:
+ {%- if __on_ubuntucore__ %}
+ lsb.release >= '19.1'
+ {%- else %}
+ lsb.release >= '19.1'
+ package.name == 'python3-zbar'
+ package.name == 'python3-pyqrcode'
+ package.name == 'python3-pil'
+ {% endif -%}
+command:
+ roundtrip_qr.py {{ name }}
+_description:
+ Generates a QRcode representing a random string of ascii letters. This is
+ written to tty1 using ascii escape codes. Either the PiCamera python module or
+ a gstreamer pipeline is used to capture an image of the display. An attempt
+ to decode a QRcode in the image is then made and data compared against the
+ random string.
+user: root
diff --git a/units/camera/packaging.pxu b/units/camera/packaging.pxu
new file mode 100644
index 00000000..0b90c611
--- /dev/null
+++ b/units/camera/packaging.pxu
@@ -0,0 +1,15 @@
+# For camera/roundtrip-qrcode_.*
+unit: packaging meta-data
+os-id: ubuntu
+Depends: python3-zbar
+
+# For camera/roundtrip-qrcode_.*
+unit: packaging meta-data
+os-id: ubuntu
+Depends: python3-pil
+
+# For camera/roundtrip-qrcode_.*
+unit: packaging meta-data
+os-id: ubuntu
+Depends: python3-pyqrcode
+
diff --git a/units/camera/test-plan.pxu b/units/camera/test-plan.pxu
index c3b8378b..d69706c5 100644
--- a/units/camera/test-plan.pxu
+++ b/units/camera/test-plan.pxu
@@ -84,5 +84,6 @@ estimated_duration: 1h30m
 include:
 camera/multiple-resolution-images-_.*
 camera/multiple-resolution-images-rpi_.*
+ camera/roundtrip-qrcode_.*
 bootstrap_include:
 device
diff --git a/units/disk/encryption.pxu b/units/disk/encryption.pxu
index b805a37e..d4c5df8c 100644
--- a/units/disk/encryption.pxu
+++ b/units/disk/encryption.pxu
@@ -37,12 +37,14 @@ _purpose:
 BIOS), the device won't be able to get the master key and decrypt its data
 partition.
 _steps:
+ NOTE!!!! YOU'LL HAVE TO RE-INSTALL THE IMAGE AFTER THIS TEST
 1. Install the image and make sure it boots and you can log in.
- 2. Turn the device off and upgrade/downgrade the BIOS or modify Secure Boot state
- 3. Make sure the BIOS is set up properly (e.g. TPM enabled, UEFI boot mode)
- 4. Start the device
+ 2. Make sure the BIOS is set up properly (e.g. TPM enabled, UEFI boot mode)
+ 3. Based on your project's situation, choose one of the following methods to clean TPM:
+ a. Turn the device off and upgrade/downgrade the BIOS or modify Secure Boot state
+ b. Clean TPM via BIOS menu
+ c. Install checkbox, execute "checkbox-[project name].checkbox-cli run com.canonical.certification::tpm2.0_3.0.4/tpm2_takeownership"
+ 4. Start or reboot the device
 _verification:
 Mark this test as "Passed" if the device cannot boot anymore.
- Note: You must flash the BIOS back to the latest version, re-enable Secure Boot
- and re-install the image afterwards.
 
diff --git a/units/firmware/jobs.pxu b/units/firmware/jobs.pxu
index 9ebc7346..0353fbf1 100644
--- a/units/firmware/jobs.pxu
+++ b/units/firmware/jobs.pxu
@@ -77,7 +77,7 @@ _description:
 _summary: Run FWTS Server Cert selected tests.
 environ: PLAINBOX_SESSION_SHARE
 command:
- checkbox-support-fwts_test --server -q -l $PLAINBOX_SESSION_SHARE/fwts_server_results.log
+ checkbox-support-fwts_test --server -q -f critical -l $PLAINBOX_SESSION_SHARE/fwts_server_results.log
 
 plugin: attachment
 category_id: com.canonical.plainbox::firmware
diff --git a/units/miscellanea/jobs.pxu b/units/miscellanea/jobs.pxu
index ebc7c4de..d7222950 100644
--- a/units/miscellanea/jobs.pxu
+++ b/units/miscellanea/jobs.pxu
@@ -77,20 +77,6 @@ environ: PLAINBOX_SESSION_SHARE
 command:
 checkbox-support-fwts_test -l $PLAINBOX_SESSION_SHARE/fwts_results.log
 
-plugin: shell
-category_id: com.canonical.plainbox::miscellanea
-id: miscellanea/fwupdate
-estimated_duration: 1.0
-depends: miscellanea/efi_boot_mode
-requires:
- cpuinfo.platform in ("i386", "x86_64", "aarch64", "armv7l")
- package.name == 'fwupdate' or executable.name == 'fwupdate'
-_description:
- Determine if EFI firmware supports update from OS.
-_summary:
- Check for firmware update support.
-command: fwupdate -s
-
 plugin: attachment
 category_id: com.canonical.plainbox::miscellanea
 id: miscellanea/fwts_results.log
diff --git a/units/miscellanea/test-plan.pxu b/units/miscellanea/test-plan.pxu
index b2027279..5b23880e 100644
--- a/units/miscellanea/test-plan.pxu
+++ b/units/miscellanea/test-plan.pxu
@@ -71,7 +71,6 @@ mandatory_include:
 miscellanea/cpus_are_not_samples
 miscellanea/ipmi_test certification-status=blocker
 miscellanea/bmc_info
- miscellanea/fwupdate
 miscellanea/dmitest_server
 miscellanea/sosreport
 miscellanea/sosreport_attachment
diff --git a/units/nvdimm/category.pxu b/units/nvdimm/category.pxu
new file mode 100644
index 00000000..84d724e7
--- /dev/null
+++ b/units/nvdimm/category.pxu
@@ -0,0 +1,3 @@
+unit: category
+id: nvdimm
+_name: NVDIMM device tests
diff --git a/units/nvdimm/jobs.pxu b/units/nvdimm/jobs.pxu
new file mode 100644
index 00000000..6db1b668
--- /dev/null
+++ b/units/nvdimm/jobs.pxu
@@ -0,0 +1,27 @@
+plugin: shell
+category_id: com.canonical.certification::nvdimm
+id: nvdimm/info
+estimated_duration: 1.0 
+user: root
+requires: 
+ executable.name == "ipmctl"
+ lsb.release >= "18.04"
+command: ipmctl show -dimm
+_summary:
+ Verify that NVDIMMs are discovered
+_description:
+ This test will probe any installed NVDIMMs and list them and their capacities.
+
+plugin: shell
+category_id: com.canonical.certification::nvdimm
+id: nvdimm/health
+estimated_duration: 1.0
+user: root
+requires: 
+ executable.name == "ipmctl"
+ lsb.release >= "18.04"
+command: ipmctl show -d CurrentValue,CurrentState -sensor Health
+_summary:
+ Report health state of installed NVDIMM devices
+_description:
+ This test will do a quick health check of installed NVDIMM devices.
diff --git a/units/nvdimm/packaging.pxu b/units/nvdimm/packaging.pxu
new file mode 100644
index 00000000..8b048c57
--- /dev/null
+++ b/units/nvdimm/packaging.pxu
@@ -0,0 +1,4 @@
+# nvdimms require ipmctl for health check jobs
+unit: packaging meta-data
+os-id: debian
+Depends: ipmctl [amd64]
diff --git a/units/nvdimm/test-plan.pxu b/units/nvdimm/test-plan.pxu
new file mode 100644
index 00000000..99510394
--- /dev/null
+++ b/units/nvdimm/test-plan.pxu
@@ -0,0 +1,7 @@
+id: nvdimm-full
+unit: test plan
+_name: NVDIM Specific Tests
+_description: NVDIMM related tests
+include:
+ nvidmm/info
+ nvdimm/health