From 48c38f969deb767935fe2b6212b428fecd79b69c Mon Sep 17 00:00:00 2001 From: Sylvain Pineau Date: Tue, 14 Jul 2020 13:24:20 +0200 Subject: bin:sleep_test_log_check -> sleep_test_log_check.py --- bin/sleep_test_log_check | 201 -------------------------------------------- bin/sleep_test_log_check.py | 201 ++++++++++++++++++++++++++++++++++++++++++++ units/hibernate/jobs.pxu | 2 +- units/stress/jobs.pxu | 8 +- units/stress/s3s4.pxu | 4 +- units/suspend/suspend.pxu | 4 +- 6 files changed, 210 insertions(+), 210 deletions(-) delete mode 100755 bin/sleep_test_log_check create mode 100755 bin/sleep_test_log_check.py diff --git a/bin/sleep_test_log_check b/bin/sleep_test_log_check deleted file mode 100755 index aae7556..0000000 --- a/bin/sleep_test_log_check +++ /dev/null @@ -1,201 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# -# This file is part of Checkbox. -# -# Copyright 2014 Canonical Ltd. -# -# Authors -# Jeff Lane -# Daniel Manrique -# -# Checkbox is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, -# as published by the Free Software Foundation. -# -# Checkbox is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Checkbox. If not, see . - - -''' -This script is used to parse the log generated by fwts and check it for certain -errors detected during testing. It expects that this is a log file created by -fwts at runtime using the -l option. - -It's written now specifically for checking ater the fwts s3 and s4 tests but -can be adapted to look for other tests, or all tests. -''' - -import sys -import collections -import re - -from argparse import ArgumentParser -import logging - -# Definitions of when a level starts, how a failure looks, -# and when a level ends. -start_level_re = r'^(?P.+) failures: (?PNONE|\d+)$' -start_level_re = re.compile(start_level_re) -failure_re = re.compile(r'^ (?P(s3|s4)): (?P
.+)$') -end_level_re = re.compile(r"$^") - - -def parse_summary(summary, results): - """ - Parses an entire "Test Failure Summary" section, which contains a short - summary of failures observed per level. Returns nothing, but adds the - results to the passed results dictionary. - - :param summary: - A list of lines comprised in this summary section - - :param results: - The results dictionary into which to put the end result. Should be a - dict with keys for each level, the values are dicts with keys for each - test (s3, s4) which in turn contain a list of all the failures observed - for that level and test. - """ - current_level = None - current_acum = [] - - for logline in summary: - level_matches = start_level_re.search(logline) - if level_matches: - logging.debug("Found a level: %s", level_matches.group('level')) - current_level = level_matches.group('level') - elif end_level_re.search(logline) and current_level: - if current_level: - logging.debug("Current level (%s) has %s", - current_level, current_acum) - # By passing results[current_level] a key in results will be - # created for every level we see, regardless of whether it - # reports failures or not. This is OK because we can later - # check results' keys to ensure we saw at least one level; if - # results has no keys, it could mean a malformed fwts log file. - parse_level(current_acum, results[current_level]) - else: - logging.debug("Discarding junk") - current_acum = [] - current_level = None - else: - current_acum.append(logline) - - -def parse_level(level_lines, level_results): - """ - Parses the level's lines, appending the failures to the level's results. - level_results is a dictionary with a key per test type (s3, s4, and so on). - Returns nothing, but adds the results to the passed results dictionary for - this level. - - :param level_lines: - A list of lines comprised in this level's list of failures. - - : param level_results: - A dictionary containing this level's results. Should be a dict with - keys for each test, to which the failures for the level will be - appended. - """ - for failureline in level_lines: - failure_matches = failure_re.search(failureline) - if failure_matches: - test = failure_matches.group('test') - details = failure_matches.group('details') - logging.debug("fail %s was %s", test, details) - level_results[test].append(details) - - -def main(): - parser = ArgumentParser() - parser.add_argument('-d', '--debug', - action='store_const', - const=logging.DEBUG, - default=logging.INFO, - help="Show debugging information.") - parser.add_argument('-v', '--verbose', - action='store_true', - default=False, - help="Display each error discovered. May provide \ - very long output. Also, this option will only \ - provide a list of UNIQUE errors encountered in \ - the log file. It will not display duplicates. \ - Default is [%(default)s]") - parser.add_argument('test', - action='store', - choices=['s3', 's4'], - help='The test to check (s3 or s4)') - parser.add_argument('logfile', - action='store', - help='The log file to parse') - - args = parser.parse_args() - - logging.basicConfig(level=args.debug) - - #Create a generator and get our lines - log = (line.rstrip() for line in open(args.logfile, 'rt', encoding="UTF-8")) - - # End result will be a dictionary with a key per level, value is another - # dictionary with a key per test (s3, s4, ...) and a list of all failures - # for each test. Duplicates are possible, because we should also indicate - # the number of instances for each failure. - results = collections.defaultdict(lambda: collections.defaultdict(list)) - - sum_acum = [] - summaries_found = 0 - - # Start parsing the fwts log. Gather each "Test Failure Summary" section - # and when it's complete, pass it to the parse_summary function to extract - # levels and tests. - for logline in log: - if "Test Failure Summary" in logline: - parse_summary(sum_acum, results) - summaries_found += 1 - sum_acum = [] - else: - sum_acum.append(logline) - # We reached the end, so add the last accumulated summary - if sum_acum: - parse_summary(sum_acum, results) - - # Report what I found - for level in sorted(results.keys()): - if results[level]: # Yes, we can have an empty level. We may have - # seen the levelheader but had it report no - # failures. - print("{} failures:".format(level)) - for test in results[level].keys(): - print(" {}: {} failures".format(test, - len(results[level][test]))) - if args.verbose: - print('='*40) - counts = collections.Counter(results[level][test]) - for failure in counts: - print(" {} (x {})".format(failure, counts[failure])) - - # Decide on the outcome based on the collected information - if not summaries_found: - logging.error("No fwts test summaries found, " - "possible malformed fwts log file") - return_code = 2 - elif not results.keys(): # If it has no keys, means we didn't see any - # FWTS levels - logging.error("None of the summaries contained failure levels, " - "possible malformed fwts log file") - return_code = 2 - elif any(results.values()): # If any of the results' levels has errors - return_code = 1 - else: - print("No errors detected") - return_code = 0 - - return return_code - -if __name__ == '__main__': - sys.exit(main()) diff --git a/bin/sleep_test_log_check.py b/bin/sleep_test_log_check.py new file mode 100755 index 0000000..aae7556 --- /dev/null +++ b/bin/sleep_test_log_check.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# This file is part of Checkbox. +# +# Copyright 2014 Canonical Ltd. +# +# Authors +# Jeff Lane +# Daniel Manrique +# +# Checkbox is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, +# as published by the Free Software Foundation. +# +# Checkbox is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Checkbox. If not, see . + + +''' +This script is used to parse the log generated by fwts and check it for certain +errors detected during testing. It expects that this is a log file created by +fwts at runtime using the -l option. + +It's written now specifically for checking ater the fwts s3 and s4 tests but +can be adapted to look for other tests, or all tests. +''' + +import sys +import collections +import re + +from argparse import ArgumentParser +import logging + +# Definitions of when a level starts, how a failure looks, +# and when a level ends. +start_level_re = r'^(?P.+) failures: (?PNONE|\d+)$' +start_level_re = re.compile(start_level_re) +failure_re = re.compile(r'^ (?P(s3|s4)): (?P
.+)$') +end_level_re = re.compile(r"$^") + + +def parse_summary(summary, results): + """ + Parses an entire "Test Failure Summary" section, which contains a short + summary of failures observed per level. Returns nothing, but adds the + results to the passed results dictionary. + + :param summary: + A list of lines comprised in this summary section + + :param results: + The results dictionary into which to put the end result. Should be a + dict with keys for each level, the values are dicts with keys for each + test (s3, s4) which in turn contain a list of all the failures observed + for that level and test. + """ + current_level = None + current_acum = [] + + for logline in summary: + level_matches = start_level_re.search(logline) + if level_matches: + logging.debug("Found a level: %s", level_matches.group('level')) + current_level = level_matches.group('level') + elif end_level_re.search(logline) and current_level: + if current_level: + logging.debug("Current level (%s) has %s", + current_level, current_acum) + # By passing results[current_level] a key in results will be + # created for every level we see, regardless of whether it + # reports failures or not. This is OK because we can later + # check results' keys to ensure we saw at least one level; if + # results has no keys, it could mean a malformed fwts log file. + parse_level(current_acum, results[current_level]) + else: + logging.debug("Discarding junk") + current_acum = [] + current_level = None + else: + current_acum.append(logline) + + +def parse_level(level_lines, level_results): + """ + Parses the level's lines, appending the failures to the level's results. + level_results is a dictionary with a key per test type (s3, s4, and so on). + Returns nothing, but adds the results to the passed results dictionary for + this level. + + :param level_lines: + A list of lines comprised in this level's list of failures. + + : param level_results: + A dictionary containing this level's results. Should be a dict with + keys for each test, to which the failures for the level will be + appended. + """ + for failureline in level_lines: + failure_matches = failure_re.search(failureline) + if failure_matches: + test = failure_matches.group('test') + details = failure_matches.group('details') + logging.debug("fail %s was %s", test, details) + level_results[test].append(details) + + +def main(): + parser = ArgumentParser() + parser.add_argument('-d', '--debug', + action='store_const', + const=logging.DEBUG, + default=logging.INFO, + help="Show debugging information.") + parser.add_argument('-v', '--verbose', + action='store_true', + default=False, + help="Display each error discovered. May provide \ + very long output. Also, this option will only \ + provide a list of UNIQUE errors encountered in \ + the log file. It will not display duplicates. \ + Default is [%(default)s]") + parser.add_argument('test', + action='store', + choices=['s3', 's4'], + help='The test to check (s3 or s4)') + parser.add_argument('logfile', + action='store', + help='The log file to parse') + + args = parser.parse_args() + + logging.basicConfig(level=args.debug) + + #Create a generator and get our lines + log = (line.rstrip() for line in open(args.logfile, 'rt', encoding="UTF-8")) + + # End result will be a dictionary with a key per level, value is another + # dictionary with a key per test (s3, s4, ...) and a list of all failures + # for each test. Duplicates are possible, because we should also indicate + # the number of instances for each failure. + results = collections.defaultdict(lambda: collections.defaultdict(list)) + + sum_acum = [] + summaries_found = 0 + + # Start parsing the fwts log. Gather each "Test Failure Summary" section + # and when it's complete, pass it to the parse_summary function to extract + # levels and tests. + for logline in log: + if "Test Failure Summary" in logline: + parse_summary(sum_acum, results) + summaries_found += 1 + sum_acum = [] + else: + sum_acum.append(logline) + # We reached the end, so add the last accumulated summary + if sum_acum: + parse_summary(sum_acum, results) + + # Report what I found + for level in sorted(results.keys()): + if results[level]: # Yes, we can have an empty level. We may have + # seen the levelheader but had it report no + # failures. + print("{} failures:".format(level)) + for test in results[level].keys(): + print(" {}: {} failures".format(test, + len(results[level][test]))) + if args.verbose: + print('='*40) + counts = collections.Counter(results[level][test]) + for failure in counts: + print(" {} (x {})".format(failure, counts[failure])) + + # Decide on the outcome based on the collected information + if not summaries_found: + logging.error("No fwts test summaries found, " + "possible malformed fwts log file") + return_code = 2 + elif not results.keys(): # If it has no keys, means we didn't see any + # FWTS levels + logging.error("None of the summaries contained failure levels, " + "possible malformed fwts log file") + return_code = 2 + elif any(results.values()): # If any of the results' levels has errors + return_code = 1 + else: + print("No errors detected") + return_code = 0 + + return return_code + +if __name__ == '__main__': + sys.exit(main()) diff --git a/units/hibernate/jobs.pxu b/units/hibernate/jobs.pxu index e33c0b7..dc88366 100644 --- a/units/hibernate/jobs.pxu +++ b/units/hibernate/jobs.pxu @@ -72,7 +72,7 @@ _verification: plugin: shell category_id: com.canonical.plainbox::hibernate id: power-management/hibernate-single-log-check -command: [ -e $PLAINBOX_SESSION_SHARE/hibernate-single.log ] && sleep_test_log_check -v s4 $PLAINBOX_SESSION_SHARE/hibernate-single.log +command: [ -e $PLAINBOX_SESSION_SHARE/hibernate-single.log ] && sleep_test_log_check.py -v s4 $PLAINBOX_SESSION_SHARE/hibernate-single.log _description: Automated check of the hibernate log for errors discovered by fwts diff --git a/units/stress/jobs.pxu b/units/stress/jobs.pxu index 21c7b57..78d3bb1 100644 --- a/units/stress/jobs.pxu +++ b/units/stress/jobs.pxu @@ -61,7 +61,7 @@ plugin: shell category_id: com.canonical.plainbox::stress id: power-management/hibernate-30-cycles-log-check estimated_duration: 1.0 -command: [ -e $PLAINBOX_SESSION_SHARE/hibernate_30_cycles.log ] && sleep_test_log_check -v s4 $PLAINBOX_SESSION_SHARE/hibernate_30_cycles.log +command: [ -e $PLAINBOX_SESSION_SHARE/hibernate_30_cycles.log ] && sleep_test_log_check.py -v s4 $PLAINBOX_SESSION_SHARE/hibernate_30_cycles.log _description: Automated check of the 30 cycle hibernate log for errors detected by fwts. @@ -125,7 +125,7 @@ category_id: com.canonical.plainbox::stress id: power-management/suspend-30-cycles-log-check depends: power-management/suspend_30_cycles estimated_duration: 1.0 -command: [ -e $PLAINBOX_SESSION_SHARE/suspend_30_cycles.log ] && sleep_test_log_check -v s3 $PLAINBOX_SESSION_SHARE/suspend_30_cycles.log +command: [ -e $PLAINBOX_SESSION_SHARE/suspend_30_cycles.log ] && sleep_test_log_check.py -v s3 $PLAINBOX_SESSION_SHARE/suspend_30_cycles.log _description: Automated check of the 30 cycle suspend log for errors detected by fwts. @@ -134,14 +134,14 @@ category_id: com.canonical.plainbox::stress id: power-management/suspend-30-cycles-log-check-with-reboots depends: power-management/suspend_30_cycles_with_reboots estimated_duration: 1.0 -command: [ -e $PLAINBOX_SESSION_SHARE/pm_test.reboot.3.log ] && sleep_test_log_check -v s3 $PLAINBOX_SESSION_SHARE/pm_test.reboot.3.log +command: [ -e $PLAINBOX_SESSION_SHARE/pm_test.reboot.3.log ] && sleep_test_log_check.py -v s3 $PLAINBOX_SESSION_SHARE/pm_test.reboot.3.log _summary: 30 suspend/resume cycles and 1 reboot, 3 times (check logs for errors) _description: Automated check of the '30 cycle suspend and 1 reboot times 3' logs for errors detected by fwts. _siblings: [ { "id": "power-management/suspend-30-cycles-log-check-with-coldboots", "depends": "power-management/suspend_30_cycles_with_coldboots", - "command": "[ -e $PLAINBOX_SESSION_SHARE/pm_test.poweroff.3.log ] && sleep_test_log_check -v s3 $PLAINBOX_SESSION_SHARE/pm_test.poweroff.3.log", + "command": "[ -e $PLAINBOX_SESSION_SHARE/pm_test.poweroff.3.log ] && sleep_test_log_check.py -v s3 $PLAINBOX_SESSION_SHARE/pm_test.poweroff.3.log", "_description": "Automated check of the '30 cycle suspend and 1 poweroff times 3' logs for errors detected by fwts.", "_summary": "30 suspend/resume cycles and 1 poweroff, 3 times (check logs for errors)" } diff --git a/units/stress/s3s4.pxu b/units/stress/s3s4.pxu index 9a62f8e..f7a3c77 100644 --- a/units/stress/s3s4.pxu +++ b/units/stress/s3s4.pxu @@ -68,7 +68,7 @@ category_id: stress-tests/suspend id: stress-tests/suspend-{s3_iterations}-cycles-log-check after: stress-tests/suspend_{s3_iterations}_cycles estimated_duration: 1.0 -command: [ -e $PLAINBOX_SESSION_SHARE/suspend_{s3_iterations}_cycles.log ] && sleep_test_log_check -v s3 $PLAINBOX_SESSION_SHARE/suspend_{s3_iterations}_cycles.log +command: [ -e $PLAINBOX_SESSION_SHARE/suspend_{s3_iterations}_cycles.log ] && sleep_test_log_check.py -v s3 $PLAINBOX_SESSION_SHARE/suspend_{s3_iterations}_cycles.log _description: Automated check of the {s3_iterations} cycles suspend log for errors detected by fwts. @@ -118,7 +118,7 @@ category_id: stress-tests/hibernate id: stress-tests/hibernate-{s4_iterations}-cycles-log-check after: stress-tests/hibernate_{s4_iterations}_cycles estimated_duration: 1.0 -command: [ -e $PLAINBOX_SESSION_SHARE/hibernate_{s4_iterations}_cycles.log ] && sleep_test_log_check -v s4 $PLAINBOX_SESSION_SHARE/hibernate_{s4_iterations}_cycles.log +command: [ -e $PLAINBOX_SESSION_SHARE/hibernate_{s4_iterations}_cycles.log ] && sleep_test_log_check.py -v s4 $PLAINBOX_SESSION_SHARE/hibernate_{s4_iterations}_cycles.log _description: Automated check of the {s4_iterations} cycles hibernate log for errors detected by fwts. diff --git a/units/suspend/suspend.pxu b/units/suspend/suspend.pxu index e59d931..101e27f 100644 --- a/units/suspend/suspend.pxu +++ b/units/suspend/suspend.pxu @@ -307,7 +307,7 @@ category_id: com.canonical.plainbox::suspend id: suspend/suspend-single-log-check depends: suspend/suspend_advanced_auto estimated_duration: 1.2 -command: [ -e $PLAINBOX_SESSION_SHARE/suspend_single.log ] && sleep_test_log_check -v s3 $PLAINBOX_SESSION_SHARE/suspend_single.log +command: [ -e $PLAINBOX_SESSION_SHARE/suspend_single.log ] && sleep_test_log_check.py -v s3 $PLAINBOX_SESSION_SHARE/suspend_single.log _summary: Automated check of the suspend log for errors reported by fwts plugin: attachment @@ -334,7 +334,7 @@ category_id: com.canonical.plainbox::suspend id: suspend/{index}_hybrid-sleep-single-log-check depends: suspend/{index}_hybrid_sleep_{product_slug} estimated_duration: 1.2 -command: [ -e $PLAINBOX_SESSION_SHARE/{index}_hybrid_sleep_single.log ] && sleep_test_log_check -v s3 $PLAINBOX_SESSION_SHARE/{index}_hybrid_sleep_single.log +command: [ -e $PLAINBOX_SESSION_SHARE/{index}_hybrid_sleep_single.log ] && sleep_test_log_check.py -v s3 $PLAINBOX_SESSION_SHARE/{index}_hybrid_sleep_single.log _summary: Automated check of the hybrid sleep log for errors reported by fwts unit: template -- cgit v1.2.3