leandrof/linux
1
0
Fork 0
forked from Minki/linux
Code Issues Pull Requests Packages Projects Releases Wiki Activity

tools: tc-testing: better error reporting

Browse Source 
Do a better job with error handling - in pre- and post-suite,
in pre- and post-case.  Show a traceback for errors.

Signed-off-by: Brenda J. Butler <bjb@mojatatu.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
This commit is contained in:
Brenda J. Butler 2018-02-23 12:16:38 -05:00 committed by David S. Miller
parent 6a7b75f7c1
commit f9b63a1c91
1 changed files with 69 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

82
tools/testing/selftests/tc-testing/tdc.py
View File

@ -15,6 +15,7 @@ import importlib
import json
import subprocess
import time
import traceback
from collections import OrderedDict
from string import Template
@ -23,6 +24,13 @@ from tdc_helper import *
import TdcPlugin
class PluginMgrTestFail(Exception):
def __init__(self, stage, output, message):
self.stage = stage
self.output = output
self.message = message
class PluginMgr:
def __init__(self, argparser):
super().__init__()
@ -135,7 +143,7 @@ def exec_cmd(args, pm, stage, command):
return proc, foutput
def prepare_env(args, pm, stage, prefix, cmdlist):
def prepare_env(args, pm, stage, prefix, cmdlist, output = None):
"""
Execute the setup/teardown commands for a test case.
Optionally terminate test execution if the command fails.
@ -164,7 +172,9 @@ def prepare_env(args, pm, stage, prefix, cmdlist):
print("\n{} *** Aborting test run.".format(prefix), file=sys.stderr)
print("\n\n{} *** stdout ***".format(proc.stdout), file=sys.stderr)
print("\n\n{} *** stderr ***".format(proc.stderr), file=sys.stderr)
raise Exception('"{}" did not complete successfully'.format(prefix))
raise PluginMgrTestFail(
stage, output,
'"{}" did not complete successfully'.format(prefix))
def run_one_test(pm, args, index, tidx):
result = True
@ -194,8 +204,11 @@ def run_one_test(pm, args, index, tidx):
match_pattern = re.compile(
str(tidx["matchPattern"]), re.DOTALL | re.MULTILINE)
(p, procout) = exec_cmd(args, pm, 'verify', tidx["verifyCmd"])
match_index = re.findall(match_pattern, procout)
if len(match_index) != int(tidx["matchCount"]):
if procout:
match_index = re.findall(match_pattern, procout)
if len(match_index) != int(tidx["matchCount"]):
result = False
elif int(tidx["matchCount"]) != 0:
result = False
if not result:
@ -204,9 +217,12 @@ def run_one_test(pm, args, index, tidx):
tap += tresult
if result == False:
tap += procout
if procout:
tap += procout
else:
tap += 'No output!\n'
prepare_env(args, pm, 'teardown', '-----> teardown stage', tidx['teardown'])
prepare_env(args, pm, 'teardown', '-----> teardown stage', tidx['teardown'], procout)
pm.call_post_case()
index += 1
@ -227,30 +243,70 @@ def test_runner(pm, args, filtered_tests):
index = 1
tap = str(index) + ".." + str(tcount) + "\n"
badtest = None
stage = None
emergency_exit = False
emergency_exit_message = ''
pm.call_pre_suite(tcount, [tidx['id'] for tidx in testlist])
try:
pm.call_pre_suite(tcount, [tidx['id'] for tidx in testlist])
except Exception as ee:
ex_type, ex, ex_tb = sys.exc_info()
print('Exception {} {} (caught in pre_suite).'.
format(ex_type, ex))
# when the extra print statements are uncommented,
# the traceback does not appear between them
# (it appears way earlier in the tdc.py output)
# so don't bother ...
# print('--------------------(')
# print('traceback')
traceback.print_tb(ex_tb)
# print('--------------------)')
emergency_exit_message = 'EMERGENCY EXIT, call_pre_suite failed with exception {} {}\n'.format(ex_type, ex)
emergency_exit = True
stage = 'pre-SUITE'
if emergency_exit:
pm.call_post_suite(index)
return emergency_exit_message
if args.verbose > 1:
print('Run tests here')
print('give test rig 2 seconds to stabilize')
time.sleep(2)
for tidx in testlist:
if "flower" in tidx["category"] and args.device == None:
if args.verbose > 1:
print('Not executing test {} {} because DEV2 not defined'.
format(tidx['id'], tidx['name']))
continue
try:
badtest = tidx # in case it goes bad
tap += run_one_test(pm, args, index, tidx)
except Exception as ee:
print('Exception {} (caught in test_runner, running test {} {} {})'.
format(ee, index, tidx['id'], tidx['name']))
except PluginMgrTestFail as pmtf:
ex_type, ex, ex_tb = sys.exc_info()
stage = pmtf.stage
message = pmtf.message
output = pmtf.output
print(message)
print('Exception {} {} (caught in test_runner, running test {} {} {} stage {})'.
format(ex_type, ex, index, tidx['id'], tidx['name'], stage))
print('---------------')
print('traceback')
traceback.print_tb(ex_tb)
print('---------------')
if stage == 'teardown':
print('accumulated output for this test:')
if pmtf.output:
print(pmtf.output)
print('---------------')
break
index += 1
# if we failed in setup or teardown,
# fill in the remaining tests with not ok
# fill in the remaining tests with ok-skipped
count = index
tap += 'about to flush the tap output if tests need to be skipped\n'
if tcount + 1 != index:
for tidx in testlist[index - 1:]:
msg = 'skipped - previous setup or teardown failed'
msg = 'skipped - previous {} failed'.format(stage)
tap += 'ok {} - {} # {} {} {}\n'.format(
count, tidx['id'], msg, index, badtest.get('id', '--Unknown--'))
count += 1