Commit c5dd06a8 authored by Maxime Perrotin's avatar Maxime Perrotin
Browse files

Update test scripts

parent 6a233a56
DISABLED_TESTS=test-branchcoverage/
DISABLED_TESTS=test-branchcoverage/ logs/
MAKEFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST)))
TEST_BASE_DIR := $(dir $(MAKEFILE_PATH))
TEST_CASES=$(filter-out $(DISABLED_TESTS), $(sort $(dir $(wildcard */))))
DISABLED_BENCHMARKS=test1/ test2/ test3/ test4/ test5/ test6/ test7/ test9/ \
......@@ -6,7 +8,7 @@ DISABLED_BENCHMARKS=test1/ test2/ test3/ test4/ test5/ test6/ test7/ test9/ \
BENCHMARK_CASES=$(filter-out $(DISABLED_BENCHMARKS), $(sort $(dir $(wildcard */))))
export EXPECTED_FAILURES=test-typecheck/ test-procedure/ test-demoopengeode/ test-branchcoverage/ \
test-nestedpriority/
test-nestedpriority/ test-iterators/
export QGEN_UNSUPPORTED=test-debug/ test-demotaste/ test-instance/ test-eds2/ test-asn1constants/\
test-nocif/ test-nocif2/ test-provided2/ test-processfpar1/ test-processfpar2/\
test-recursion/
......@@ -30,6 +32,7 @@ test-qgen-gt-c:
@python3 test.py test-qgen-gt-c $(TEST_CASES)
test-ada:
rm -f ${TEST_BASE_DIR}/logs/*
@python3 test.py test-ada $(TEST_CASES)
test-c:
......
......@@ -27,6 +27,12 @@ testsWork = {
'test-vhdl': work1
}
def openLog(name, mode='w'):
logs = os.path.join(os.path.dirname(os.path.abspath(__file__)),'logs')
os.makedirs(logs, exist_ok=True)
return open(os.path.join(logs, name + '.err.txt'), mode)
def colorMe(result, msg):
if sys.stdout.isatty():
code = "1" if result else "2"
......@@ -46,11 +52,13 @@ def main():
start = time.time()
results = []
op = parse_args()
paths = sys.argv[2:]
xfails = os.environ['EXPECTED_FAILURES']
if op.rule in ['test-qgen-parse', 'test-qgen-ada', 'test-qgen-c', 'test-qgen-gt-ada', 'test-qgen-gt-c']:
qgen_unsup = os.environ['QGEN_UNSUPPORTED']
else:
qgen_unsup = ""
print ("Running {} tests using {} processors".format(len(paths), cpu_count()))
with futures.ProcessPoolExecutor(max_workers=cpu_count()) as executor:
fs = [executor.submit(partial(partial(make, op.rule)), path)
......@@ -63,7 +71,9 @@ def main():
result = each.result()
errcode, stdout, stderr, path, rule = result
name = path.replace("/", "")
print("%40s: %s" % (name, colorMe(errcode,
print("(%3d / %3d) %40s: %s" %
(len(results)+1, len(paths),
name, colorMe(errcode,
'[OK]' if errcode==0 else
('[EXPECTED FAILURE]'
if path in xfails
......@@ -73,7 +83,7 @@ def main():
sys.stdout.flush()
if errcode != 0:
# Failure: save the log immediately
with open("/tmp/{}.err".format(name), 'w') as f:
with openLog(name) as f:
f.write("=" * 80)
f.write("ERROR: %s %s" % (name, rule))
if stdout:
......@@ -83,6 +93,9 @@ def main():
f.write("-- stderr " + "-" * 70)
f.write(stderr.decode())
f.write("-" * 80)
if errcode != 0 and name in xfails:
# for "expected failures", set errcode to None
result = (None, stdout, stderr, path, rule)
results.append(result)
executor.map(partial(make, 'clean'), op.paths)
sys.stdout.write('\n')
......@@ -115,26 +128,17 @@ def make(rule, path):
def summarize(results, elapsed):
''' At the end display the errors of project that failed '''
failed = 0
with open("/tmp/opengeode.err", "w") as f:
f.write("opengeode test report")
f.write("---------------------")
expected_failures = 0
for errcode, stdout, stderr, path, rule in results:
if errcode == 0:
continue
failed += 1
with open("/tmp/kazoo.err", 'a') as f:
f.write("=" * 80)
f.write("ERROR: %s %s" % (path, rule))
if stdout:
f.write("-- stdout " + "-" * 70)
f.write(stdout.decode("latin-1"))
if stderr:
f.write("-- stderr " + "-" * 70)
f.write(stderr.decode("latin-1"))
f.write("-" * 80)
if errcode is not None:
failed += 1
else:
expected_failures += 1
print("Finished in %.3fs" % elapsed)
print("%s tests, %s errors" % (len(results), failed))
print("%s expected failure(s)" % expected_failures)
return 0 if not failed else 1
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment