Commit b733c071 authored by dbarbera's avatar dbarbera
Browse files

Show executed/failed cases in the benchmark summary

parent 4eed812e
import os import os
import string
import subprocess import subprocess
import sys import sys
import time as t import time as t
...@@ -9,37 +10,40 @@ from tabulate import tabulate ...@@ -9,37 +10,40 @@ from tabulate import tabulate
def main(): def main():
start = t.time() start = t.time()
results = [] results = []
errors = 0 testfolders = sys.argv[1:]
for testfolder in sys.argv[1:]:
result = benchmark(testfolder) for testfolder in testfolders:
results.append(benchmark(testfolder))
make(testfolder, 'clean') make(testfolder, 'clean')
if result:
results.append(result)
else:
errors += 1
sys.stdout.write('.') sys.stdout.write('.')
sys.stdout.flush() sys.stdout.flush()
sys.stdout.write('\n') print ""
elapsed = t.time() - start elapsed = t.time() - start
sys.exit(summarize(results, errors, elapsed)) sys.exit(summarize(results, elapsed))
def benchmark(testfolder): def benchmark(testfolder):
result = {
"name": testfolder[:-1],
}
for rule in ("test-llvm", "test-ada"): for rule in ("test-llvm", "test-ada"):
if make(testfolder, rule) != 0: if make(testfolder, rule) != 0:
return result["status"] = "ERROR"
return result
llvm_bin = os.path.join(testfolder, "test_ada") llvm_bin = os.path.join(testfolder, "test_ada")
ada_bin = os.path.join(testfolder, "test_llvm") ada_bin = os.path.join(testfolder, "test_llvm")
for bin_name in (llvm_bin, ada_bin): for bin_name in (llvm_bin, ada_bin):
if not os.path.isfile(bin_name): if not os.path.isfile(bin_name):
return result["status"] = "ERROR"
return result
result = { result.update({
"name": testfolder[:-1], "status": "OK",
"size": { "size": {
"ada": size(llvm_bin), "ada": size(llvm_bin),
"llvm": size(ada_bin), "llvm": size(ada_bin),
...@@ -48,7 +52,7 @@ def benchmark(testfolder): ...@@ -48,7 +52,7 @@ def benchmark(testfolder):
"ada": time(ada_bin), "ada": time(ada_bin),
"llvm": time(llvm_bin), "llvm": time(llvm_bin),
} }
} })
return result return result
...@@ -64,35 +68,50 @@ def time(file, iters=1000): ...@@ -64,35 +68,50 @@ def time(file, iters=1000):
return (t.time() - start) / iters return (t.time() - start) / iters
def summarize(results, errors, elapsed): def summarize(results, elapsed):
print ""
print "Summary"
print "-------"
print ""
max_name_len = max([len(r["name"]) for r in results]) + 3
num_errors = 0
valid_results = []
for r in results:
print "%s [%s]" % (string.ljust(r["name"], max_name_len, '.'), r["status"])
if r["status"] == "OK":
valid_results.append(r)
else:
num_errors += 1
print ""
print "Finished in %.3fs" % elapsed print "Finished in %.3fs" % elapsed
print "%s benchmarks, %s errors" % (len(results) + errors, errors) print "%s benchmarks, %s errors" % (len(results), num_errors)
print ""
if not results: if not valid_results:
print "No results" print "No results"
return 1 return 1
print "Summary" print "Size: Ada %.2f%% LLVM %.2f%%" % diff([r["size"] for r in valid_results])
print "-------" print "Time: Ada %.2f%% LLVM %.2f%%" % diff([r["time"] for r in valid_results])
print ""
print "Size: Ada %.2f%% LLVM %.2f%%" % diff([r["size"] for r in results])
print "Time: Ada %.2f%% LLVM %.2f%%" % diff([r["time"] for r in results])
print "" print ""
headers = ["Benchmark", "Ada size (B)", "LLVM size (B)", "Ada time (us)", "LLVM time (us)"] headers = ["Benchmark", "Ada size (B)", "LLVM size (B)", "Ada time (us)", "LLVM time (us)"]
table = [] table = []
for result in results: for r in valid_results:
table.append([ table.append([
result["name"], r["name"],
result["size"]["ada"], r["size"]["ada"],
result["size"]["llvm"], r["size"]["llvm"],
int(round(result["time"]["ada"] * (10 ** 6))), int(round(r["time"]["ada"] * (10 ** 6))),
int(round(result["time"]["llvm"] * (10 ** 6))), int(round(r["time"]["llvm"] * (10 ** 6))),
]) ])
print tabulate(table, headers, tablefmt="orgtbl") print tabulate(table, headers, tablefmt="orgtbl")
return 0 if results and not errors else 1 return 0 if num_errors == 0 else 1
def diff(results): def diff(results):
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment