Commit 0faf9936 authored by dbarbera's avatar dbarbera
Browse files

Add benchmark script

parent 3a62aadc
......@@ -9,6 +9,9 @@ test-ada:
test-llvm:
@make -s -C tests/regression test-llvm
benchmark:
@make -s -C tests/regression benchmark
coverage:
@make -s -C tests/regression coverage
......@@ -34,5 +37,5 @@ clean:
@find . -name '*~' | xargs rm -f
@find . -name '*.o' | xargs rm -f
.PHONY: all test-parse test-ada test-llvm flake8 coverage compile-all install \
publish clean
.PHONY: all test-parse test-ada test-llvm benchmark flake8 coverage \
compile-all install publish clean
......@@ -17,6 +17,17 @@ TEST_CASES = \
test-operators \
test-substrings \
BENCHMARK_CASES = \
test8 \
test9 \
test10 \
test11 \
test-controlflow \
test-exitnested \
test-expressions \
test-operators \
test-substrings \
test-parse:
@python test.py test-parse $(TEST_CASES)
......@@ -26,6 +37,10 @@ test-ada:
test-llvm:
@python test.py test-llvm $(TEST_CASES)
benchmark:
@python benchmark.py $(BENCHMARK_CASES)
@make clean
coverage:
for v in $(TEST_CASES) ; do make -C $$v coverage && mv $$v/.coverage* . \
|| exit 1 ; done ; coverage combine
......@@ -33,4 +48,4 @@ coverage:
clean:
for v in $(TEST_CASES); do make -C $$v clean ; done
.PHONY: test-parse test-ada test-llvm coverage clean
.PHONY: test-parse test-ada test-llvm benchmark coverage clean
import os
import subprocess
import sys
import time as t
def main():
start = t.time()
results = []
errors = 0
for testfolder in sys.argv[1:]:
result = benchmark(testfolder)
if result:
results.append(result)
else:
errors += 1
sys.stdout.write('.')
sys.stdout.flush()
sys.stdout.write('\n')
elapsed = t.time() - start
summarize(results, errors, elapsed)
def benchmark(testfolder):
for rule in ("test-llvm", "test-ada"):
if make(testfolder, rule) != 0:
return
llvm_bin = os.path.join(testfolder, "test_ada")
ada_bin = os.path.join(testfolder, "test_llvm")
for bin_name in (llvm_bin, ada_bin):
if not os.path.isfile(bin_name):
return
return {
"size": {
"ada": os.path.getsize(llvm_bin),
"llvm": os.path.getsize(ada_bin),
},
"time": {
"ada": time(ada_bin),
"llvm": time(llvm_bin),
}
}
def time(file, iters=1000):
start = t.time()
call(["/bin/bash", "-c", "for i in {1..%s} ; do %s ; done" % (iters, file)])
return (t.time() - start) / iters
def summarize(results, errors, elapsed):
print "Finished in %.3fs" % elapsed
print "%s cases, %s errors" % (len(results), errors)
if not results:
print "No results"
return
print "Summary:"
print " Size: Ada %.2f%% LLVM %.2f%%" % diff(results, "size")
print " Time: Ada %.2f%% LLVM %.2f%%" % diff(results, "time")
def diff(results, metric):
metrics = [r[metric] for r in results]
return 100, mean(([float(m["llvm"]) / float(m["ada"]) * 100 for m in metrics]))
def mean(values):
return sum(values) / len(values)
def make(path, rule):
return call(["make", "-C", path, rule])[0]
def call(args):
proc = subprocess.Popen(
args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
stdout, stderr = proc.communicate()
errcode = proc.wait()
return (errcode, stdout, stderr)
if __name__ == '__main__':
main()
benchmark:
@make test-ada > /dev/null 2>&1 && ([ -f test_ada ] || exit 1)
@make test-llvm > /dev/null 2>&1 && ([ -f test_llvm ] || exit 1)
@echo "binary-size:"
@echo " test_llvm: $$(du -b test_llvm | cut -f1)"
@echo " test_ada: $$(du -b test_ada | cut -f1)"
@python ../benchmark.py .
clean:
@rm -rf *.adb *.ads *.pyc runSpark.sh spark.idx *.o *.ali gnat.cfg \
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment