Commit dde057ab authored by dbarbera's avatar dbarbera
Browse files

Add optimization levels to benchmarks

parent 7ad2e7d1
all: compile-all
test-parse:
@$(MAKE) -s -C tests/regression test-parse
@$(MAKE) -s -C tests/regression $@
test-ada:
@$(MAKE) -s -C tests/regression test-ada
@$(MAKE) -s -C tests/regression $@
test-llvm:
@$(MAKE) -s -C tests/regression test-llvm
@$(MAKE) -s -C tests/regression $@
benchmark:
@$(MAKE) -s -C tests/regression benchmark
@$(MAKE) -s -C tests/regression $@
benchmark-O1:
@$(MAKE) -s -C tests/regression $@
benchmark-O2:
@$(MAKE) -s -C tests/regression $@
benchmark-O3:
@$(MAKE) -s -C tests/regression $@
coverage:
@$(MAKE) -s -C tests/regression coverage
@$(MAKE) -s -C tests/regression $@
flake8:
@echo Generating flake8_report file
......@@ -45,9 +54,9 @@ publish: install
@python setup.py sdist upload
clean:
@$(MAKE) -s -C tests/regression clean
@$(MAKE) -s -C tests/regression $@
@find . -name '*~' | xargs rm -f
@find . -name '*.o' | xargs rm -f
.PHONY: all test-parse test-ada test-llvm benchmark flake8 coverage \
compile-all install publish clean
.PHONY: all test-parse test-ada test-llvm benchmark benchmark-O1 benchmark-O2 \
benchmark-O3 flake8 coverage compile-all install publish clean
......@@ -1801,9 +1801,9 @@ def parse_args():
help='Generate Ada code for the .pr file')
parser.add_argument('--llvm', dest='llvm', action='store_true',
help='Generate LLVM IR code for the .pr file (experimental)')
parser.add_argument("-O", dest="optimization", metavar="LEVEL", type=int,
parser.add_argument("-O", dest="optimization", metavar="level", type=int,
action="store", choices=[0, 1, 2, 3], default=0,
help="Optimize generated LLVM IR code")
help="Set optimization level for the generated LLVM IR code")
parser.add_argument('--png', dest='png', action='store_true',
help='Generate a PNG file for the process')
parser.add_argument('--pdf', dest='pdf', action='store_true',
......
......@@ -17,6 +17,15 @@ test-llvm:
benchmark:
@python benchmark.py $(BENCHMARK_CASES)
benchmark-O1:
@python benchmark.py $(BENCHMARK_CASES) -O1
benchmark-O2:
@python benchmark.py $(BENCHMARK_CASES) -O2
benchmark-O3:
@python benchmark.py $(BENCHMARK_CASES) -O3
coverage:
@for v in $(TEST_CASES) ; \
do $(MAKE) -C $$v coverage && mv $$v/.coverage* . || exit 1 ; \
......@@ -28,4 +37,5 @@ clean:
do $(MAKE) -s -C $$v clean ; \
done
.PHONY: test-parse test-ada test-llvm benchmark coverage clean
.PHONY: test-parse test-ada test-llvm benchmark benchmark-O1 benchmark-O2 \
benchmark-O3 coverage clean
import argparse
import os
import string
import subprocess
......@@ -8,12 +9,13 @@ from tabulate import tabulate
def main():
options = parse_args()
start = t.time()
results = []
testfolders = sys.argv[1:]
for testfolder in testfolders:
results.append(benchmark(testfolder))
for testfolder in options.testfolders:
results.append(benchmark(testfolder, options.optimization))
make(testfolder, 'clean')
sys.stdout.write('.')
sys.stdout.flush()
......@@ -21,16 +23,26 @@ def main():
print ""
elapsed = t.time() - start
sys.exit(summarize(results, elapsed))
sys.exit(summarize(results, elapsed, options))
def parse_args():
parser = argparse.ArgumentParser()
parser.add_argument("-O", dest="optimization", metavar="level", type=int,
action="store", choices=[0, 1, 2, 3], default=0,
help="set optimization level")
parser.add_argument('testfolders', metavar='testfolder', type=str, nargs='*',
help='test folder(s)')
return parser.parse_args()
def benchmark(testfolder):
def benchmark(testfolder, opt_level):
result = {
"name": testfolder[:-1],
}
for rule in ("test-llvm", "test-ada"):
if make(testfolder, rule) != 0:
if make(testfolder, rule, "O=%d" % opt_level) != 0:
result["status"] = "ERROR"
return result
......@@ -68,7 +80,7 @@ def time(file, iters=1000):
return (t.time() - start) / iters
def summarize(results, elapsed):
def summarize(results, elapsed, options):
print ""
print "Summary"
print "-------"
......@@ -94,6 +106,9 @@ def summarize(results, elapsed):
print "No results"
return 1
if options.optimization != 0:
print "Optimization level: %d" % options.optimization
print ""
print "Size: Ada %.2f%% LLVM %.2f%%" % diff([r["size"] for r in valid_results])
print "Time: Ada %.2f%% LLVM %.2f%%" % diff([r["time"] for r in valid_results])
print ""
......@@ -122,8 +137,10 @@ def mean(values):
return sum(values) / len(values)
def make(path, rule):
return call(["make", "-C", path, rule])[0]
def make(path, rule, *args):
call_args = ["make", "-C", path, rule]
call_args.extend(args)
return call(call_args)[0]
def call(args):
......
......@@ -5,6 +5,7 @@ LLC=llc
GNATMAKE=gnatmake
GNATBIND=gnatbind
GNATLINK=gnatlink
O=0
clean:
rm -rf *.adb *.ads *.pyc runSpark.sh spark.idx *.o *.ali gnat.cfg \
......@@ -13,21 +14,21 @@ clean:
*.autosave
%.o: %.pr FORCE
$(OPENGEODE) $< system_structure.pr --llvm
$(OPENGEODE) $< system_structure.pr --llvm -O$(O)
$(LLC) $*.ll
$(CC) -c $*.s
$(CC) -O$(O) -c $*.s
%.ali: %.pr FORCE
$(OPENGEODE) $< system_structure.pr --toAda
$(ASN1SCC) -Ada dataview-uniq.asn -typePrefix asn1Scc -equal
$(GNATMAKE) -c *.adb
$(GNATMAKE) -O$(O) -c *.adb
%.o: %.asn FORCE
$(ASN1SCC) -c $< -typePrefix asn1Scc -equal
$(CC) -c $*.c
$(CC) -O$(O) -c $*.c
%.o: %.c FORCE
$(CC) -c $<
$(CC) -O$(O) -c $<
FORCE:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment