Commit b0421901 authored by Maxime Perrotin's avatar Maxime Perrotin
Browse files

Merge pull request #20 from dbrabera/benchmarks

Benchmarks
parents 584a8ac0 a47603ff
all: compile-all all: compile-all
test-parse: test-parse:
@python tests/test.py test-parse @$(MAKE) -s -C tests/regression test-parse
test-ada: test-ada:
@python tests/test.py test-ada @$(MAKE) -s -C tests/regression test-ada
test-llvm: test-llvm:
@python tests/test.py test-llvm @$(MAKE) -s -C tests/regression test-llvm
flake8: benchmark:
@echo Generating flake8_report file @$(MAKE) -s -C tests/regression benchmark
flake8 opengeode.py Pr.py sdlSymbols.py genericSymbols.py ogParser.py AdaGenerator.py Renderer.py Clipboard.py Lander.py ogAST.py undoCommands.py Connectors.py Asn1scc.py Helper.py Statechart.py >flake8_report
coverage: coverage:
make -C tests/regression coverage @$(MAKE) -s -C tests/regression coverage
flake8:
@echo Generating flake8_report file
@flake8 opengeode.py Pr.py sdlSymbols.py genericSymbols.py ogParser.py \
AdaGenerator.py Renderer.py Clipboard.py Lander.py ogAST.py \
undoCommands.py Connectors.py Asn1scc.py Helper.py \
Statechart.py > flake8_report
compile-all: compile-all:
pyside-rcc opengeode.qrc -o icons.py @pyside-rcc opengeode.qrc -o icons.py
if [ ! -f antlr-3.1.3.tar.bz2 ]; then wget http://download.tuxfamily.org/taste/misc/antlr-3.1.3.tar.bz2 ; tar jxvf antlr-3.1.3.tar.bz2 ; fi @if [ ! -f antlr-3.1.3.tar.bz2 ] ; \
CLASSPATH=$$PWD/antlr-3.1.3/lib/antlr-3.1.3.jar java org.antlr.Tool sdl92.g then wget http://download.tuxfamily.org/taste/misc/antlr-3.1.3.tar.bz2 ; \
tar jxvf antlr-3.1.3.tar.bz2 ; \
fi
@CLASSPATH=$$PWD/antlr-3.1.3/lib/antlr-3.1.3.jar java org.antlr.Tool sdl92.g
install: compile-all install: compile-all
mkdir -p opengeode @mkdir -p opengeode
for f in AdaGenerator.py __init__.py Pr.py genericSymbols.py icons.py ogAST.py ogParser.py opengeode.py Renderer.py samnmax.py sdl92Lexer.py sdl92Parser.py sdlSymbols.py undoCommands.py Clipboard.py Statechart.py LlvmGenerator.py Lander.py Helper.py Connectors.py Asn1scc.py; do echo Installing $$f && cp $$f opengeode; done @for f in AdaGenerator.py __init__.py Pr.py genericSymbols.py icons.py \
python setup.py install ogAST.py ogParser.py opengeode.py Renderer.py samnmax.py \
sdl92Lexer.py sdl92Parser.py sdlSymbols.py undoCommands.py \
Clipboard.py Statechart.py LlvmGenerator.py Lander.py Helper.py \
Connectors.py Asn1scc.py ; \
do echo Installing $$f && cp $$f opengeode ; \
done
@python setup.py install
publish: install publish: install
python setup.py sdist upload @python setup.py sdist upload
clean: clean:
make -C tests/regression clean @$(MAKE) -s -C tests/regression clean
find . -name '*~' | xargs rm -f @find . -name '*~' | xargs rm -f
find . -name '*.o' | xargs rm -f @find . -name '*.o' | xargs rm -f
.PHONY: all test-parse test-ada test-llvm benchmark flake8 coverage \
compile-all install publish clean
EXAMPLES=test1 test2 test3 test4 test5 test6 test7 test8 test9 test10 \ DISABLED_TESTS=test-branchcoverage/
test11 test12 test-substrings test-expressions test-controlflow \ TEST_CASES=$(filter-out $(DISABLED_TESTS), $(sort $(dir $(wildcard */))))
test-exitnested test-operators
DISABLED_BENCHMARKS=test1/ test2/ test3/ test4/ test5/ test6/ test7/ test12/
BENCHMARK_CASES=$(filter-out $(DISABLED_BENCHMARKS), $(sort $(dir $(wildcard */))))
test-parse:
@python test.py test-parse $(TEST_CASES)
test-ada:
@python test.py test-ada $(TEST_CASES)
test-llvm:
@python test.py test-llvm $(TEST_CASES)
benchmark:
@python benchmark.py $(BENCHMARK_CASES)
coverage: coverage:
for v in $(EXAMPLES) ; do make -C $$v coverage && mv $$v/.coverage* . \ @for v in $(TEST_CASES) ; \
|| exit 1 ; done ; coverage combine do $(MAKE) -C $$v coverage && mv $$v/.coverage* . || exit 1 ; \
done
@coverage combine
clean: clean:
for v in $(EXAMPLES); do make -C $$v clean ; done @for v in $(TEST_CASES) ; \
do $(MAKE) -s -C $$v clean ; \
done
.PHONY: test-parse test-ada test-llvm benchmark coverage clean
import os
import subprocess
import sys
import time as t
def main():
start = t.time()
results = []
errors = 0
for testfolder in sys.argv[1:]:
result = benchmark(testfolder)
make(testfolder, 'clean')
if result:
results.append(result)
else:
errors += 1
sys.stdout.write('.')
sys.stdout.flush()
sys.stdout.write('\n')
elapsed = t.time() - start
sys.exit(summarize(results, errors, elapsed))
def benchmark(testfolder):
for rule in ("test-llvm", "test-ada"):
if make(testfolder, rule) != 0:
return
llvm_bin = os.path.join(testfolder, "test_ada")
ada_bin = os.path.join(testfolder, "test_llvm")
for bin_name in (llvm_bin, ada_bin):
if not os.path.isfile(bin_name):
return
result = {
"size": {
"ada": os.path.getsize(llvm_bin),
"llvm": os.path.getsize(ada_bin),
},
"time": {
"ada": time(ada_bin),
"llvm": time(llvm_bin),
}
}
return result
def time(file, iters=1000):
start = t.time()
call(["/bin/bash", "-c", "for i in {1..%s} ; do %s ; done" % (iters, file)])
return (t.time() - start) / iters
def summarize(results, errors, elapsed):
print "Finished in %.3fs" % elapsed
print "%s benchmarks, %s errors" % (len(results) + errors, errors)
if not results:
print "No results"
return 1
print "Summary:"
print " Size: Ada %.2f%% LLVM %.2f%%" % diff(results, "size")
print " Time: Ada %.2f%% LLVM %.2f%%" % diff(results, "time")
return 0 if results and not errors else 1
def diff(results, metric):
metrics = [r[metric] for r in results]
return 100, mean(([float(m["llvm"]) / float(m["ada"]) * 100 for m in metrics]))
def mean(values):
return sum(values) / len(values)
def make(path, rule):
return call(["make", "-C", path, rule])[0]
def call(args):
proc = subprocess.Popen(
args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
stdout, stderr = proc.communicate()
errcode = proc.wait()
return (errcode, stdout, stderr)
if __name__ == '__main__':
main()
OPENGEODE=../../../opengeode.py
ASN1SCC=asn1.exe
CC=gcc
LLC=llc
GNATMAKE=gnatmake
GNATBIND=gnatbind
GNATLINK=gnatlink
clean:
rm -rf *.adb *.ads *.pyc runSpark.sh spark.idx *.o *.ali gnat.cfg \
examiner bin *.wrn *.gpr *.ll *.s dataview-uniq.c dataview-uniq.h \
real.c xer.c ber.c acn.c asn1crt.c asn1crt.h test_ada test_llvm \
*.autosave
.PHONY: clean
\ No newline at end of file
include ../shared.mk
all: test-ada all: test-ada
edit: edit:
../../../opengeode.py myfunction.pr system_structure.pr $(OPENGEODE) myfunction.pr system_structure.pr
check: check:
../../../opengeode.py myfunction.pr system_structure.pr --check $(OPENGEODE) myfunction.pr system_structure.pr --check
test-parse: test-parse:
../../../opengeode.py myfunction.pr system_structure.pr --check $(OPENGEODE) myfunction.pr system_structure.pr --check
test-ada: test-ada:
../../../opengeode.py --toAda myfunction.pr system_structure.pr 2>&1 | sort | diff expected - || exit 0 $(OPENGEODE) --toAda myfunction.pr system_structure.pr 2>&1 | sort | diff expected - || exit 0
coverage: coverage:
coverage run -p ../../../opengeode.py myfunction.pr system_structure.pr --toAda coverage run -p $(OPENGEODE) myfunction.pr system_structure.pr --toAda
clean: .PHONY: all edit check test-parse test-ada coverage
rm -rf *.adb *.ads *.pyc runSpark.sh spark.idx *.o *.ali gnat.cfg examiner bin *.wrn *.gpr datav*.? ber.c xer.c asn1crt.? acn.c real.c testcase \ No newline at end of file
all: test-ada include ../shared.mk
all: test-ada test-llvm
edit: edit:
../../../opengeode.py controlflow.pr system_structure.pr $(OPENGEODE) controlflow.pr system_structure.pr
test-parse: test-parse:
../../../opengeode.py controlflow.pr system_structure.pr --check $(OPENGEODE) controlflow.pr system_structure.pr --check
test-ada: test-ada:
../../../opengeode.py controlflow.pr system_structure.pr --check --toAda $(OPENGEODE) controlflow.pr system_structure.pr --check --toAda
asn1.exe -Ada dataview-uniq.asn -typePrefix asn1Scc -equal $(ASN1SCC) -Ada dataview-uniq.asn -typePrefix asn1Scc -equal
asn1.exe -c dataview-uniq.asn -typePrefix asn1Scc $(ASN1SCC) -c dataview-uniq.asn -typePrefix asn1Scc
gnatmake -c *.adb $(GNATMAKE) -c *.adb
gcc -c test_ada.c $(CC) -c test_ada.c
gnatbind -n controlflow.ali $(GNATBIND) -n controlflow.ali
gnatlink test_ada.o controlflow.ali -lgnat -lm -o testcase $(GNATLINK) test_ada.o controlflow.ali -lgnat -lm -o test_ada
./testcase | diff expected - ./test_ada | diff expected -
test-llvm: test-llvm:
../../../opengeode.py controlflow.pr system_structure.pr --check --llvm $(OPENGEODE) controlflow.pr system_structure.pr --check --llvm
asn1.exe -c dataview-uniq.asn -typePrefix asn1Scc -equal $(ASN1SCC) -c dataview-uniq.asn -typePrefix asn1Scc -equal
llc controlflow.ll $(LLC) controlflow.ll
gcc -c controlflow.s $(CC) -c controlflow.s
gcc -c dataview-uniq.c $(CC) -c dataview-uniq.c
gcc -c test_llvm.c $(CC) -c test_llvm.c
gcc controlflow.o dataview-uniq.o test_llvm.o -o testcase $(CC) controlflow.o dataview-uniq.o test_llvm.o -o test_llvm -lm
./testcase | diff expected - ./test_llvm | diff expected -
coverage:
coverage run -p $(OPENGEODE) controlflow.pr system_structure.pr --toAda
clean: .PHONY: all edit test-parse test-ada test-llvm coverage
rm -rf *.adb *.ads *.pyc runSpark.sh spark.idx *.o *.ali gnat.cfg examiner \ \ No newline at end of file
bin *.wrn *.gpr testcase *.ll *.s dataview-uniq.c dataview-uniq.h real.c xer.c \
ber.c acn.c asn1crt.c asn1crt.h
\ No newline at end of file
include ../shared.mk
all: test-ada test-llvm all: test-ada test-llvm
edit: edit:
../../../opengeode.py challenge.pr system_structure.pr $(OPENGEODE) challenge.pr system_structure.pr
test-parse: test-parse:
../../../opengeode.py challenge.pr system_structure.pr --check $(OPENGEODE) challenge.pr system_structure.pr --check
test-ada: test-ada:
../../../opengeode.py --toAda challenge.pr system_structure.pr $(OPENGEODE) --toAda challenge.pr system_structure.pr
asn1.exe -Ada dataview-uniq.asn -typePrefix asn1Scc -equal $(ASN1SCC) -Ada dataview-uniq.asn -typePrefix asn1Scc -equal
gnatmake -c *.adb $(GNATMAKE) -c *.adb
gcc -c test_ada.c $(CC) -c test_ada.c
gnatbind -n challenge.ali $(GNATBIND) -n challenge.ali
gnatlink -o testcase test_ada.o challenge.ali -lgnat -lm $(GNATLINK) -o test_ada test_ada.o challenge.ali -lgnat -lm
./testcase | diff expected - ./test_ada | diff expected -
test-llvm: test-llvm:
../../../opengeode.py challenge.pr system_structure.pr --check --llvm $(OPENGEODE) challenge.pr system_structure.pr --check --llvm
asn1.exe -c dataview-uniq.asn -typePrefix asn1Scc -equal $(ASN1SCC) -c dataview-uniq.asn -typePrefix asn1Scc -equal
llc challenge.ll $(LLC) challenge.ll
gcc -c challenge.s $(CC) -c challenge.s
gcc -c dataview-uniq.c $(CC) -c dataview-uniq.c
gcc -c test_llvm.c $(CC) -c test_llvm.c
gcc challenge.o dataview-uniq.o test_llvm.o -o testcase -lm $(CC) challenge.o dataview-uniq.o test_llvm.o -o test_llvm -lm
./testcase | diff expected - ./test_llvm | diff expected -
coverage: coverage:
coverage run -p ../../../opengeode.py challenge.pr system_structure.pr --toAda coverage run -p $(OPENGEODE) challenge.pr system_structure.pr --toAda
clean: .PHONY: all edit test-parse test-ada test-llvm coverage
rm -rf *.adb *.ads *.pyc runSpark.sh spark.idx *.o *.ali gnat.cfg examiner \ \ No newline at end of file
bin *.wrn *.gpr testcase *.ll *.s dataview-uniq.c dataview-uniq.h real.c xer.c \
ber.c acn.c asn1crt.c asn1crt.h
all: test-ada include ../shared.mk
all: test-ada test-llvm
edit: edit:
../../../opengeode.py expressions.pr system_structure.pr $(OPENGEODE) expressions.pr system_structure.pr
test-parse: test-parse:
../../../opengeode.py expressions.pr system_structure.pr --check $(OPENGEODE) expressions.pr system_structure.pr --check
test-ada: test-ada:
../../../opengeode.py expressions.pr system_structure.pr --check --toAda $(OPENGEODE) expressions.pr system_structure.pr --check --toAda
asn1.exe -Ada dataview-uniq.asn -typePrefix asn1Scc -equal $(ASN1SCC) -Ada dataview-uniq.asn -typePrefix asn1Scc -equal
asn1.exe -c dataview-uniq.asn -typePrefix asn1Scc $(ASN1SCC) -c dataview-uniq.asn -typePrefix asn1Scc
gnatmake -c *.adb $(GNATMAKE) -c *.adb
gcc -c test_ada.c $(CC) -c test_ada.c
gnatbind -n expressions.ali $(GNATBIND) -n expressions.ali
gnatlink test_ada.o expressions.ali -lgnat -lm -o testcase $(GNATLINK) test_ada.o expressions.ali -lgnat -lm -o test_ada
./testcase ./test_ada
test-llvm: test-llvm:
../../../opengeode.py expressions.pr system_structure.pr --check --llvm $(OPENGEODE) expressions.pr system_structure.pr --check --llvm
asn1.exe -c dataview-uniq.asn -typePrefix asn1Scc -equal $(ASN1SCC) -c dataview-uniq.asn -typePrefix asn1Scc -equal
llc expressions.ll $(LLC) expressions.ll
gcc -c expressions.s $(CC) -c expressions.s
gcc -c dataview-uniq.c $(CC) -c dataview-uniq.c
gcc -c test_llvm.c $(CC) -c test_llvm.c
gcc expressions.o dataview-uniq.o test_llvm.o -o testcase $(CC) expressions.o dataview-uniq.o test_llvm.o -o test_llvm -lm
./testcase ./test_llvm
coverage:
coverage run -p $(OPENGEODE) expressions.pr system_structure.pr --toAda
clean: .PHONY: all edit test-parse test-ada test-llvm coverage
rm -rf *.adb *.ads *.pyc runSpark.sh spark.idx *.o *.ali gnat.cfg examiner \ \ No newline at end of file
bin *.wrn *.gpr testcase *.ll *.s dataview-uniq.c dataview-uniq.h real.c xer.c \
ber.c acn.c asn1crt.c asn1crt.h
all: test-ada include ../shared.mk
all: test-ada test-llvm
edit: edit:
../../../opengeode.py operators.pr system_structure.pr $(OPENGEODE) operators.pr system_structure.pr
test-parse: test-parse:
../../../opengeode.py operators.pr system_structure.pr --check $(OPENGEODE) operators.pr system_structure.pr --check
test-ada: test-ada:
../../../opengeode.py operators.pr system_structure.pr --check --toAda $(OPENGEODE) operators.pr system_structure.pr --check --toAda
asn1.exe -Ada dataview-uniq.asn -typePrefix asn1Scc -equal $(ASN1SCC) -Ada dataview-uniq.asn -typePrefix asn1Scc -equal
asn1.exe -c dataview-uniq.asn -typePrefix asn1Scc $(ASN1SCC) -c dataview-uniq.asn -typePrefix asn1Scc
gnatmake -c *.adb $(GNATMAKE) -c *.adb
gcc -c test_ada.c $(CC) -c test_ada.c
gnatbind -n operators.ali $(GNATBIND) -n operators.ali
gnatlink test_ada.o operators.ali -lgnat -lm -o testcase $(GNATLINK) test_ada.o operators.ali -lgnat -lm -o test_ada
./testcase | diff expected - ./test_ada | diff expected -
test-llvm: test-llvm:
../../../opengeode.py operators.pr system_structure.pr --check --llvm $(OPENGEODE) operators.pr system_structure.pr --check --llvm
asn1.exe -c dataview-uniq.asn -typePrefix asn1Scc -equal $(ASN1SCC) -c dataview-uniq.asn -typePrefix asn1Scc -equal
llc operators.ll $(LLC) operators.ll
clang -c operators.s $(CC) -c operators.s
clang -c dataview-uniq.c $(CC) -c dataview-uniq.c
clang -c test_llvm.c $(CC) -c test_llvm.c
clang operators.o dataview-uniq.o test_llvm.o -o testcase -lm $(CC) operators.o dataview-uniq.o test_llvm.o -o test_llvm -lm
./testcase | diff expected - ./test_llvm | diff expected -
coverage:
coverage run -p $(OPENGEODE) operators.pr system_structure.pr --toAda
clean: .PHONY: all edit test-parse test-ada test-llvm coverage
rm -rf *.adb *.ads *.pyc runSpark.sh spark.idx *.o *.ali gnat.cfg examiner \ \ No newline at end of file
bin *.wrn *.gpr testcase *.ll *.s dataview-uniq.c dataview-uniq.h real.c xer.c \
ber.c acn.c asn1crt.c asn1crt.h
all: test-ada include ../shared.mk
edit: all: test-ada test-llvm
../../../opengeode.py myfunction.pr system_structure.pr
check: edit:
../../../opengeode.py myfunction.pr system_structure.pr --check $(OPENGEODE) myfunction.pr system_structure.pr
test-parse: test-parse:
../../../opengeode.py myfunction.pr system_structure.pr --check $(OPENGEODE) myfunction.pr system_structure.pr --check
test-ada: test-ada:
../../../opengeode.py --toAda myfunction.pr system_structure.pr || exit 1 $(OPENGEODE) --toAda myfunction.pr system_structure.pr || exit 1
asn1.exe -Ada dataview-uniq.asn -typePrefix asn1Scc -equal $(ASN1SCC) -Ada dataview-uniq.asn -typePrefix asn1Scc -equal
asn1.exe -c dataview-uniq.asn -typePrefix asn1Scc $(ASN1SCC) -c dataview-uniq.asn -typePrefix asn1Scc
gnatmake -c myfunction.adb $(GNATMAKE) -c myfunction.adb
gcc -c test.c $(CC) -c test_ada.c
gnatbind -n myfunction.ali $(GNATBIND) -n myfunction.ali
gnatlink -o testcase test.o myfunction.ali -lgnat $(GNATLINK) -o test_ada test_ada.o myfunction.ali -lgnat
./testcase | diff expected - ./test_ada | diff expected -
test-llvm: test-llvm:
../../../opengeode.py myfunction.pr system_structure.pr --check --llvm $(OPENGEODE) myfunction.pr system_structure.pr --check --llvm
asn1.exe -c dataview-uniq.asn -equal -typePrefix asn1Scc $(ASN1SCC) -c dataview-uniq.asn -equal -typePrefix asn1Scc
llc myfunction.ll $(LLC) myfunction.ll
gcc -c myfunction.s $(CC) -c myfunction.s
gcc -c dataview-uniq.c $(CC) -c dataview-uniq.c
gcc -c test_llvm.c $(CC) -c test_llvm.c
gcc myfunction.o dataview-uniq.o test_llvm.o -o testcase $(CC) myfunction.o dataview-uniq.o test_llvm.o -o test_llvm -lm
./testcase | diff expected - ./test_llvm | diff expected -
coverage: coverage:
coverage run -p ../../../opengeode.py myfunction.pr system_structure.pr --toAda coverage run -p $(OPENGEODE) myfunction.pr system_structure.pr --toAda
clean:
rm -rf *.adb *.ads *.pyc runSpark.sh spark.idx *.o *.ali gnat.cfg examiner \
bin *.wrn *.gpr datav*.? ber.c xer.c asn1crt.? acn.c real.c *.ll *.s testcase
.PHONY: all edit test-parse test-ada test-llvm coverage
\ No newline at end of file
import os
import subprocess import subprocess
import sys import sys
import time import time
tests_folder = os.path.dirname(__file__)
paths = [
'regression/test1',
'regression/test2',
'regression/test3',
'regression/test4',
'regression/test5',
'regression/test6',
'regression/test7',
'regression/test8',
'regression/test9',
'regression/test10',
'regression/test11',
'regression/test12',
'regression/test-controlflow',
'regression/test-expressions',
'regression/test-operators',