Skip to content

GitLab

Commit b0421901 authored by Maxime Perrotin's avatar Maxime Perrotin
Browse files

Merge pull request #20 from dbrabera/benchmarks 

Benchmarks
parents 584a8ac0 a47603ff
Hide whitespace changes
Inline Side-by-side
Makefile
View file @ b0421901
all: compile-all
test-parse:
@python tests/test.py test-parse
@$(MAKE) -s -C tests/regression test-parse
test-ada:
@python tests/test.py test-ada
@$(MAKE) -s -C tests/regression test-ada
test-llvm:
@python tests/test.py test-llvm
@$(MAKE) -s -C tests/regression test-llvm
flake8:
@echo Generating flake8_report file
flake8 opengeode.py Pr.py sdlSymbols.py genericSymbols.py ogParser.py AdaGenerator.py Renderer.py Clipboard.py Lander.py ogAST.py undoCommands.py Connectors.py Asn1scc.py Helper.py Statechart.py >flake8_report
benchmark:
@$(MAKE) -s -C tests/regression benchmark
coverage:
make -C tests/regression coverage
@$(MAKE) -s -C tests/regression coverage
flake8:
@echo Generating flake8_report file
@flake8 opengeode.py Pr.py sdlSymbols.py genericSymbols.py ogParser.py \
AdaGenerator.py Renderer.py Clipboard.py Lander.py ogAST.py \
undoCommands.py Connectors.py Asn1scc.py Helper.py \
Statechart.py > flake8_report
compile-all:
pyside-rcc opengeode.qrc -o icons.py
if [ ! -f antlr-3.1.3.tar.bz2 ]; then wget http://download.tuxfamily.org/taste/misc/antlr-3.1.3.tar.bz2 ; tar jxvf antlr-3.1.3.tar.bz2 ; fi
CLASSPATH=$$PWD/antlr-3.1.3/lib/antlr-3.1.3.jar java org.antlr.Tool sdl92.g
@pyside-rcc opengeode.qrc -o icons.py
@if [ ! -f antlr-3.1.3.tar.bz2 ] ; \
then wget http://download.tuxfamily.org/taste/misc/antlr-3.1.3.tar.bz2 ; \
tar jxvf antlr-3.1.3.tar.bz2 ; \
fi
@CLASSPATH=$$PWD/antlr-3.1.3/lib/antlr-3.1.3.jar java org.antlr.Tool sdl92.g
install: compile-all
mkdir -p opengeode
for f in AdaGenerator.py __init__.py Pr.py genericSymbols.py icons.py ogAST.py ogParser.py opengeode.py Renderer.py samnmax.py sdl92Lexer.py sdl92Parser.py sdlSymbols.py undoCommands.py Clipboard.py Statechart.py LlvmGenerator.py Lander.py Helper.py Connectors.py Asn1scc.py; do echo Installing $$f && cp $$f opengeode; done
python setup.py install
@mkdir -p opengeode
@for f in AdaGenerator.py __init__.py Pr.py genericSymbols.py icons.py \
ogAST.py ogParser.py opengeode.py Renderer.py samnmax.py \
sdl92Lexer.py sdl92Parser.py sdlSymbols.py undoCommands.py \
Clipboard.py Statechart.py LlvmGenerator.py Lander.py Helper.py \
Connectors.py Asn1scc.py ; \
do echo Installing $$f && cp $$f opengeode ; \
done
@python setup.py install
publish: install
python setup.py sdist upload
@python setup.py sdist upload
clean:
make -C tests/regression clean
find . -name '*~' | xargs rm -f
find . -name '*.o' | xargs rm -f
@$(MAKE) -s -C tests/regression clean
@find . -name '*~' | xargs rm -f
@find . -name '*.o' | xargs rm -f
.PHONY: all test-parse test-ada test-llvm benchmark flake8 coverage \
compile-all install publish clean
tests/regression/Makefile
View file @ b0421901
EXAMPLES=test1 test2 test3 test4 test5 test6 test7 test8 test9 test10 \
test11 test12 test-substrings test-expressions test-controlflow \
test-exitnested test-operators
DISABLED_TESTS=test-branchcoverage/
TEST_CASES=$(filter-out $(DISABLED_TESTS), $(sort $(dir $(wildcard */))))
DISABLED_BENCHMARKS=test1/ test2/ test3/ test4/ test5/ test6/ test7/ test12/
BENCHMARK_CASES=$(filter-out $(DISABLED_BENCHMARKS), $(sort $(dir $(wildcard */))))
test-parse:
@python test.py test-parse $(TEST_CASES)
test-ada:
@python test.py test-ada $(TEST_CASES)
test-llvm:
@python test.py test-llvm $(TEST_CASES)
benchmark:
@python benchmark.py $(BENCHMARK_CASES)
coverage:
for v in $(EXAMPLES) ; do make -C $$v coverage && mv $$v/.coverage* . \
|| exit 1 ; done ; coverage combine
@for v in $(TEST_CASES) ; \
do $(MAKE) -C $$v coverage && mv $$v/.coverage* . || exit 1 ; \
done
@coverage combine
clean:
for v in $(EXAMPLES); do make -C $$v clean ; done
@for v in $(TEST_CASES) ; \
do $(MAKE) -s -C $$v clean ; \
done
.PHONY: test-parse test-ada test-llvm benchmark coverage clean
tests/regression/benchmark.py 0 → 100644
View file @ b0421901
import os
import subprocess
import sys
import time as t
def main():
start = t.time()
results = []
errors = 0
for testfolder in sys.argv[1:]:
result = benchmark(testfolder)
make(testfolder, 'clean')
if result:
results.append(result)
else:
errors += 1
sys.stdout.write('.')
sys.stdout.flush()
sys.stdout.write('\n')
elapsed = t.time() - start
sys.exit(summarize(results, errors, elapsed))
def benchmark(testfolder):
for rule in ("test-llvm", "test-ada"):
if make(testfolder, rule) != 0:
return
llvm_bin = os.path.join(testfolder, "test_ada")
ada_bin = os.path.join(testfolder, "test_llvm")
for bin_name in (llvm_bin, ada_bin):
if not os.path.isfile(bin_name):
return
result = {
"size": {
"ada": os.path.getsize(llvm_bin),
"llvm": os.path.getsize(ada_bin),
},
"time": {
"ada": time(ada_bin),
"llvm": time(llvm_bin),
}
}
return result
def time(file, iters=1000):
start = t.time()
call(["/bin/bash", "-c", "for i in {1..%s} ; do %s ; done" % (iters, file)])
return (t.time() - start) / iters
def summarize(results, errors, elapsed):
print "Finished in %.3fs" % elapsed
print "%s benchmarks, %s errors" % (len(results) + errors, errors)
if not results:
print "No results"
return 1
print "Summary:"
print " Size: Ada %.2f%% LLVM %.2f%%" % diff(results, "size")
print " Time: Ada %.2f%% LLVM %.2f%%" % diff(results, "time")
return 0 if results and not errors else 1
def diff(results, metric):
metrics = [r[metric] for r in results]
return 100, mean(([float(m["llvm"]) / float(m["ada"]) * 100 for m in metrics]))
def mean(values):
return sum(values) / len(values)
def make(path, rule):
return call(["make", "-C", path, rule])[0]
def call(args):
proc = subprocess.Popen(
args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
stdout, stderr = proc.communicate()
errcode = proc.wait()
return (errcode, stdout, stderr)
if __name__ == '__main__':
main()
tests/regression/shared.mk 0 → 100644
View file @ b0421901
OPENGEODE=../../../opengeode.py
ASN1SCC=asn1.exe
CC=gcc
LLC=llc
GNATMAKE=gnatmake
GNATBIND=gnatbind
GNATLINK=gnatlink
clean:
rm -rf *.adb *.ads *.pyc runSpark.sh spark.idx *.o *.ali gnat.cfg \
examiner bin *.wrn *.gpr *.ll *.s dataview-uniq.c dataview-uniq.h \
real.c xer.c ber.c acn.c asn1crt.c asn1crt.h test_ada test_llvm \
*.autosave
.PHONY: clean
\ No newline at end of file
tests/regression/test-branchcoverage/Makefile
View file @ b0421901
include ../shared.mk
all: test-ada
edit:
../../../opengeode.py myfunction.pr system_structure.pr
$(OPENGEODE) myfunction.pr system_structure.pr
check:
../../../opengeode.py myfunction.pr system_structure.pr --check
$(OPENGEODE) myfunction.pr system_structure.pr --check
test-parse:
../../../opengeode.py myfunction.pr system_structure.pr --check
$(OPENGEODE) myfunction.pr system_structure.pr --check
test-ada:
../../../opengeode.py --toAda myfunction.pr system_structure.pr 2>&1 | sort | diff expected - || exit 0
$(OPENGEODE) --toAda myfunction.pr system_structure.pr 2>&1 | sort | diff expected - || exit 0
coverage:
coverage run -p ../../../opengeode.py myfunction.pr system_structure.pr --toAda
coverage run -p $(OPENGEODE) myfunction.pr system_structure.pr --toAda
clean:
rm -rf *.adb *.ads *.pyc runSpark.sh spark.idx *.o *.ali gnat.cfg examiner bin *.wrn *.gpr datav*.? ber.c xer.c asn1crt.? acn.c real.c testcase
.PHONY: all edit check test-parse test-ada coverage
\ No newline at end of file
tests/regression/test-controlflow/Makefile
View file @ b0421901
all: test-ada
include ../shared.mk
all: test-ada test-llvm
edit:
../../../opengeode.py controlflow.pr system_structure.pr
$(OPENGEODE) controlflow.pr system_structure.pr
test-parse:
../../../opengeode.py controlflow.pr system_structure.pr --check
$(OPENGEODE) controlflow.pr system_structure.pr --check
test-ada:
../../../opengeode.py controlflow.pr system_structure.pr --check --toAda
asn1.exe -Ada dataview-uniq.asn -typePrefix asn1Scc -equal
asn1.exe -c dataview-uniq.asn -typePrefix asn1Scc
gnatmake -c *.adb
gcc -c test_ada.c
gnatbind -n controlflow.ali
gnatlink test_ada.o controlflow.ali -lgnat -lm -o testcase
./testcase | diff expected -
$(OPENGEODE) controlflow.pr system_structure.pr --check --toAda
$(ASN1SCC) -Ada dataview-uniq.asn -typePrefix asn1Scc -equal
$(ASN1SCC) -c dataview-uniq.asn -typePrefix asn1Scc
$(GNATMAKE) -c *.adb
$(CC) -c test_ada.c
$(GNATBIND) -n controlflow.ali
$(GNATLINK) test_ada.o controlflow.ali -lgnat -lm -o test_ada
./test_ada | diff expected -
test-llvm:
../../../opengeode.py controlflow.pr system_structure.pr --check --llvm
asn1.exe -c dataview-uniq.asn -typePrefix asn1Scc -equal
llc controlflow.ll
gcc -c controlflow.s
gcc -c dataview-uniq.c
gcc -c test_llvm.c
gcc controlflow.o dataview-uniq.o test_llvm.o -o testcase
./testcase | diff expected -
$(OPENGEODE) controlflow.pr system_structure.pr --check --llvm
$(ASN1SCC) -c dataview-uniq.asn -typePrefix asn1Scc -equal
$(LLC) controlflow.ll
$(CC) -c controlflow.s
$(CC) -c dataview-uniq.c
$(CC) -c test_llvm.c
$(CC) controlflow.o dataview-uniq.o test_llvm.o -o test_llvm -lm
./test_llvm | diff expected -
coverage:
coverage run -p $(OPENGEODE) controlflow.pr system_structure.pr --toAda
clean:
rm -rf *.adb *.ads *.pyc runSpark.sh spark.idx *.o *.ali gnat.cfg examiner \
bin *.wrn *.gpr testcase *.ll *.s dataview-uniq.c dataview-uniq.h real.c xer.c \
ber.c acn.c asn1crt.c asn1crt.h
\ No newline at end of file
.PHONY: all edit test-parse test-ada test-llvm coverage
\ No newline at end of file
tests/regression/test-exitnested/Makefile
View file @ b0421901
include ../shared.mk
all: test-ada test-llvm
edit:
../../../opengeode.py challenge.pr system_structure.pr
$(OPENGEODE) challenge.pr system_structure.pr
test-parse:
../../../opengeode.py challenge.pr system_structure.pr --check
$(OPENGEODE) challenge.pr system_structure.pr --check
test-ada:
../../../opengeode.py --toAda challenge.pr system_structure.pr
asn1.exe -Ada dataview-uniq.asn -typePrefix asn1Scc -equal
gnatmake -c *.adb
gcc -c test_ada.c
gnatbind -n challenge.ali
gnatlink -o testcase test_ada.o challenge.ali -lgnat -lm
./testcase | diff expected -
$(OPENGEODE) --toAda challenge.pr system_structure.pr
$(ASN1SCC) -Ada dataview-uniq.asn -typePrefix asn1Scc -equal
$(GNATMAKE) -c *.adb
$(CC) -c test_ada.c
$(GNATBIND) -n challenge.ali
$(GNATLINK) -o test_ada test_ada.o challenge.ali -lgnat -lm
./test_ada | diff expected -
test-llvm:
../../../opengeode.py challenge.pr system_structure.pr --check --llvm
asn1.exe -c dataview-uniq.asn -typePrefix asn1Scc -equal
llc challenge.ll
gcc -c challenge.s
gcc -c dataview-uniq.c
gcc -c test_llvm.c
gcc challenge.o dataview-uniq.o test_llvm.o -o testcase -lm
./testcase | diff expected -
$(OPENGEODE) challenge.pr system_structure.pr --check --llvm
$(ASN1SCC) -c dataview-uniq.asn -typePrefix asn1Scc -equal
$(LLC) challenge.ll
$(CC) -c challenge.s
$(CC) -c dataview-uniq.c
$(CC) -c test_llvm.c
$(CC) challenge.o dataview-uniq.o test_llvm.o -o test_llvm -lm
./test_llvm | diff expected -
coverage:
coverage run -p ../../../opengeode.py challenge.pr system_structure.pr --toAda
coverage run -p $(OPENGEODE) challenge.pr system_structure.pr --toAda
clean:
rm -rf *.adb *.ads *.pyc runSpark.sh spark.idx *.o *.ali gnat.cfg examiner \
bin *.wrn *.gpr testcase *.ll *.s dataview-uniq.c dataview-uniq.h real.c xer.c \
ber.c acn.c asn1crt.c asn1crt.h
.PHONY: all edit test-parse test-ada test-llvm coverage
\ No newline at end of file
tests/regression/test-expressions/Makefile
View file @ b0421901
all: test-ada
include ../shared.mk
all: test-ada test-llvm
edit:
../../../opengeode.py expressions.pr system_structure.pr
$(OPENGEODE) expressions.pr system_structure.pr
test-parse:
../../../opengeode.py expressions.pr system_structure.pr --check
$(OPENGEODE) expressions.pr system_structure.pr --check
test-ada:
../../../opengeode.py expressions.pr system_structure.pr --check --toAda
asn1.exe -Ada dataview-uniq.asn -typePrefix asn1Scc -equal
asn1.exe -c dataview-uniq.asn -typePrefix asn1Scc
gnatmake -c *.adb
gcc -c test_ada.c
gnatbind -n expressions.ali
gnatlink test_ada.o expressions.ali -lgnat -lm -o testcase
./testcase
$(OPENGEODE) expressions.pr system_structure.pr --check --toAda
$(ASN1SCC) -Ada dataview-uniq.asn -typePrefix asn1Scc -equal
$(ASN1SCC) -c dataview-uniq.asn -typePrefix asn1Scc
$(GNATMAKE) -c *.adb
$(CC) -c test_ada.c
$(GNATBIND) -n expressions.ali
$(GNATLINK) test_ada.o expressions.ali -lgnat -lm -o test_ada
./test_ada
test-llvm:
../../../opengeode.py expressions.pr system_structure.pr --check --llvm
asn1.exe -c dataview-uniq.asn -typePrefix asn1Scc -equal
llc expressions.ll
gcc -c expressions.s
gcc -c dataview-uniq.c
gcc -c test_llvm.c
gcc expressions.o dataview-uniq.o test_llvm.o -o testcase
./testcase
$(OPENGEODE) expressions.pr system_structure.pr --check --llvm
$(ASN1SCC) -c dataview-uniq.asn -typePrefix asn1Scc -equal
$(LLC) expressions.ll
$(CC) -c expressions.s
$(CC) -c dataview-uniq.c
$(CC) -c test_llvm.c
$(CC) expressions.o dataview-uniq.o test_llvm.o -o test_llvm -lm
./test_llvm
coverage:
coverage run -p $(OPENGEODE) expressions.pr system_structure.pr --toAda
clean:
rm -rf *.adb *.ads *.pyc runSpark.sh spark.idx *.o *.ali gnat.cfg examiner \
bin *.wrn *.gpr testcase *.ll *.s dataview-uniq.c dataview-uniq.h real.c xer.c \
ber.c acn.c asn1crt.c asn1crt.h
.PHONY: all edit test-parse test-ada test-llvm coverage
\ No newline at end of file
tests/regression/test-operators/Makefile
View file @ b0421901
all: test-ada
include ../shared.mk
all: test-ada test-llvm
edit:
../../../opengeode.py operators.pr system_structure.pr
$(OPENGEODE) operators.pr system_structure.pr
test-parse:
../../../opengeode.py operators.pr system_structure.pr --check
$(OPENGEODE) operators.pr system_structure.pr --check
test-ada:
../../../opengeode.py operators.pr system_structure.pr --check --toAda
asn1.exe -Ada dataview-uniq.asn -typePrefix asn1Scc -equal
asn1.exe -c dataview-uniq.asn -typePrefix asn1Scc
gnatmake -c *.adb
gcc -c test_ada.c
gnatbind -n operators.ali
gnatlink test_ada.o operators.ali -lgnat -lm -o testcase
./testcase | diff expected -
$(OPENGEODE) operators.pr system_structure.pr --check --toAda
$(ASN1SCC) -Ada dataview-uniq.asn -typePrefix asn1Scc -equal
$(ASN1SCC) -c dataview-uniq.asn -typePrefix asn1Scc
$(GNATMAKE) -c *.adb
$(CC) -c test_ada.c
$(GNATBIND) -n operators.ali
$(GNATLINK) test_ada.o operators.ali -lgnat -lm -o test_ada
./test_ada | diff expected -
test-llvm:
../../../opengeode.py operators.pr system_structure.pr --check --llvm
asn1.exe -c dataview-uniq.asn -typePrefix asn1Scc -equal
llc operators.ll
clang -c operators.s
clang -c dataview-uniq.c
clang -c test_llvm.c
clang operators.o dataview-uniq.o test_llvm.o -o testcase -lm
./testcase | diff expected -
$(OPENGEODE) operators.pr system_structure.pr --check --llvm
$(ASN1SCC) -c dataview-uniq.asn -typePrefix asn1Scc -equal
$(LLC) operators.ll
$(CC) -c operators.s
$(CC) -c dataview-uniq.c
$(CC) -c test_llvm.c
$(CC) operators.o dataview-uniq.o test_llvm.o -o test_llvm -lm
./test_llvm | diff expected -
coverage:
coverage run -p $(OPENGEODE) operators.pr system_structure.pr --toAda
clean:
rm -rf *.adb *.ads *.pyc runSpark.sh spark.idx *.o *.ali gnat.cfg examiner \
bin *.wrn *.gpr testcase *.ll *.s dataview-uniq.c dataview-uniq.h real.c xer.c \
ber.c acn.c asn1crt.c asn1crt.h
.PHONY: all edit test-parse test-ada test-llvm coverage
\ No newline at end of file
tests/regression/test-substrings/Makefile
View file @ b0421901
all: test-ada
include ../shared.mk
edit:
../../../opengeode.py myfunction.pr system_structure.pr
all: test-ada test-llvm
check:
../../../opengeode.py myfunction.pr system_structure.pr --check
edit:
$(OPENGEODE) myfunction.pr system_structure.pr
test-parse:
../../../opengeode.py myfunction.pr system_structure.pr --check
$(OPENGEODE) myfunction.pr system_structure.pr --check
test-ada:
../../../opengeode.py --toAda myfunction.pr system_structure.pr || exit 1
asn1.exe -Ada dataview-uniq.asn -typePrefix asn1Scc -equal
asn1.exe -c dataview-uniq.asn -typePrefix asn1Scc
gnatmake -c myfunction.adb
gcc -c test.c
gnatbind -n myfunction.ali
gnatlink -o testcase test.o myfunction.ali -lgnat
./testcase | diff expected -
$(OPENGEODE) --toAda myfunction.pr system_structure.pr || exit 1
$(ASN1SCC) -Ada dataview-uniq.asn -typePrefix asn1Scc -equal
$(ASN1SCC) -c dataview-uniq.asn -typePrefix asn1Scc
$(GNATMAKE) -c myfunction.adb
$(CC) -c test_ada.c
$(GNATBIND) -n myfunction.ali
$(GNATLINK) -o test_ada test_ada.o myfunction.ali -lgnat
./test_ada | diff expected -
test-llvm:
../../../opengeode.py myfunction.pr system_structure.pr --check --llvm
asn1.exe -c dataview-uniq.asn -equal -typePrefix asn1Scc
llc myfunction.ll
gcc -c myfunction.s
gcc -c dataview-uniq.c
gcc -c test_llvm.c
gcc myfunction.o dataview-uniq.o test_llvm.o -o testcase
./testcase | diff expected -
$(OPENGEODE) myfunction.pr system_structure.pr --check --llvm
$(ASN1SCC) -c dataview-uniq.asn -equal -typePrefix asn1Scc
$(LLC) myfunction.ll
$(CC) -c myfunction.s
$(CC) -c dataview-uniq.c
$(CC) -c test_llvm.c
$(CC) myfunction.o dataview-uniq.o test_llvm.o -o test_llvm -lm
./test_llvm | diff expected -
coverage:
coverage run -p ../../../opengeode.py myfunction.pr system_structure.pr --toAda
clean:
rm -rf *.adb *.ads *.pyc runSpark.sh spark.idx *.o *.ali gnat.cfg examiner \
bin *.wrn *.gpr datav*.? ber.c xer.c asn1crt.? acn.c real.c *.ll *.s testcase
coverage run -p $(OPENGEODE) myfunction.pr system_structure.pr --toAda
.PHONY: all edit test-parse test-ada test-llvm coverage
\ No newline at end of file
tests/test.py → tests/regression/test.py
View file @ b0421901
import os
import subprocess
import sys
import time
tests_folder = os.path.dirname(__file__)
paths = [
'regression/test1',
'regression/test2',
'regression/test3',
'regression/test4',
'regression/test5',
'regression/test6',
'regression/test7',
'regression/test8',
'regression/test9',
'regression/test10',
'regression/test11',
'regression/test12',
'regression/test-controlflow',
'regression/test-expressions',
'regression/test-operators',
'regression/test-substrings',
'regression/test-exitnested',
]
def main():
start = time.time()
results = []
for rule in sys.argv[1:]:
for path in paths:
full_path = os.path.join(tests_folder, path)
result = make(full_path, rule)
make(full_path, 'clean')
results.append(result)
sys.stdout.write('.' if result[0] == 0 else 'F')
sys.stdout.flush()
rule = sys.argv[1]
paths = sys.argv[2:]
for path in paths:
result = make(path, rule)
make(path, 'clean')
results.append(result)
sys.stdout.write('.' if result[0] == 0 else 'F')
sys.stdout.flush()
sys.stdout.write('\n')
......
tests/regression/test1/Makefile
View file @ b0421901
include ../shared.mk
all: test-ada test-llvm
edit:
../../../opengeode.py og.pr system_structure.pr
$(OPENGEODE) og.pr system_structure.pr
test-parse: