Add optimization levels to benchmarks

Makefile

 all: compile-all
 
 test-parse:
-	@$(MAKE) -s -C tests/regression test-parse
+	@$(MAKE) -s -C tests/regression $@
 
 test-ada:
-	@$(MAKE) -s -C tests/regression test-ada
+	@$(MAKE) -s -C tests/regression $@
 
 test-llvm:
-	@$(MAKE) -s -C tests/regression test-llvm
+	@$(MAKE) -s -C tests/regression $@
 
 benchmark:
-	@$(MAKE) -s -C tests/regression benchmark
+	@$(MAKE) -s -C tests/regression $@
+
+benchmark-O1:
+	@$(MAKE) -s -C tests/regression $@
+
+benchmark-O2:
+	@$(MAKE) -s -C tests/regression $@
+
+benchmark-O3:
+	@$(MAKE) -s -C tests/regression $@
 
 coverage:
-	@$(MAKE) -s -C tests/regression coverage
+	@$(MAKE) -s -C tests/regression $@
 
 flake8:
 	@echo Generating flake8_report file
@@ -45,9 +54,9 @@ publish: install
 	@python setup.py sdist upload
 
 clean:
-	@$(MAKE) -s -C tests/regression clean
+	@$(MAKE) -s -C tests/regression $@
 	@find . -name '*~' | xargs rm -f
 	@find . -name '*.o' | xargs rm -f
 
-.PHONY: all test-parse test-ada test-llvm benchmark flake8 coverage \
-	    compile-all install publish clean
+.PHONY: all test-parse test-ada test-llvm benchmark benchmark-O1 benchmark-O2 \
+	    benchmark-O3 flake8 coverage compile-all install publish clean

opengeode.py

@@ -1801,9 +1801,9 @@ def parse_args():
             help='Generate Ada code for the .pr file')
     parser.add_argument('--llvm', dest='llvm', action='store_true',
             help='Generate LLVM IR code for the .pr file (experimental)')
-    parser.add_argument("-O", dest="optimization", metavar="LEVEL", type=int,
+    parser.add_argument("-O", dest="optimization", metavar="level", type=int,
             action="store", choices=[0, 1, 2, 3], default=0,
-            help="Optimize generated LLVM IR code")
+            help="Set optimization level for the generated LLVM IR code")
     parser.add_argument('--png', dest='png', action='store_true',
             help='Generate a PNG file for the process')
     parser.add_argument('--pdf', dest='pdf', action='store_true',

tests/regression/Makefile

@@ -17,6 +17,15 @@ test-llvm:
 benchmark:
 	@python benchmark.py $(BENCHMARK_CASES)
 
+benchmark-O1:
+	@python benchmark.py $(BENCHMARK_CASES) -O1
+
+benchmark-O2:
+	@python benchmark.py $(BENCHMARK_CASES) -O2
+
+benchmark-O3:
+	@python benchmark.py $(BENCHMARK_CASES) -O3
+
 coverage:
 	@for v in $(TEST_CASES) ; \
 		do $(MAKE) -C $$v coverage && mv $$v/.coverage* . || exit 1 ; \
@@ -28,4 +37,5 @@ clean:
 		do $(MAKE) -s -C $$v clean ; \
 	done
 
-.PHONY: test-parse test-ada test-llvm benchmark coverage clean
+.PHONY: test-parse test-ada test-llvm benchmark benchmark-O1 benchmark-O2 \
+	    benchmark-O3 coverage clean

tests/regression/benchmark.py

+import argparse
 import os
 import string
 import subprocess
@@ -8,12 +9,13 @@ from tabulate import tabulate
 
 
 def main():
+    options = parse_args()
+
     start = t.time()
     results = []
-    testfolders = sys.argv[1:]
 
-    for testfolder in testfolders:
-        results.append(benchmark(testfolder))
+    for testfolder in options.testfolders:
+        results.append(benchmark(testfolder, options.optimization))
         make(testfolder, 'clean')
         sys.stdout.write('.')
         sys.stdout.flush()
@@ -21,16 +23,26 @@ def main():
     print ""
 
     elapsed = t.time() - start
-    sys.exit(summarize(results, elapsed))
+    sys.exit(summarize(results, elapsed, options))
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-O", dest="optimization", metavar="level", type=int,
+            action="store", choices=[0, 1, 2, 3], default=0,
+            help="set optimization level")
+    parser.add_argument('testfolders', metavar='testfolder', type=str, nargs='*',
+            help='test folder(s)')
+    return parser.parse_args()
 
 
-def benchmark(testfolder):
+def benchmark(testfolder, opt_level):
     result = {
         "name": testfolder[:-1],
     }
 
     for rule in ("test-llvm", "test-ada"):
-        if make(testfolder, rule) != 0:
+        if make(testfolder, rule, "O=%d" % opt_level) != 0:
             result["status"] = "ERROR"
             return result
 
@@ -68,7 +80,7 @@ def time(file, iters=1000):
     return (t.time() - start) / iters
 
 
-def summarize(results, elapsed):
+def summarize(results, elapsed, options):
     print ""
     print "Summary"
     print "-------"
@@ -94,6 +106,9 @@ def summarize(results, elapsed):
         print "No results"
         return 1
 
+    if options.optimization != 0:
+        print "Optimization level: %d" % options.optimization
+        print ""
     print "Size: Ada %.2f%% LLVM %.2f%%" % diff([r["size"] for r in valid_results])
     print "Time: Ada %.2f%% LLVM %.2f%%" % diff([r["time"] for r in valid_results])
     print ""
@@ -122,8 +137,10 @@ def mean(values):
     return sum(values) / len(values)
 
 
-def make(path, rule):
-    return call(["make", "-C", path, rule])[0]
+def make(path, rule, *args):
+    call_args = ["make", "-C", path, rule]
+    call_args.extend(args)
+    return call(call_args)[0]
 
 
 def call(args):

tests/regression/shared.mk

@@ -5,6 +5,7 @@ LLC=llc
 GNATMAKE=gnatmake
 GNATBIND=gnatbind
 GNATLINK=gnatlink
+O=0
 
 clean:
 	rm -rf *.adb *.ads *.pyc runSpark.sh spark.idx *.o *.ali gnat.cfg \
@@ -13,21 +14,21 @@ clean:
 	       *.autosave
 
 %.o: %.pr FORCE
-	$(OPENGEODE) $< system_structure.pr --llvm
+	$(OPENGEODE) $< system_structure.pr --llvm -O$(O)
 	$(LLC) $*.ll
-	$(CC) -c $*.s
+	$(CC) -O$(O) -c $*.s
 
 %.ali: %.pr FORCE
 	$(OPENGEODE) $< system_structure.pr --toAda
 	$(ASN1SCC) -Ada dataview-uniq.asn -typePrefix asn1Scc -equal
-	$(GNATMAKE) -c *.adb
+	$(GNATMAKE) -O$(O) -c *.adb
 
 %.o: %.asn FORCE
 	$(ASN1SCC) -c $< -typePrefix asn1Scc -equal
-	$(CC) -c $*.c
+	$(CC) -O$(O) -c $*.c
 
 %.o: %.c FORCE
-	$(CC) -c $<
+	$(CC) -O$(O) -c $<
 
 FORCE: