benchmark.py 3.16 KB
Newer Older
dbarbera's avatar
dbarbera committed
1
import os
2
import string
dbarbera's avatar
dbarbera committed
3
4
5
6
import subprocess
import sys
import time as t

dbarbera's avatar
dbarbera committed
7
8
from tabulate import tabulate

dbarbera's avatar
dbarbera committed
9
10
11
12

def main():
    start = t.time()
    results = []
13
14
15
16
    testfolders = sys.argv[1:]

    for testfolder in testfolders:
        results.append(benchmark(testfolder))
dbarbera's avatar
dbarbera committed
17
        make(testfolder, 'clean')
dbarbera's avatar
dbarbera committed
18
19
20
        sys.stdout.write('.')
        sys.stdout.flush()

21
    print ""
dbarbera's avatar
dbarbera committed
22
23

    elapsed = t.time() - start
24
    sys.exit(summarize(results, elapsed))
dbarbera's avatar
dbarbera committed
25
26
27


def benchmark(testfolder):
28
29
30
31
    result = {
        "name": testfolder[:-1],
    }

dbarbera's avatar
dbarbera committed
32
33
    for rule in ("test-llvm", "test-ada"):
        if make(testfolder, rule) != 0:
34
35
            result["status"] = "ERROR"
            return result
dbarbera's avatar
dbarbera committed
36
37
38
39
40
41

    llvm_bin = os.path.join(testfolder, "test_ada")
    ada_bin = os.path.join(testfolder, "test_llvm")

    for bin_name in (llvm_bin, ada_bin):
        if not os.path.isfile(bin_name):
42
43
            result["status"] = "ERROR"
            return result
dbarbera's avatar
dbarbera committed
44

45
46
    result.update({
        "status": "OK",
dbarbera's avatar
dbarbera committed
47
        "size": {
dbarbera's avatar
dbarbera committed
48
49
            "ada": size(llvm_bin),
            "llvm": size(ada_bin),
dbarbera's avatar
dbarbera committed
50
51
52
53
54
        },
        "time": {
            "ada": time(ada_bin),
            "llvm": time(llvm_bin),
        }
55
    })
dbarbera's avatar
dbarbera committed
56

dbarbera's avatar
dbarbera committed
57
58
    return result

dbarbera's avatar
dbarbera committed
59

dbarbera's avatar
dbarbera committed
60
61
62
63
64
def size(file):
    call(["strip", os.path.abspath(file)])
    return os.path.getsize(file)


dbarbera's avatar
dbarbera committed
65
66
67
68
69
70
def time(file, iters=1000):
    start = t.time()
    call(["/bin/bash", "-c", "for i in {1..%s} ; do %s ; done" % (iters, file)])
    return (t.time() - start) / iters


71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
def summarize(results, elapsed):
    print ""
    print "Summary"
    print "-------"
    print ""

    max_name_len = max([len(r["name"]) for r in results]) + 3
    num_errors = 0
    valid_results = []

    for r in results:
        print "%s [%s]" % (string.ljust(r["name"], max_name_len, '.'), r["status"])
        if r["status"] == "OK":
            valid_results.append(r)
        else:
            num_errors += 1

    print ""
dbarbera's avatar
dbarbera committed
89
    print "Finished in %.3fs" % elapsed
90
91
    print "%s benchmarks, %s errors" % (len(results), num_errors)
    print ""
dbarbera's avatar
dbarbera committed
92

93
    if not valid_results:
dbarbera's avatar
dbarbera committed
94
        print "No results"
dbarbera's avatar
dbarbera committed
95
        return 1
dbarbera's avatar
dbarbera committed
96

97
98
    print "Size: Ada %.2f%% LLVM %.2f%%" % diff([r["size"] for r in valid_results])
    print "Time: Ada %.2f%% LLVM %.2f%%" % diff([r["time"] for r in valid_results])
dbarbera's avatar
dbarbera committed
99
100
    print ""

101
    headers = ["Benchmark", "Ada size (B)", "LLVM size (B)", "Ada time (us)", "LLVM time (us)"]
dbarbera's avatar
dbarbera committed
102
    table = []
103
    for r in valid_results:
104
        table.append([
105
106
107
108
109
            r["name"],
            r["size"]["ada"],
            r["size"]["llvm"],
            int(round(r["time"]["ada"] * (10 ** 6))),
            int(round(r["time"]["llvm"] * (10 ** 6))),
110
111
112
        ])

    print tabulate(table, headers, tablefmt="orgtbl")
dbarbera's avatar
dbarbera committed
113

114
    return 0 if num_errors == 0 else 1
dbarbera's avatar
dbarbera committed
115

dbarbera's avatar
dbarbera committed
116

dbarbera's avatar
dbarbera committed
117
118
def diff(results):
    return 100, mean(([float(r["llvm"]) / float(r["ada"]) * 100 for r in results]))
dbarbera's avatar
dbarbera committed
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141


def mean(values):
    return sum(values) / len(values)


def make(path, rule):
    return call(["make", "-C", path, rule])[0]


def call(args):
    proc = subprocess.Popen(
        args,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
    )
    stdout, stderr = proc.communicate()
    errcode = proc.wait()
    return (errcode, stdout, stderr)


if __name__ == '__main__':
    main()