regression.py 4.99 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
#!/usr/bin/env python3
"""
This uses all available cores to spawn processes that build
all test projects (Demo_*/)
"""
import os
import sys
import time
import signal
import multiprocessing


# Tell the orchestrator to..
#
# - stop complaining about POHI issues under multicores
os.putenv("DISABLE_MULTICORE_CHECK", "1")

# - immediately abort in case of error (No "Hit ENTER...")
os.putenv("CLEANUP", "1")


def get_tests():
    """Collect all Demo_... folders not excluded via 'NOTEST' """
Thanassis Tsiodras's avatar
Thanassis Tsiodras committed
24
    for folder in sorted(os.listdir(".")):
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
        if all([folder.startswith('Demo'),
                not os.path.exists(folder + "/NOTEST")]):
            yield folder


def build_me(folder):
    """
    Performs a build in the passed-in folder, and also spawns regression.py
    (if it exists in that folder).

    Spawned from a multiprocessing pool, returns the result back in tuple form
    for asynchronous reporting.
    """
    root = os.getcwd()
    os.chdir(folder)
    try:
41
42
43
44
45
        if os.getenv("CIRCLECI") is not None:
            # CircleCI times out if it doesn't see stdout activity after 10min...
            res = os.system("bash -c './bui*.sh 2>&1 | tee build.log ; exit ${PIPESTATUS[0]}'")
        else:
            res = os.system("./bui*.sh > build.log 2>&1 || exit 1")
Thanassis Tsiodras's avatar
Typo    
Thanassis Tsiodras committed
46
        if os.getenv("CIRCLE_ARTIFACTS") is not None:
47
            os.system("cp build.log $CIRCLE_ARTIFACTS/" + folder + "_build.log")
48
49
        if res != 0:
            return folder, False, "Build failed..."
50
51
52
53
54
55
56
57
58
59
        for cmd in ["./regression.sh", "./regression.py"]:
            if os.path.exists(cmd):
                break
        else:
            return folder, True, "OK"
        res = os.system(cmd + " > regression.log 2>&1 || exit 1")
        if os.getenv("CIRCLE_ARTIFACTS") is not None:
            os.system("cp regression.log $CIRCLE_ARTIFACTS/" + folder + "_regression.log")
        if res != 0:
            return folder, False, "Regression check failed..."
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
        return folder, True, "OK"
    finally:
        os.chdir(root)


def color_me(result, msg):
    """Color the output according to the result."""
    if sys.stdout.isatty():
        msg = chr(27) + "[3" + (
            "2" if result else "1") + "m" + msg + chr(27) + "[0m"
    return msg


def main():
    """Run the TASTE regression-checking project suite"""
    cwd = os.path.dirname(os.path.realpath(__file__))

77
78
79
    # Stop Maxime's GUI progress bar from appearing
    os.putenv("DISABLE_PROGRESS_BAR", "1")

80
81
82
    # Navigate to the folder hosting all test projects (i.e. "Demo_*/")
    os.chdir(cwd)

83
84
    os.system("rm -rf Demo*/binary*")

85
    # Detect how many workers we will use
86
87
88
89
    if os.getenv("CIRCLECI") is not None:
        total_cpus = 1
    else:
        total_cpus = multiprocessing.cpu_count()
90
91
92
93
94
    total_tests = len(list(get_tests()))

    # Measure time to execute
    start_time = time.time()

95
    if os.getenv("CIRCLECI") is not None:
96
97
        failed_tests = 0
        run_tests = 0
98
        msgs = []
99
100
        for tst in get_tests():
            res = build_me(tst)
101
102
            run_tests += 1
            test, result, msg = res
103
104
            msgs.append((test, result, msg))
            # print("%40s: %s" % (test, color_me(result, msg)))
105
106
            if not result:
                failed_tests += 1
107
108
109
110
111

        # Print summary of execution at the end
        for res in msgs:
            test, result, msg = res
            print("%40s: %s" % (test, color_me(result, msg)))
112
    else:
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
        print("Spawning", total_cpus, "workers to run", total_tests, "tests...\n")
        pool = multiprocessing.Pool(total_cpus)
        try:
            failed_tests = 0
            run_tests = 0
            results = pool.imap_unordered(build_me, get_tests())
            for res in results:
                run_tests += 1
                test, result, msg = res
                print("%40s: %s" % (test, color_me(result, msg)))
                if not result:
                    failed_tests += 1
        except KeyboardInterrupt:
            # The devil is always in the details...  This is the only
            # way I found to propagate the SIGINT to the builds done
            # by the worker processes of the pool AND immediately
            # terminate. Numerous people cry about it on the web...
            # ...and I had to look at the code of concurrent.futures
            # (which I abandoned), multiprocessing/pool.py,
            # multiprocessing/process.py AND subprocess.py to see
            # what I had to do. Gotta love our profession.
            for job in pool._pool:  # pylint: disable=W0212
                os.kill(job.pid, signal.SIGINT)
                job.terminate()
            pool.terminate()
            pool.join()
        else:
            pool.close()
            pool.join()
142
143
144
145

    print("\nFinished in %d seconds." % (time.time() - start_time))
    print(run_tests, "tests,", failed_tests, "failure" + (
        "s" if (failed_tests > 1 or failed_tests == 0) else "") + ".")
146
    sys.exit(0 if failed_tests == 0 else 1)
147
148
149
150
151
152


if __name__ == "__main__":
    main()

# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4