regression.py 4.9 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
#!/usr/bin/env python3
"""
This uses all available cores to spawn processes that build
all test projects (Demo_*/)
"""
import os
import sys
import time
import signal
import multiprocessing


# Tell the orchestrator to..
#
# - stop complaining about POHI issues under multicores
os.putenv("DISABLE_MULTICORE_CHECK", "1")

# - immediately abort in case of error (No "Hit ENTER...")
os.putenv("CLEANUP", "1")


def get_tests():
    """Collect all Demo_... folders not excluded via 'NOTEST' """
Thanassis Tsiodras's avatar
Thanassis Tsiodras committed
24
    for folder in sorted(os.listdir(".")):
25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40
        if all([folder.startswith('Demo'),
                not os.path.exists(folder + "/NOTEST")]):
            yield folder


def build_me(folder):
    """
    Performs a build in the passed-in folder, and also spawns regression.py
    (if it exists in that folder).

    Spawned from a multiprocessing pool, returns the result back in tuple form
    for asynchronous reporting.
    """
    root = os.getcwd()
    os.chdir(folder)
    try:
41 42 43 44 45
        if os.getenv("CIRCLECI") is not None:
            # CircleCI times out if it doesn't see stdout activity after 10min...
            res = os.system("bash -c './bui*.sh 2>&1 | tee build.log ; exit ${PIPESTATUS[0]}'")
        else:
            res = os.system("./bui*.sh > build.log 2>&1 || exit 1")
Thanassis Tsiodras's avatar
Thanassis Tsiodras committed
46
        if os.getenv("CIRCLE_ARTIFACTS") is not None:
47
            os.system("cp build.log $CIRCLE_ARTIFACTS/" + folder + "_build.log")
48 49 50 51
        if res != 0:
            return folder, False, "Build failed..."
        if os.path.exists("regression.py"):
            res = os.system("./regression.py > regression.log 2>&1 || exit 1")
52 53
            if os.getenv("CIRCLE_ARTIFACTS") is not None:
                os.system("cp regression.log $CIRCLE_ARTIFACTS/" + folder + "_regression.log")
54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72
            if res != 0:
                return folder, False, "Regression check failed..."
        return folder, True, "OK"
    finally:
        os.chdir(root)


def color_me(result, msg):
    """Color the output according to the result."""
    if sys.stdout.isatty():
        msg = chr(27) + "[3" + (
            "2" if result else "1") + "m" + msg + chr(27) + "[0m"
    return msg


def main():
    """Run the TASTE regression-checking project suite"""
    cwd = os.path.dirname(os.path.realpath(__file__))

73 74 75
    # Stop Maxime's GUI progress bar from appearing
    os.putenv("DISABLE_PROGRESS_BAR", "1")

76 77 78
    # Navigate to the folder hosting all test projects (i.e. "Demo_*/")
    os.chdir(cwd)

79 80
    os.system("rm -rf Demo*/binary*")

81
    # Detect how many workers we will use
82 83 84 85
    if os.getenv("CIRCLECI") is not None:
        total_cpus = 1
    else:
        total_cpus = multiprocessing.cpu_count()
86 87 88 89 90
    total_tests = len(list(get_tests()))

    # Measure time to execute
    start_time = time.time()

91
    if os.getenv("CIRCLECI") is not None:
92 93
        failed_tests = 0
        run_tests = 0
94
        msgs = []
95 96
        for tst in get_tests():
            res = build_me(tst)
97 98
            run_tests += 1
            test, result, msg = res
99 100
            msgs.append((test, result, msg))
            # print("%40s: %s" % (test, color_me(result, msg)))
101 102
            if not result:
                failed_tests += 1
103 104 105 106 107

        # Print summary of execution at the end
        for res in msgs:
            test, result, msg = res
            print("%40s: %s" % (test, color_me(result, msg)))
108
    else:
109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137
        print("Spawning", total_cpus, "workers to run", total_tests, "tests...\n")
        pool = multiprocessing.Pool(total_cpus)
        try:
            failed_tests = 0
            run_tests = 0
            results = pool.imap_unordered(build_me, get_tests())
            for res in results:
                run_tests += 1
                test, result, msg = res
                print("%40s: %s" % (test, color_me(result, msg)))
                if not result:
                    failed_tests += 1
        except KeyboardInterrupt:
            # The devil is always in the details...  This is the only
            # way I found to propagate the SIGINT to the builds done
            # by the worker processes of the pool AND immediately
            # terminate. Numerous people cry about it on the web...
            # ...and I had to look at the code of concurrent.futures
            # (which I abandoned), multiprocessing/pool.py,
            # multiprocessing/process.py AND subprocess.py to see
            # what I had to do. Gotta love our profession.
            for job in pool._pool:  # pylint: disable=W0212
                os.kill(job.pid, signal.SIGINT)
                job.terminate()
            pool.terminate()
            pool.join()
        else:
            pool.close()
            pool.join()
138 139 140 141

    print("\nFinished in %d seconds." % (time.time() - start_time))
    print(run_tests, "tests,", failed_tests, "failure" + (
        "s" if (failed_tests > 1 or failed_tests == 0) else "") + ".")
142
    sys.exit(0 if failed_tests == 0 else 1)
143 144 145 146 147 148


if __name__ == "__main__":
    main()

# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4