run-tests 20.8 KB
Newer Older
1
#! /usr/bin/env python3
Markus Siemens's avatar
Markus Siemens committed
2
3
4
5

import os
import subprocess
import sys
6
import platform
7
import argparse
8
import re
Markus Siemens's avatar
Markus Siemens committed
9
10
from glob import glob

11
12
13
# Tests require at least CPython 3.3. If your default python3 executable
# is of lower version, you can point MICROPY_CPYTHON3 environment var
# to the correct executable.
Markus Siemens's avatar
Markus Siemens committed
14
if os.name == 'nt':
15
    CPYTHON3 = os.getenv('MICROPY_CPYTHON3', 'python3.exe')
16
    MICROPYTHON = os.getenv('MICROPY_MICROPYTHON', '../windows/micropython.exe')
Markus Siemens's avatar
Markus Siemens committed
17
else:
18
    CPYTHON3 = os.getenv('MICROPY_CPYTHON3', 'python3')
19
    MICROPYTHON = os.getenv('MICROPY_MICROPYTHON', '../unix/micropython')
Markus Siemens's avatar
Markus Siemens committed
20

21
22
23
# mpy-cross is only needed if --via-mpy command-line arg is passed
MPYCROSS = os.getenv('MICROPY_MPYCROSS', '../mpy-cross/mpy-cross')

24
# Set PYTHONIOENCODING so that CPython will use utf-8 on systems which set another encoding in the locale
Paul Sokolovsky's avatar
Paul Sokolovsky committed
25
os.environ['PYTHONIOENCODING'] = 'utf-8'
26

27
28
29
30
def rm_f(fname):
    if os.path.exists(fname):
        os.remove(fname)

31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51

# unescape wanted regex chars and escape unwanted ones
def convert_regex_escapes(line):
    cs = []
    escape = False
    for c in str(line, 'utf8'):
        if escape:
            escape = False
            cs.append(c)
        elif c == '\\':
            escape = True
        elif c in ('(', ')', '[', ']', '{', '}', '.', '*', '+', '^', '$'):
            cs.append('\\' + c)
        else:
            cs.append(c)
    # accept carriage-return(s) before final newline
    if cs[-1] == '\n':
        cs[-1] = '\r*\n'
    return bytes(''.join(cs), 'utf8')


52
def run_micropython(pyb, args, test_file):
53
    special_tests = ('micropython/meminfo.py', 'basics/bytes_compare3.py', 'basics/builtin_help.py', 'thread/thread_exc2.py')
54
    is_special = False
55
56
    if pyb is None:
        # run on PC
57
        if test_file.startswith(('cmdline/', 'feature_check/')) or test_file in special_tests:
58
            # special handling for tests of the unix cmdline program
59
            is_special = True
60
61
62
63
64
65

            # check for any cmdline options needed for this test
            args = [MICROPYTHON]
            with open(test_file, 'rb') as f:
                line = f.readline()
                if line.startswith(b'# cmdline:'):
66
67
                    # subprocess.check_output on Windows only accepts strings, not bytes
                    args += [str(c, 'utf-8') for c in line[10:].strip().split()]
68
69
70

            # run the test, possibly with redirected input
            try:
71
                if 'repl_' in test_file:
72
                    # Need to use a PTY to test command line editing
73
74
75
76
77
                    try:
                        import pty
                    except ImportError:
                        # in case pty module is not available, like on Windows
                        return b'SKIP\n'
78
79
                    import select

80
                    def get(required=False):
81
82
83
84
85
86
                        rv = b''
                        while True:
                            ready = select.select([master], [], [], 0.02)
                            if ready[0] == [master]:
                                rv += os.read(master, 1024)
                            else:
87
88
                                if not required or rv:
                                    return rv
89
90
91
92
93
94
95
96
97
98

                    def send_get(what):
                        os.write(master, what)
                        return get()

                    with open(test_file, 'rb') as f:
                        # instead of: output_mupy = subprocess.check_output(args, stdin=f)
                        master, slave = pty.openpty()
                        p = subprocess.Popen(args, stdin=slave, stdout=slave,
                                             stderr=subprocess.STDOUT, bufsize=0)
99
                        banner = get(True)
100
                        output_mupy = banner + b''.join(send_get(line) for line in f)
101
                        send_get(b'\x04') # exit the REPL, so coverage info is saved
102
103
104
                        p.kill()
                        os.close(master)
                        os.close(slave)
105
106
107
                else:
                    output_mupy = subprocess.check_output(args + [test_file])
            except subprocess.CalledProcessError:
108
                return b'CRASH'
109
110

        else:
111
112
113
114
115
116
117
118
119
120
121
122
            # a standard test run on PC

            # create system command
            cmdlist = [MICROPYTHON, '-X', 'emit=' + args.emit]
            if args.heapsize is not None:
                cmdlist.extend(['-X', 'heapsize=' + args.heapsize])

            # if running via .mpy, first compile the .py file
            if args.via_mpy:
                subprocess.check_output([MPYCROSS, '-mcache-lookup-bc', '-o', 'mpytest.mpy', test_file])
                cmdlist.extend(['-m', 'mpytest'])
            else:
123
                cmdlist.append(test_file)
124
125
126

            # run the actual test
            try:
127
                output_mupy = subprocess.check_output(cmdlist)
128
129
            except subprocess.CalledProcessError:
                output_mupy = b'CRASH'
130
131
132
133
134

            # clean up if we had an intermediate .mpy file
            if args.via_mpy:
                rm_f('mpytest.mpy')

135
136
137
138
139
    else:
        # run on pyboard
        import pyboard
        pyb.enter_raw_repl()
        try:
140
            output_mupy = pyb.execfile(test_file)
141
142
143
        except pyboard.PyboardError:
            output_mupy = b'CRASH'

144
145
146
    # canonical form for all ports/platforms is to use \n for end-of-line
    output_mupy = output_mupy.replace(b'\r\n', b'\n')

147
148
149
150
    # don't try to convert the output if we should skip this test
    if output_mupy == b'SKIP\n':
        return output_mupy

151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
    if is_special or test_file in special_tests:
        # convert parts of the output that are not stable across runs
        with open(test_file + '.exp', 'rb') as f:
            lines_exp = []
            for line in f.readlines():
                if line == b'########\n':
                    line = (line,)
                else:
                    line = (line, re.compile(convert_regex_escapes(line)))
                lines_exp.append(line)
        lines_mupy = [line + b'\n' for line in output_mupy.split(b'\n')]
        if output_mupy.endswith(b'\n'):
            lines_mupy = lines_mupy[:-1] # remove erroneous last empty line
        i_mupy = 0
        for i in range(len(lines_exp)):
            if lines_exp[i][0] == b'########\n':
                # 8x #'s means match 0 or more whole lines
                line_exp = lines_exp[i + 1]
                skip = 0
                while i_mupy + skip < len(lines_mupy) and not line_exp[1].match(lines_mupy[i_mupy + skip]):
                    skip += 1
                if i_mupy + skip >= len(lines_mupy):
                    lines_mupy[i_mupy] = b'######## FAIL\n'
                    break
                del lines_mupy[i_mupy:i_mupy + skip]
                lines_mupy.insert(i_mupy, b'########\n')
                i_mupy += 1
            else:
                # a regex
                if lines_exp[i][1].match(lines_mupy[i_mupy]):
                    lines_mupy[i_mupy] = lines_exp[i][0]
                else:
                    #print("don't match: %r %s" % (lines_exp[i][1], lines_mupy[i_mupy])) # DEBUG
                    pass
                i_mupy += 1
            if i_mupy >= len(lines_mupy):
                break
        output_mupy = b''.join(lines_mupy)

190
191
    return output_mupy

192
def run_tests(pyb, tests, args):
193
194
195
196
    test_count = 0
    testcase_count = 0
    passed_count = 0
    failed_tests = []
197
    skipped_tests = []
198

199
    skip_tests = set()
200
    skip_native = False
201
    skip_int_big = False
202
    skip_set_type = False
203
    skip_async = False
204

205
    # Check if micropython.native is supported, and skip such tests if it's not
206
    native = run_micropython(pyb, args, 'feature_check/native_check.py')
207
    if native == b'CRASH':
208
        skip_native = True
209

210
211
    # Check if arbitrary-precision integers are supported, and skip such tests if it's not
    native = run_micropython(pyb, args, 'feature_check/int_big.py')
212
    if native != b'1000000000000000000000000000000000000000000000\n':
213
214
        skip_int_big = True

215
216
217
218
219
    # Check if set type (and set literals) is supported, and skip such tests if it's not
    native = run_micropython(pyb, args, 'feature_check/set_check.py')
    if native == b'CRASH':
        skip_set_type = True

220
221
222
223
224
    # Check if async/await keywords are supported, and skip such tests if it's not
    native = run_micropython(pyb, args, 'feature_check/async_check.py')
    if native == b'CRASH':
        skip_async = True

225
    # Check if emacs repl is supported, and skip such tests if it's not
226
    t = run_micropython(pyb, args, 'feature_check/repl_emacs_check.py')
227
228
229
    if not 'True' in str(t, 'ascii'):
        skip_tests.add('cmdline/repl_emacs_keys.py')

230
    upy_byteorder = run_micropython(pyb, args, 'feature_check/byteorder.py')
231
    has_complex = run_micropython(pyb, args, 'feature_check/complex.py') == b'complex\n'
232
    has_coverage = run_micropython(pyb, args, 'feature_check/coverage.py') == b'coverage\n'
233
    cpy_byteorder = subprocess.check_output([CPYTHON3, 'feature_check/byteorder.py'])
234
235
    skip_endian = (upy_byteorder != cpy_byteorder)

236
237
238
    # Some tests shouldn't be run under Travis CI
    if os.getenv('TRAVIS') == 'true':
        skip_tests.add('basics/memoryerror.py')
239
        skip_tests.add('thread/thread_gc1.py') # has reliability issues
240
        skip_tests.add('thread/thread_lock4.py') # has reliability issues
241
        skip_tests.add('thread/stress_heap.py') # has reliability issues
242
        skip_tests.add('thread/stress_recurse.py') # has reliability issues
243

244
245
246
247
248
249
    if not has_complex:
        skip_tests.add('float/complex1.py')
        skip_tests.add('float/int_big_float.py')
        skip_tests.add('float/true_value.py')
        skip_tests.add('float/types.py')

250
251
252
    if not has_coverage:
        skip_tests.add('cmdline/cmd_parsetree.py')

253
254
255
256
257
258
259
    # Some tests shouldn't be run on a PC
    if pyb is None:
        # unix build does not have the GIL so can't run thread mutation tests
        for t in tests:
            if t.startswith('thread/mutate_'):
                skip_tests.add(t)

260
261
    # Some tests shouldn't be run on pyboard
    if pyb is not None:
262
        skip_tests.add('basics/exception_chain.py') # warning is not printed
263
        skip_tests.add('float/float_divmod.py') # tested by float/float_divmod_relaxed.py instead
264
        skip_tests.add('float/float2int_doubleprec.py') # requires double precision floating point to work
265
        skip_tests.add('micropython/meminfo.py') # output is very different to PC output
266
        skip_tests.add('extmod/machine_mem.py') # raw memory access not supported
267

268
269
270
271
272
273
        if args.target == 'wipy':
            skip_tests.add('misc/print_exception.py')       # requires error reporting full
            skip_tests.add('misc/recursion.py')             # requires stack checking enabled
            skip_tests.add('misc/recursive_data.py')        # requires stack checking enabled
            skip_tests.add('misc/recursive_iternext.py')    # requires stack checking enabled
            skip_tests.add('misc/rge_sm.py')                # requires floating point
274
            skip_tests.update({'extmod/uctypes_%s.py' % t for t in 'bytearray le native_le ptr_le ptr_native_le sizeof sizeof_native array_assign_le array_assign_native_le'.split()}) # requires uctypes
275
            skip_tests.add('extmod/zlibd_decompress.py')    # requires zlib
276
277
            skip_tests.add('extmod/ujson_dumps_float.py')   # requires floating point
            skip_tests.add('extmod/ujson_loads_float.py')   # requires floating point
278
            skip_tests.add('extmod/uheapq1.py')             # uheapq not supported by WiPy
279
280
            skip_tests.add('extmod/urandom_basic.py')       # requires urandom
            skip_tests.add('extmod/urandom_extra.py')       # requires urandom
281
282
283
284
285
286
        elif args.target == 'esp8266':
            skip_tests.add('float/float2int.py')            # requires at least fp32, there's float2int_fp30.py instead
            skip_tests.add('float/string_format.py')        # requires at least fp32, there's string_format_fp30.py instead
            skip_tests.add('float/bytes_construct.py')      # requires fp32
            skip_tests.add('float/bytearray_construct.py')  # requires fp32
            skip_tests.add('misc/rge_sm.py')                # too large
287

288
289
    # Some tests are known to fail on 64-bit machines
    if pyb is None and platform.architecture()[0] == '64bit':
290
        pass
291

292
293
    # Some tests use unsupported features on Windows
    if os.name == 'nt':
294
        skip_tests.add('import/import_file.py') # works but CPython prints forward slashes
295

296
297
298
    # Some tests are known to fail with native emitter
    # Remove them from the below when they work
    if args.emit == 'native':
299
        skip_tests.update({'basics/%s.py' % t for t in 'gen_yield_from gen_yield_from_close gen_yield_from_ducktype gen_yield_from_exc gen_yield_from_iter gen_yield_from_send gen_yield_from_stopped gen_yield_from_throw gen_yield_from_throw2 gen_yield_from_throw3 generator1 generator2 generator_args generator_close generator_closure generator_exc generator_return generator_send'.split()}) # require yield
300
        skip_tests.update({'basics/%s.py' % t for t in 'bytes_gen class_store_class globals_del string_join'.split()}) # require yield
301
        skip_tests.update({'basics/async_%s.py' % t for t in 'def await await2 for for2 with with2'.split()}) # require yield
302
        skip_tests.update({'basics/%s.py' % t for t in 'try_reraise try_reraise2'.split()}) # require raise_varargs
303
        skip_tests.update({'basics/%s.py' % t for t in 'with_break with_continue with_return'.split()}) # require complete with support
304
305
        skip_tests.add('basics/array_construct2.py') # requires generators
        skip_tests.add('basics/bool1.py') # seems to randomly fail
306
        skip_tests.add('basics/class_bind_self.py') # requires yield
307
308
        skip_tests.add('basics/del_deref.py') # requires checking for unbound local
        skip_tests.add('basics/del_local.py') # requires checking for unbound local
309
        skip_tests.add('basics/exception_chain.py') # raise from is not supported
310
        skip_tests.add('basics/for_range.py') # requires yield_value
311
312
        skip_tests.add('basics/try_finally_loops.py') # requires proper try finally code
        skip_tests.add('basics/try_finally_return.py') # requires proper try finally code
313
        skip_tests.add('basics/try_finally_return2.py') # requires proper try finally code
314
315
316
317
        skip_tests.add('basics/unboundlocal.py') # requires checking for unbound local
        skip_tests.add('import/gen_context.py') # requires yield_value
        skip_tests.add('misc/features.py') # requires raise_varargs
        skip_tests.add('misc/rge_sm.py') # requires yield
318
        skip_tests.add('misc/print_exception.py') # because native doesn't have proper traceback info
319
        skip_tests.add('misc/sys_exc_info.py') # sys.exc_info() is not supported for native
320
        skip_tests.add('micropython/heapalloc_traceback.py') # because native doesn't have proper traceback info
321
        skip_tests.add('micropython/heapalloc_iter.py') # requires generators
322

323
    for test_file in tests:
324
        test_file = test_file.replace('\\', '/')
325
326
        test_basename = os.path.basename(test_file)
        test_name = os.path.splitext(test_basename)[0]
327
        is_native = test_name.startswith("native_") or test_name.startswith("viper_")
328
        is_endian = test_name.endswith("_endian")
329
        is_int_big = test_name.startswith("int_big") or test_name.endswith("_intbig")
330
        is_set_type = test_name.startswith("set_") or test_name.startswith("frozenset")
331
        is_async = test_name.startswith("async_")
332
333
334
335

        skip_it = test_file in skip_tests
        skip_it |= skip_native and is_native
        skip_it |= skip_endian and is_endian
336
        skip_it |= skip_int_big and is_int_big
337
        skip_it |= skip_set_type and is_set_type
338
        skip_it |= skip_async and is_async
339

340
        if skip_it:
341
            print("skip ", test_file)
342
            skipped_tests.append(test_name)
343
344
345
346
347
348
349
350
351
            continue

        # get expected output
        test_file_expected = test_file + '.exp'
        if os.path.isfile(test_file_expected):
            # expected output given by a file, so read that in
            with open(test_file_expected, 'rb') as f:
                output_expected = f.read()
        else:
352
            # run CPython to work out expected output
353
354
            try:
                output_expected = subprocess.check_output([CPYTHON3, '-B', test_file])
355
356
357
                if args.write_exp:
                    with open(test_file_expected, 'wb') as f:
                        f.write(output_expected)
358
359
360
            except subprocess.CalledProcessError:
                output_expected = b'CPYTHON3 CRASH'

361
362
363
        # canonical form for all host platforms is to use \n for end-of-line
        output_expected = output_expected.replace(b'\r\n', b'\n')

364
365
366
        if args.write_exp:
            continue

367
        # run Micro Python
368
        output_mupy = run_micropython(pyb, args, test_file)
369

370
        if output_mupy == b'SKIP\n':
371
372
373
374
375
376
            print("skip ", test_file)
            skipped_tests.append(test_name)
            continue

        testcase_count += len(output_expected.splitlines())

377
378
379
380
381
382
383
384
385
        filename_expected = test_basename + ".exp"
        filename_mupy = test_basename + ".out"

        if output_expected == output_mupy:
            print("pass ", test_file)
            passed_count += 1
            rm_f(filename_expected)
            rm_f(filename_mupy)
        else:
386
387
388
389
            with open(filename_expected, "wb") as f:
                f.write(output_expected)
            with open(filename_mupy, "wb") as f:
                f.write(output_mupy)
390
391
392
393
394
395
396
397
            print("FAIL ", test_file)
            failed_tests.append(test_name)

        test_count += 1

    print("{} tests performed ({} individual testcases)".format(test_count, testcase_count))
    print("{} tests passed".format(passed_count))

398
399
    if len(skipped_tests) > 0:
        print("{} tests skipped: {}".format(len(skipped_tests), ' '.join(skipped_tests)))
400
401
402
403
404
405
406
407
    if len(failed_tests) > 0:
        print("{} tests failed: {}".format(len(failed_tests), ' '.join(failed_tests)))
        return False

    # all tests succeeded
    return True

def main():
408
    cmd_parser = argparse.ArgumentParser(description='Run tests for MicroPython.')
409
    cmd_parser.add_argument('--target', default='unix', help='the target platform')
410
411
412
413
    cmd_parser.add_argument('--device', default='/dev/ttyACM0', help='the serial device or the IP address of the pyboard')
    cmd_parser.add_argument('-b', '--baudrate', default=115200, help='the baud rate of the serial device')
    cmd_parser.add_argument('-u', '--user', default='micro', help='the telnet login username')
    cmd_parser.add_argument('-p', '--password', default='python', help='the telnet login password')
414
    cmd_parser.add_argument('-d', '--test-dirs', nargs='*', help='input test directories (if no files given)')
415
    cmd_parser.add_argument('--write-exp', action='store_true', help='save .exp files to run tests w/o CPython')
416
    cmd_parser.add_argument('--emit', default='bytecode', help='MicroPython emitter to use (bytecode or native)')
417
    cmd_parser.add_argument('--heapsize', help='heapsize to use (use default if not specified)')
418
    cmd_parser.add_argument('--via-mpy', action='store_true', help='compile .py files to .mpy first')
419
    cmd_parser.add_argument('--keep-path', action='store_true', help='do not clear MICROPYPATH when running tests')
420
421
422
    cmd_parser.add_argument('files', nargs='*', help='input test files')
    args = cmd_parser.parse_args()

423
424
    EXTERNAL_TARGETS = ('pyboard', 'wipy', 'esp8266')
    if args.target in EXTERNAL_TARGETS:
425
        import pyboard
426
        pyb = pyboard.Pyboard(args.device, args.baudrate, args.user, args.password)
427
        pyb.enter_raw_repl()
428
    elif args.target == 'unix':
429
        pyb = None
430
    else:
431
        raise ValueError('target must be either %s or unix' % ", ".join(EXTERNAL_TARGETS))
432
433

    if len(args.files) == 0:
434
        if args.test_dirs is None:
435
            if args.target == 'pyboard':
436
                # run pyboard tests
437
                test_dirs = ('basics', 'micropython', 'float', 'misc', 'stress', 'extmod', 'pyb', 'pybnative', 'inlineasm')
438
            elif args.target == 'esp8266':
439
                test_dirs = ('basics', 'micropython', 'float', 'misc', 'extmod')
440
441
            elif args.target == 'wipy':
                # run WiPy tests
442
                test_dirs = ('basics', 'micropython', 'misc', 'extmod', 'wipy')
443
444
            else:
                # run PC tests
445
                test_dirs = ('basics', 'micropython', 'float', 'import', 'io', 'misc', 'stress', 'unicode', 'extmod', 'unix', 'cmdline')
446
        else:
447
448
            # run tests from these directories
            test_dirs = args.test_dirs
449
        tests = sorted(test_file for test_files in (glob('{}/*.py'.format(dir)) for dir in test_dirs) for test_file in test_files)
Markus Siemens's avatar
Markus Siemens committed
450
    else:
451
452
        # tests explicitly given
        tests = args.files
Markus Siemens's avatar
Markus Siemens committed
453

454
455
456
457
    if not args.keep_path:
        # clear search path to make sure tests use only builtin modules
        os.environ['MICROPYPATH'] = ''

458
    if not run_tests(pyb, tests, args):
459
        sys.exit(1)
Markus Siemens's avatar
Markus Siemens committed
460

461
462
if __name__ == "__main__":
    main()