py/lexer: Provide generic mp_lexer_new_from_file based on mp_reader.

If a port defines MICROPY_READER_POSIX or MICROPY_READER_FATFS then
lexer.c now provides an implementation of mp_lexer_new_from_file using
the mp_reader_new_file function.

cc3200/application.mk

@@ -162,7 +162,6 @@ APP_STM_SRC_C = $(addprefix stmhal/,\
 	import.c \
 	input.c \
 	irq.c \
-	lexerfatfs.c \
 	moduselect.c \
 	pybstdio.c \
 	)

esp8266/main.c

@@ -109,17 +109,13 @@ void user_init(void) {
     system_init_done_cb(init_done);
 }
 
-mp_lexer_t *fat_vfs_lexer_new_from_file(const char *filename);
 mp_import_stat_t fat_vfs_import_stat(const char *path);
 
+#if !MICROPY_VFS_FAT
 mp_lexer_t *mp_lexer_new_from_file(const char *filename) {
-    #if MICROPY_VFS_FAT
-    return fat_vfs_lexer_new_from_file(filename);
-    #else
-    (void)filename;
     return NULL;
-    #endif
 }
+#endif
 
 mp_import_stat_t mp_import_stat(const char *path) {
     #if MICROPY_VFS_FAT

esp8266/mpconfigport.h

@@ -16,7 +16,7 @@
 #define MICROPY_MEM_STATS           (0)
 #define MICROPY_DEBUG_PRINTERS      (1)
 #define MICROPY_DEBUG_PRINTER_DEST  mp_debug_print
-#define MICROPY_READER_FATFS        (1)
+#define MICROPY_READER_FATFS        (MICROPY_VFS_FAT)
 #define MICROPY_ENABLE_GC           (1)
 #define MICROPY_STACK_CHECK         (1)
 #define MICROPY_ENABLE_EMERGENCY_EXCEPTION_BUF (1)

extmod/vfs_fat_lexer.c

-/*
- * This file is part of the Micro Python project, http://micropython.org/
- *
- * The MIT License (MIT)
- *
- * Copyright (c) 2013, 2014 Damien P. George
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-
-#include "py/mpconfig.h"
-// *_ADHOC part is for cc3200 port which doesn't use general uPy
-// infrastructure and instead duplicates code. TODO: Resolve.
-#if MICROPY_VFS_FAT || MICROPY_FSUSERMOUNT || MICROPY_FSUSERMOUNT_ADHOC
-
-#include "py/lexer.h"
-#include "lib/fatfs/ff.h"
-
-typedef struct _mp_lexer_file_buf_t {
-    FIL fp;
-    byte buf[20];
-    uint16_t len;
-    uint16_t pos;
-} mp_lexer_file_buf_t;
-
-STATIC mp_uint_t file_buf_next_byte(mp_lexer_file_buf_t *fb) {
-    if (fb->pos >= fb->len) {
-        if (fb->len < sizeof(fb->buf)) {
-            return MP_LEXER_EOF;
-        } else {
-            UINT n;
-            f_read(&fb->fp, fb->buf, sizeof(fb->buf), &n);
-            if (n == 0) {
-                return MP_LEXER_EOF;
-            }
-            fb->len = n;
-            fb->pos = 0;
-        }
-    }
-    return fb->buf[fb->pos++];
-}
-
-STATIC void file_buf_close(mp_lexer_file_buf_t *fb) {
-    f_close(&fb->fp);
-    m_del_obj(mp_lexer_file_buf_t, fb);
-}
-
-mp_lexer_t *fat_vfs_lexer_new_from_file(const char *filename);
-
-mp_lexer_t *fat_vfs_lexer_new_from_file(const char *filename) {
-    mp_lexer_file_buf_t *fb = m_new_obj_maybe(mp_lexer_file_buf_t);
-    if (fb == NULL) {
-        return NULL;
-    }
-    FRESULT res = f_open(&fb->fp, filename, FA_READ);
-    if (res != FR_OK) {
-        m_del_obj(mp_lexer_file_buf_t, fb);
-        return NULL;
-    }
-    UINT n;
-    f_read(&fb->fp, fb->buf, sizeof(fb->buf), &n);
-    fb->len = n;
-    fb->pos = 0;
-    return mp_lexer_new(qstr_from_str(filename), fb, (mp_lexer_stream_next_byte_t)file_buf_next_byte, (mp_lexer_stream_close_t)file_buf_close);
-}
-
-#endif // MICROPY_VFS_FAT

py/lexer.c

@@ -759,6 +759,19 @@ mp_lexer_t *mp_lexer_new_from_str_len(qstr src_name, const char *str, mp_uint_t
     return mp_lexer_new(src_name, reader.data, (mp_lexer_stream_next_byte_t)reader.readbyte, (mp_lexer_stream_close_t)reader.close);
 }
 
+#if MICROPY_READER_POSIX || MICROPY_READER_FATFS
+
+mp_lexer_t *mp_lexer_new_from_file(const char *filename) {
+    mp_reader_t reader;
+    int ret = mp_reader_new_file(&reader, filename);
+    if (ret != 0) {
+        return NULL;
+    }
+    return mp_lexer_new(qstr_from_str(filename), reader.data, (mp_lexer_stream_next_byte_t)reader.readbyte, (mp_lexer_stream_close_t)reader.close);
+}
+
+#endif
+
 void mp_lexer_free(mp_lexer_t *lex) {
     if (lex) {
         if (lex->stream_close) {

py/lexerunix.c

@@ -85,12 +85,4 @@ mp_lexer_t *mp_lexer_new_from_fd(qstr filename, int fd, bool close_fd) {
     return mp_lexer_new(filename, fb, (mp_lexer_stream_next_byte_t)file_buf_next_byte, (mp_lexer_stream_close_t)file_buf_close);
 }
 
-mp_lexer_t *mp_lexer_new_from_file(const char *filename) {
-    int fd = open(filename, O_RDONLY);
-    if (fd < 0) {
-        return NULL;
-    }
-    return mp_lexer_new_from_fd(qstr_from_str(filename), fd, true);
-}
-
 #endif // MICROPY_HELPER_LEXER_UNIX

py/py.mk

@@ -229,7 +229,6 @@ PY_O_BASENAME = \
 	../extmod/vfs_fat_diskio.o \
 	../extmod/vfs_fat_file.o \
 	../extmod/vfs_fat_reader.o \
-	../extmod/vfs_fat_lexer.o \
 	../extmod/vfs_fat_misc.o \
 	../extmod/utime_mphal.o \
 	../extmod/uos_dupterm.o \

stmhal/Makefile

@@ -154,7 +154,6 @@ SRC_C = \
 	modusocket.c \
 	modnetwork.c \
 	import.c \
-	lexerfatfs.c \
 	extint.c \
 	usrsw.c \
 	rng.c \

stmhal/lexerfatfs.c

-/*
- * This file is part of the MicroPython project, http://micropython.org/
- *
- * The MIT License (MIT)
- *
- * Copyright (c) 2013, 2014 Damien P. George
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-
-#include "py/lexer.h"
-
-mp_lexer_t *fat_vfs_lexer_new_from_file(const char *filename);
-
-// TODO: Instead of such shims, probably better to let port #define
-// mp_lexer_new_from_file to a function it wants to use.
-mp_lexer_t *mp_lexer_new_from_file(const char *filename) {
-    return fat_vfs_lexer_new_from_file(filename);
-}

teensy/lexerfatfs.c

-#include <stdint.h>
-#include <stdio.h>
-
-#include "misc.h"
-#include "mpconfig.h"
-#include "qstr.h"
-#include "lexer.h"
-typedef int FIL;
-#include "../stmhal/lexerfatfs.h"
-
-mp_lexer_t *mp_lexer_new_from_file(const char *filename) {
-  printf("import not implemented\n");
-  return NULL;
-}
-