Commit c5621529 authored by Damien George's avatar Damien George
Browse files

stmhal/ Add support for files with invalid utf8 bytes.

parent 4d9dce77
......@@ -42,12 +42,17 @@ class Lexer:
def __init__(self, filename):
self.file = open(filename, 'rt')
self.file = open(filename, 'rb')
self.line_number = 0
def next_match(self, strictly_next=False):
while True:
line = self.file.readline()
line = str(line, 'utf8')
except ValueError:
# some files have invalid utf8 bytes, so filter them out
line = ''.join(chr(l) for l in line if l <= 126)
self.line_number += 1
if len(line) == 0:
return ('EOF', None)
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment