Introduce syntax checking for antlr3/python3

In Python2 the syntax errors were reported as printf during the parsing,
but it is different in Python3. To detect syntax errors it is now necessary
to go recursively in the tree of nodes and look for errors after the antlr parser has completed.

opengeode/ogParser.py

@@ -16,7 +16,7 @@
     During the build of the AST this library makes a number of semantic
     checks on the SDL input mode.
 
-    Copyright (c) 2012-2019 European Space Agency
+    Copyright (c) 2012-2020 European Space Agency
 
     Designed and implemented by Maxime Perrotin
 
@@ -179,6 +179,7 @@ lineno = lambda : currentframe().f_back.f_lineno
 # as the ASN1SCC generated python AST
 USER_DEFINED_TYPES = dict()
 
+
 def types():
     ''' Return all ASN.1 and user defined types '''
     ret = getattr(DV, 'types', {}).copy()
@@ -444,8 +445,13 @@ def get_interfaces(ast, process_name):
 
 def get_input_string(root):
     ''' Return the input string of a tree node '''
-    return token_stream(root).toString(root.getTokenStartIndex(),
-            root.getTokenStopIndex())
+    try:
+       res = token_stream(root).toString(root.getTokenStartIndex(),
+                root.getTokenStopIndex())
+       return res
+    except AttributeError as err:
+        # in case there is no token_strem(root)
+        return ""
 
 
 def error(root, msg: str) -> str:
@@ -458,6 +464,32 @@ def warning(root, msg: str) -> str:
     return '{} - "{}"'.format(msg, get_input_string(root))
 
 
+def check_syntax(node: antlr3.tree.CommonTree,
+                 recursive:bool = False) -> None:
+    ''' Check if the ANTLR node is valid, otherwise raise an excption,
+    meaning there is a syntax error, and report the string that could not be
+    parsed '''
+    def check(root: antlr3.tree.CommonTree,
+              parent: antlr3.tree.CommonTree,
+              rec: bool) -> None:
+        if rec:
+            for child in root.getChildren():
+                check(child, parent, rec)
+        if isinstance(root, antlr3.tree.CommonErrorNode):
+            token = root.trappedException.token
+            token_str = token.text
+            line = token.line
+            pos = token.charPositionInLine + 1
+            if parent != root:
+                text = get_input_string(parent)
+            else:
+                text = parent.getText()  # take full node to get correct line/pos
+            syntax_error = f'In this code:\n{text}\n' \
+                    f'Unexpected "{token_str}" at line {line}, position {pos}'
+            raise SyntaxError(syntax_error)
+    check(node, parent=node, rec=recursive)
+
+
 def tmp() -> int:
     ''' Return a temporary variable name '''
     global TMPVAR
@@ -3735,10 +3767,7 @@ def state(root, parent, context):
     st_x, st_y = 0, 0
     via_stop = None
     for child in root.getChildren():
-        if isinstance(child, antlr3.tree.CommonErrorNode):
-            # There was a parsing error
-            sterr.append(f"Error parsing state: {child.getText()}")
-        elif child.type == lexer.CIF:
+        if child.type == lexer.CIF:
             # Get symbol coordinates
             (state_def.pos_x, state_def.pos_y,
             state_def.width, state_def.height) = cif(child)
@@ -5062,13 +5091,11 @@ def pr_file(root):
     # are parsed before process definition - to get signal definitions
     # and data typess references.
     processes, uses, systems = [], [], []
+
     for child in root.getChildren():
         if node_filename(child) is not None:
             ast.pr_files.add(node_filename(child))
-        if isinstance(child, antlr3.tree.CommonErrorNode):
-            # There was a parsing error
-            errors.append(f"Error parsing PR file: {child.getText()}")
-        elif child.type == lexer.PROCESS:
+        if child.type == lexer.PROCESS:
             processes.append(child)
         elif child.type == lexer.USE:
             uses.append(child)
@@ -5226,6 +5253,11 @@ def add_to_ast(ast, filename=None, string=None):
     # Root of the AST is of type antlr3.tree.CommonTree
     # Add it as a child of the common tree
     subtree = tree_rule_return_scope.tree
+    try:
+        check_syntax(node=subtree, recursive=True)
+    except SyntaxError as err:
+        LOG.error(str(err))
+        raise
     token_str = parser.getTokenStream()
     children_before = set(ast.children)
     # addChild does not simply add the subtree - it flattens it if necessary
@@ -5246,14 +5278,19 @@ def parse_pr(files=None, string=None):
     common_tree = antlr3.tree.CommonTree(None)
     for filename in files:
         sys.path.insert(0, os.path.dirname(filename))
-    for filename in files:
-        err, warn = add_to_ast(common_tree, filename=filename)
-        errors.extend(err)
-        warnings.extend(warn)
-    if string:
-        err, warn = add_to_ast(common_tree, string=string)
-        errors.extend(err)
-        warnings.extend(warn)
+    try:
+        for filename in files:
+            err, warn = add_to_ast(common_tree, filename=filename)
+            errors.extend(err)
+            warnings.extend(warn)
+        if string:
+            err, warn = add_to_ast(common_tree, string=string)
+            errors.extend(err)
+            warnings.extend(warn)
+    except SyntaxError as err:
+        errors.append([f"Parser error: syntax error!\n{str(err)}",
+                      [0, 0],
+                      ['- -']])
 
     # If syntax errors were found, raise an alarm and try to continue anyway
     if errors:
@@ -5326,6 +5363,7 @@ def parseSingleElement(elem='', string='', context=None):
         root.token_stream = parser.getTokenStream()
         backend_ptr = eval(elem)
         try:
+            check_syntax(node=root, recursive=True)
             t, semantic_errors, warnings = backend_ptr(
                                 root=root, parent=None, context=context)
         except AttributeError as err:
@@ -5336,6 +5374,8 @@ def parseSingleElement(elem='', string='', context=None):
             pass
         except NotImplementedError as err:
             syntax_errors.append('Syntax error in expression - Fix it.')
+        except SyntaxError as err:
+            syntax_errors.append(str(err))
     return(t, syntax_errors, semantic_errors, warnings,
             context.terminators)
 

opengeode/opengeode.py

@@ -3132,7 +3132,7 @@ def cli(options):
                 LOG.error('Too many errors, cannot generate code')
     else:
         if len(ast.processes) != 1:
-            LOG.error('Only one process at a time is supported')
+            LOG.error(f'Found {len(ast.processes)} process(es) instead of one')
             return 1
 
         if options.png or options.pdf or options.svg:

opengeode/sdlSymbols.py

@@ -550,10 +550,13 @@ class Join(VerticalSymbol):
         ''' Set auto-completion list - list of labels '''
         return (label.inputString for label in CONTEXT.labels)
 
-    def update_completion_list(self, pr_text):
+    def update_completion_list(self, pr_text: str) -> None:
         ''' When text was entered, update list of join terminators '''
         ast, _, _, _, _ = self.parser.parseSingleElement(self.common_name,
                                                          pr_text)
+        if not ast:
+            # in case of syntax error in the symbol text
+            return
         for each in (t for t in CONTEXT.terminators if t.kind == 'join'):
             if each.inputString == str(self):
                 # Ignore if already defined
@@ -609,6 +612,9 @@ class ProcedureStop(Join):
         ''' When text was entered, if in a nested state update exit points '''
         ast, _, _, _, _ = self.parser.parseSingleElement(self.common_name,
                                                          pr_text)
+        if not ast:
+            # in case of syntax error in the symbol text
+            return
         try:
             CONTEXT.state_exitpoints = \
                     set(CONTEXT.state_exitpoints) | set(str(self))
@@ -678,6 +684,9 @@ class Label(VerticalSymbol):
         ''' When text was entered, update list of labels in current context '''
         ast, _, _, _, _ = self.parser.parseSingleElement(self.common_name,
                                                          pr_text)
+        if not ast:
+            # in case of syntax error in the symbol text
+            return
         for each in CONTEXT.labels:
             if each.inputString == str(self):
                 # Ignore if already defined
@@ -863,6 +872,9 @@ class TextSymbol(HorizontalSymbol):
         # it it is not! - no need to investigate performance issues here
         # Get AST for the symbol
         ast, _, _, _, _ = self.parser.parseSingleElement('text_area', pr_text)
+        if not ast:
+            # in case of syntax error in the symbol text
+            return
         try:
             CONTEXT.variables.update(ast.variables)
             CONTEXT.timers = list(set(CONTEXT.timers + ast.timers))
@@ -989,8 +1001,10 @@ class State(VerticalSymbol):
         ''' When text was entered, update state completion list '''
         # Get AST for the symbol and update the context dictionnary
         ast, _, _, _, _ = self.parser.parseSingleElement('state', pr_text)
-        for each in ast.statelist:
-            CONTEXT.mapping[each.lower()] = None
+        if ast:
+            # None if there were syntax errors in the symbol
+            for each in ast.statelist:
+                CONTEXT.mapping[each.lower()] = None
 
     @property
     def completion_list(self):