Use Helper module in backends

Helper.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+    OpenGEODE - A tiny SDL Editor for TASTE
+
+    This module provides helper functions typically used by backends:
+
+        flatten(ast) : transform a model with nested states to a flat model
+        rename_everything(ast, from_name, to_name) : rename symbols
+
+    Copyright (c) 2012-2014 European Space Agency
+
+    Designed and implemented by Maxime Perrotin
+
+    Contact: maxime.perrotin@esa.int
+"""
+
+import logging
+from itertools import chain
+
+from singledispatch import singledispatch
+
+import ogAST
+
+LOG = logging.getLogger(__name__)
+
+__all__ = ['flatten', 'rename_everything', 'inner_labels_to_floating']
+
+
+def inner_labels_to_floating(process):
+    '''
+        Transform inner labels of both floating labels and transitions
+        into new floating labels, so that they can be generated in a separate
+        section of code, where they are in the scope of everybody
+        Works with processes, procedures and nested states
+    '''
+    for idx in xrange(len(process.content.floating_labels)):
+        for new_floating in find_labels(
+                      process.content.floating_labels[idx].transition):
+            process.content.floating_labels.append(new_floating)
+    for proc_tr in process.transitions:
+        for new_floating in find_labels(proc_tr):
+            process.content.floating_labels.append(new_floating)
+    for new_floating in find_labels(process.content.start.transition):
+            process.content.floating_labels.append(new_floating)
+    for each in process.content.named_start:
+        for new_floating in find_labels(each.transition):
+            process.content.floating_labels.append(new_floating)
+
+
+def flatten(process):
+    ''' Flatten the nested states:
+        Rename inner states, procedures, etc. and move them to process level
+    '''
+    def update_terminator(context, term, process):
+        '''Set next_id, identifying the next transition to run '''
+        if term.inputString.lower() in (st.statename.lower()
+                                for st in context.composite_states):
+            if not term.via:
+                term.next_id = context.mapping \
+                                          [term.inputString.lower() + '_START']
+            else:
+                term.next_id = context.mapping[term.inputString.lower()
+                                               + '_'
+                                               + term.entrypoint.lower()
+                                               + '_START']
+
+    def update_composite_state(state, process):
+        ''' Rename inner states, recursively, and add inner transitions
+            to process, updating indexes, and update terminators
+        '''
+        trans_idx = len(process.transitions)
+        prefix = state.statename + '_'
+        state.mapping = {prefix + key:state.mapping.pop(key)
+                         for key in state.mapping.keys()}
+        process.transitions.extend(state.transitions)
+
+        # Add prefix to local variable names and push them at process level
+        for dcl in state.variables.viewkeys():
+            rename_everything(state.content, dcl, prefix + dcl)
+        state.variables = {prefix + key: state.variables.pop(key)
+                           for key in state.variables.keys()}
+        process.variables.update(state.variables)
+
+        for key, value in state.mapping.viewitems():
+            # Update transition indices
+            if isinstance(value, int):
+                state.mapping[key] = value + trans_idx
+            else:
+                for inp in value:
+                    inp.transition_id += trans_idx
+        process.mapping.update(state.mapping)
+
+        # If composite state has entry procedures, add the call
+        if state.entry_procedure:
+            for each in (trans for trans in state.mapping.viewvalues()
+                         if isinstance(trans, int)):
+                call_entry = ogAST.ProcedureCall()
+                call_entry.inputString = 'entry'
+                call_entry.output = [{'outputName': prefix + 'entry',
+                                     'params': [], 'tmpVars': []}]
+                process.transitions[each].actions.insert(0, call_entry)
+
+        # If composite state has exit procedure, add the call
+        if state.exit_procedure:
+            for each in chain(state.transitions, (lab.transition for lab in
+                                              state.content.floating_labels)):
+                if each.terminator.kind == 'return':
+                    call_exit = ogAST.ProcedureCall()
+                    call_exit.inputString = 'exit'
+                    call_exit.output = [{'outputName': prefix + 'exit',
+                                     'params': [], 'tmpVars': []}]
+                    each.actions.append(call_exit)
+
+        for inner in state.composite_states:
+            # Go recursively in inner composite states
+            inner.statename = prefix + inner.statename
+            update_composite_state(inner, process)
+        for each in state.terminators:
+            # Give prefix to terminators
+            if each.label:
+                each.label.inputString = prefix + each.label.inputString
+            if each.kind == 'next_state':
+                each.inputString = prefix + each.inputString
+                # Set next transition id
+                update_terminator(state, each, process)
+            elif each.kind == 'join':
+                rename_everything(state.content,
+                                  each.inputString,
+                                  prefix + each.inputString)
+        for each in state.labels:
+            # Give prefix to labels in transitions
+            rename_everything(state.content,
+                              each.inputString,
+                              prefix + each.inputString)
+        # Add prefixed floating labels of the composite state to the process
+        process.content.floating_labels.extend(state.content.floating_labels)
+        # Rename inner procedures
+        for each in state.content.inner_procedures:
+            rename_everything(state.content, each.inputString,
+                              prefix + each.inputString)
+            each.inputString = prefix + each.inputString
+        process.content.inner_procedures.extend(state.content.inner_procedures)
+
+    def propagate_inputs(nested_state, inputlist):
+        ''' Nested states: Inputs at level N must be handled at level N-1
+            that is, all inputs of a composite states (the ones that allow
+            to exit the composite state from the outer scope) must be
+            processed by each of the substates.
+        '''
+        for _, val in nested_state.mapping.viewitems():
+            try:
+                val.extend(inputlist)
+            except AttributeError:
+                pass
+        for each in nested_state.composite_states:
+            # do the same recursively
+            propagate_inputs(each, nested_state.mapping[each.statename])
+            del nested_state.mapping[each.statename]
+
+    for each in process.composite_states:
+        update_composite_state(each, process)
+        propagate_inputs(each, process.mapping[each.statename])
+        del process.mapping[each.statename]
+
+    # Update terminators at process level
+    for each in process.terminators:
+        if each.kind == 'next_state':
+            update_terminator(process, each, process)
+
+
+@singledispatch
+def rename_everything(ast, from_name, to_name):
+    '''
+        Rename in all symbols all occurences of name_ref into new_name.
+        This is used to avoid name clashes in nested diagrams when they get
+        flattened. For example rename all accesses to a variable declared
+        in the scope of a composite state, so that they do not overwrite
+        a variable with the same name declared at a higher scope.
+    '''
+    _, _, _ = ast, from_name, to_name
+
+@rename_everything.register(ogAST.Automaton)
+def _rename_automaton(ast, from_name, to_name):
+    ''' Renaming at Automaton top level (content of digragrams) '''
+    if ast.start:
+        rename_everything(ast.start.transition, from_name, to_name)
+    for each in ast.named_start:
+        rename_everything(each.transition, from_name, to_name)
+    for each in ast.floating_labels:
+        rename_everything(each, from_name, to_name)
+    for each in ast.states:
+        for inp in each.inputs:
+            rename_everything(inp.transition, from_name, to_name)
+            for idx, param in enumerate(inp.parameters):
+                if param.lower() == from_name.lower():
+                    inp.parameter[idx] = to_name
+        if each.composite:
+            rename_everything(each.composite.content, from_name, to_name)
+    for each in ast.inner_procedures:
+        # Check that from_name is not a redefined variable in the procedure
+        for varname in each.variables.viewkeys():
+            if varname.lower() == from_name:
+                break
+        else:
+            rename_everything(each.content, from_name, to_name)
+
+
+@rename_everything.register(ogAST.Output)
+@rename_everything.register(ogAST.ProcedureCall)
+def _rename_output(ast, from_name, to_name):
+    ''' Rename actual parameter names in output/procedure calls
+        and possibly the procedure name '''
+    for each in ast.output:
+        if each['outputName'].lower() == from_name.lower():
+            each['outputName'] = to_name
+        for param in each['params']:
+            rename_everything(param, from_name, to_name)
+
+
+@rename_everything.register(ogAST.Label)
+def _rename_label(ast, from_name, to_name):
+    ''' Rename elements in the transition following a label '''
+    if ast.inputString.lower() == from_name.lower():
+        ast.inputString = to_name
+    rename_everything(ast.transition, from_name, to_name)
+
+@rename_everything.register(ogAST.Decision)
+def _rename_decision(ast, from_name, to_name):
+    ''' Rename elements in decision '''
+    if ast.kind == 'question':
+        rename_everything(ast.question, from_name, to_name)
+    for each in ast.answers:
+        rename_everything(each, from_name, to_name)
+
+
+@rename_everything.register(ogAST.Answer)
+def _rename_answer(ast, from_name, to_name):
+    ''' Rename elements in an answer branch '''
+    if ast.kind in ('constant', 'open_range'):
+        rename_everything(ast.constant, from_name, to_name)
+    elif ast.kind == 'closed_range':
+        pass # TODO when supported
+    rename_everything(ast.transition, from_name, to_name)
+
+
+@rename_everything.register(ogAST.Transition)
+def _rename_transition(ast, from_name, to_name):
+    ''' Rename in all symbols of a transition '''
+    for each in chain(ast.actions, ast.terminators):
+        # Label, output, task, decision, terminators
+        rename_everything(each, from_name, to_name)
+
+@rename_everything.register(ogAST.Terminator)
+def _rename_terminator(ast, from_name, to_name):
+    ''' Rename terminators: join/labels, next_state '''
+    if ast.inputString.lower() == from_name.lower():
+        ast.inputString = to_name
+    rename_everything(ast.label, from_name, to_name)
+    rename_everything(ast.return_expr, from_name, to_name)
+
+
+@rename_everything.register(ogAST.TaskAssign)
+def _rename_task_assign(ast, from_name, to_name):
+    ''' List of assignments '''
+    for each in ast.elems:
+        rename_everything(each, from_name, to_name)
+
+@rename_everything.register(ogAST.TaskForLoop)
+def _rename_forloop(ast, from_name, to_name):
+    ''' List of FOR loops '''
+    for each in ast.elems:
+        rename_everything(each['list'], from_name, to_name)
+        rename_everything(each['range']['start'], from_name, to_name)
+        rename_everything(each['range']['stop'], from_name, to_name)
+        rename_everything(each['transition'], from_name, to_name)
+
+@rename_everything.register(ogAST.ExprPlus)
+@rename_everything.register(ogAST.ExprMul)
+@rename_everything.register(ogAST.ExprMinus)
+@rename_everything.register(ogAST.ExprEq)
+@rename_everything.register(ogAST.ExprNeq)
+@rename_everything.register(ogAST.ExprGt)
+@rename_everything.register(ogAST.ExprGe)
+@rename_everything.register(ogAST.ExprLt)
+@rename_everything.register(ogAST.ExprLe)
+@rename_everything.register(ogAST.ExprDiv)
+@rename_everything.register(ogAST.ExprMod)
+@rename_everything.register(ogAST.ExprRem)
+@rename_everything.register(ogAST.ExprAssign)
+@rename_everything.register(ogAST.ExprOr)
+@rename_everything.register(ogAST.ExprIn)
+@rename_everything.register(ogAST.ExprAnd)
+@rename_everything.register(ogAST.ExprXor)
+@rename_everything.register(ogAST.ExprAppend)
+@rename_everything.register(ogAST.ExprAssign)
+def _rename_expr(ast, from_name, to_name):
+    ''' Two-sided expressions '''
+    rename_everything(ast.left, from_name, to_name)
+    rename_everything(ast.right, from_name, to_name)
+
+@rename_everything.register(ogAST.PrimPath)
+@rename_everything.register(ogAST.PrimVariable)
+def _rename_path(ast, from_name, to_name):
+    ''' Ultimate seek point for the renaming: primary path/variables '''
+    if ast.value[0].lower() == from_name.lower():
+        ast.value[0] = to_name
+
+
+def find_labels(trans):
+    '''
+        Yield a list of transition actions whenever a label is found
+        Used to transform labels into floating labels so that the gotos
+        in a backend can be contained within a single scope.
+    '''
+    if not trans:
+        return
+    # Terminators can have a label - add it to the transition actions
+    # (to trigger a goto at code generation)
+    if trans.terminator and trans.terminator.label:
+        trans.actions.append(trans.terminator.label)
+        trans.terminator.label = None
+    # Then for each action, check if there are labels and yield
+    # a new transition with the remaining actions (following the label)
+    for idx, action in enumerate(trans.actions):
+        if isinstance(action, ogAST.Label):
+            new_trans = ogAST.Transition()
+            # Create a floating label
+            flab = ogAST.Floating_label(label=action)
+            new_trans.actions = trans.actions[slice(idx+1, len(trans.actions))]
+            new_trans.terminator = trans.terminator
+            new_trans.terminators = trans.terminators
+            flab.transition = new_trans
+            # Transform the label into a JOIN in the original transition
+            trans.actions[idx:] = []
+            trans.terminator = ogAST.Terminator()
+            trans.terminator.inputString = action.inputString
+            trans.terminator.kind = 'join'
+            # Recursively find labels in the new transition
+            for flabel in find_labels(flab.transition):
+                yield flabel
+            # Then yield the new transition
+            yield flab
+        elif isinstance(action, ogAST.Decision):
+            for answer in action.answers:
+                for new_fl in find_labels(answer.transition):
+                    # Append the remaining actions of the transition
+                    if not new_fl.transition.terminator:
+                        new_fl.transition.actions.extend(
+                               trans.actions[slice(idx+1, len(trans.actions))])
+                        new_fl.transition.terminator = trans.terminator
+                    yield new_fl
+

LlvmGenerator.py

@@ -63,16 +63,16 @@ def _process(process):
     # Set up the optimizer pipeline.
     # Start with registering info about how the
     # target lays out data structures.
-    LLVM['pass_manager'].add(LLVM['executor'].target_data)
-    # Do simple "peephole" optimizations and bit-twiddling optzns.
-    LLVM['pass_manager'].add(passes.PASS_INSTRUCTION_COMBINING)
-    # Reassociate expressions.
-    LLVM['pass_manager'].add(passes.PASS_REASSOCIATE)
-    # Eliminate Common SubExpressions.
-    LLVM['pass_manager'].add(passes.PASS_GVN)
-    # Simplify the control flow graph (deleting unreachable blocks, etc).
-    LLVM['pass_manager'].add(passes.PASS_CFG_SIMPLIFICATION)
-    LLVM['pass_manager'].initialize()
+#   LLVM['pass_manager'].add(LLVM['executor'].target_data)
+#   # Do simple "peephole" optimizations and bit-twiddling optzns.
+#   LLVM['pass_manager'].add(passes.PASS_INSTRUCTION_COMBINING)
+#   # Reassociate expressions.
+#   LLVM['pass_manager'].add(passes.PASS_REASSOCIATE)
+#   # Eliminate Common SubExpressions.
+#   LLVM['pass_manager'].add(passes.PASS_GVN)
+#   # Simplify the control flow graph (deleting unreachable blocks, etc).
+#   LLVM['pass_manager'].add(passes.PASS_CFG_SIMPLIFICATION)
+#   LLVM['pass_manager'].initialize()
 
     # Create the runTransition function
     run_funct_name = 'run_transition'

Renderer.py

@@ -51,9 +51,19 @@ def render(ast, scene, parent, states, terminators=None):
     raise TypeError('[Renderer] Unsupported symbol in branch: ' + repr(ast))
 
 
+@render.register(ogAST.Block)
+def _block(ast, scene):
+    ''' Render a block, containing a set of process symbols '''
+    # TODO = Add text areas with signal lists, external procedures defs...
+    top_level = []
+    for each in ast.processes:
+        top_level.append(render(each, scene))
+    return top_level
+
+
 @render.register(ogAST.Process)
 def _process(ast, scene):
-    ''' Render the symbols inside a process or procedure '''
+    ''' Render a Process symbol (in a BLOCK diagram) '''
     # Set autocompletion lists for input, output, state, types, variables:
     try:
         sdlSymbols.TextSymbol.completion_list = {
@@ -71,7 +81,9 @@ def _process(ast, scene):
     sdlSymbols.ProcedureCall.completion_list = {
             proc.inputString for proc in ast.procedures}
 
-    return render(ast.content, scene)
+    symbol = sdlSymbols.Process(ast, ast)
+    scene.addItem(symbol)
+    return symbol
 
 
 @render.register(ogAST.Automaton)

ogAST.py

@@ -712,6 +712,15 @@ class Process(object):
         self.variables = {}
         # global variables can be used to inherit variables
         self.global_variables = {}
+        
+        # Set default coordinates and width/height
+        self.pos_x = self.pos_y = 150
+        self.width = 150
+        self.height = 75
+        # Optional hyperlink
+        self.hyperlink = None
+        # Optional comment
+        self.comment = None
 
         # dataview: complete AST of the ASN.1 types
         self.asn1Modules = None
@@ -734,7 +743,6 @@ class Process(object):
         # list of Procedure (external procedures)
         self.procedures = []
 
-
         # The Mapping structure should be used for code generation backends
         # dictionnary: {'stateName': [class Input1, Input2,...], ...}
         # then Input contains the inputs list and corresponding transition

ogParser.py

@@ -100,7 +100,7 @@ SPECIAL_OPERATORS = {'length': [LIST],
 # Container to keep a list of types mapped from ANTLR Tokens
 # (Used with singledispatch/visitor pattern)
 ANTLR_TOKEN_TYPES = {a: type(a, (antlr3.tree.CommonTree,), {})
-                    for a,b in lexer.__dict__.viewitems() if type(b)==int}
+                    for a, b in lexer.__dict__.viewitems() if type(b) == int}
 
 
 # Shortcut to create a new referenced ASN.1 type
@@ -726,7 +726,6 @@ def find_type(path, context):
                     raise TypeError('Field ' + elem
                                      + ' not found in expression '
                                      + '!'.join(path))
-                    result = UNKNOWN_TYPE
                     break
             # Sequence of
             elif basic.kind == 'SequenceOfType':
@@ -857,7 +856,7 @@ def fix_expression_types(expr, context):
         if asn_type.kind != 'ChoiceType' \
                 or field.lower() not in [key.lower()
                                   for key in asn_type.Children.viewkeys()]:
-                raise TypeError('Field is not valid in CHOICE:' + field)
+            raise TypeError('Field is not valid in CHOICE:' + field)
         key, = [key for key in asn_type.Children.viewkeys()
                 if key.lower() == field.lower()]
         if expr.right.value['value'].exprType == UNKNOWN_TYPE:
@@ -1069,7 +1068,7 @@ def primary_value(root, context=None):
 
     # If there were parameters or index, try to determine the type of
     # the expression
-    if isinstance(prim, ogAST.PrimPath) and len(prim.value)>1:
+    if isinstance(prim, ogAST.PrimPath) and len(prim.value) > 1:
         try:
             prim.exprType = find_type(prim.value, context)
         except TypeError as err:
@@ -1313,7 +1312,7 @@ def fpar(root):
         param_names = []
         sort = ''
         direction = 'in'
-        assert(param.type == lexer.PARAM)
+        assert param.type == lexer.PARAM
         for child in param.getChildren():
             if child.type == lexer.INOUT:
                 direction = 'out'
@@ -1743,10 +1742,16 @@ def process_definition(root, parent=None):
     process.filename = node_filename(root)
     process.parent = parent
     parent.processes.append(process)
+    coord = False
     # Prepare the transition/state mapping
     process.mapping = {name: [] for name in get_state_list(root)}
     for child in root.getChildren():
-        if child.type == lexer.ID:
+        if child.type == lexer.CIF:
+            # Get symbol coordinates
+            process.pos_x, process.pos_y, process.width, process.height =\
+                    cif(child)
+            coord = True
+        elif child.type == lexer.ID:
             # Get process (taste function) name
             process.processName = child.text
             try:
@@ -1763,6 +1768,10 @@ def process_definition(root, parent=None):
             except TypeError as error:
                 LOG.debug(str(error))
                 errors.append(str(error))
+            if coord:
+                errors = [[e, [process.pos_x, process.pos_y]] for e in errors]
+                warnings = [[w, [process.pos_x, process.pos_y]]
+                             for w in warnings]
         elif child.type == lexer.TEXTAREA:
             # Text zone where variables and operators are declared
             textarea, err, warn = text_area(child, context=process)
@@ -1807,8 +1816,9 @@ def process_definition(root, parent=None):
         elif child.type == lexer.REFERENCED:
             process.referenced = True
         else:
-            warnings.append('Unsupported process definition child type: ' +
-                    str(child.type) + '- line ' + str(child.getLine()))
+            warnings.append('Unsupported process definition child: ' +
+                             sdl92Parser.tokenNames[child.type] +
+                            ' - line ' + str(child.getLine()))
     return process, errors, warnings
 
 def input_part(root, parent, context):
@@ -1901,7 +1911,7 @@ def input_part(root, parent, context):
             context.transitions.append(trans)
             i.transition_id = len(context.transitions) - 1
         elif child.type == lexer.COMMENT:
-            i.comment, _, ___ = end(child)
+            i.comment, _, _ = end(child)
         elif child.type == lexer.HYPERLINK:
             i.hyperlink = child.getChild(0).toString()[1:-1]
         else:
@@ -2045,7 +2055,7 @@ def connect_part(root, parent, context):
         elif child.type == lexer.HYPERLINK:
             conn.hyperlink = child.getChild(0).toString()[1:-1]
         elif child.type == lexer.COMMENT:
-            conn.comment, _, ___ = end(child)
+            conn.comment, _, _ = end(child)
         else:
             warnings.append('Unsupported CONNECT PART child type: ' +
                             sdl92Parser.tokenNames[child.type])
@@ -2819,7 +2829,7 @@ def pr_file(root):
         def find_processes(block):
             ''' Recursively find processes in a system '''
             try:
-                result = [proc for proc in block.processes 
+                result = [proc for proc in block.processes
                           if not proc.referenced]
             except AttributeError:
                 result = []
@@ -2857,14 +2867,14 @@ def add_to_ast(ast, filename=None, string=None):
     # Root of the AST is of type antlr3.tree.CommonTree
     # Add it as a child of the common tree
     subtree = tree_rule_return_scope.tree
-    token_stream = parser.getTokenStream()
+    token_str = parser.getTokenStream()
     children_before = set(ast.children)
     # addChild does not simply add the subtree - it flattens it if necessary
     # this means that possibly SEVERAL subtrees can be added. We must set
     # the token_stream reference to all of them.
     ast.addChild(subtree)
     for tree in set(ast.children) - children_before:
-        tree.token_stream = token_stream
+        tree.token_stream = token_str
     return errors, warnings
 
 
@@ -2879,11 +2889,12 @@ def parse_pr(files=None, string=None):
         sys.path.insert(0, os.path.dirname(filename))
     for filename in files:
         err, warn = add_to_ast(common_tree, filename=filename)
+        errors.extend(err)
+        warnings.extend(warn)
     if string:
         err, warn = add_to_ast(common_tree, string=string)
-
-    errors.extend(err)
-    warnings.extend(warn)
+        errors.extend(err)
+        warnings.extend(warn)
 
     # At the end when common tree is complete, perform the parsing
     og_ast, err, warn = pr_file(common_tree)
@@ -2917,7 +2928,7 @@ def parseSingleElement(elem='', string=''):
     LOG.debug('Parsing string: ' + string + ' with elem ' + elem)
     parser = parser_init(string=string)
     parser_ptr = getattr(parser, elem)
-    assert(parser_ptr is not None)
+    assert parser_ptr is not None
     syntax_errors = []
     semantic_errors = []
     warnings = []

opengeode.py

@@ -56,7 +56,7 @@ from genericSymbols import(Symbol, Comment, EditableText, Cornergrabber,
                            Connection, Completer)
 from sdlSymbols import(Input, Output, Decision, DecisionAnswer, Task,
         ProcedureCall, TextSymbol, State, Start, Join, Label, Procedure,
-        ProcedureStart, ProcedureStop, StateStart, Connect)
+        ProcedureStart, ProcedureStop, StateStart, Connect, Process)
 
 # Icons and png files generated from the resource file:
 import icons  # NOQA
@@ -125,6 +125,7 @@ G_SYMBOLS = set()
 
 # Lookup table used to configure the context-dependent toolbars
 ACTIONS = {
+    'block': [Process],
     'process': [Start, State, Input, Connect, Task, Decision, DecisionAnswer,
                 Output, ProcedureCall, TextSymbol, Comment, Label,
                 Join, Procedure],
@@ -237,7 +238,7 @@ class Sdl_toolbar(QtGui.QToolBar, object):
             try:
                 for item in scene.visible_symb:
                     try:
-                        if item.is_singleton: # and item.isVisible():
+                        if item.is_singleton: