Type annotations in parser and many utilities and mappers

dmt/A_mappers/ada_A_mapper.py

@@ -26,7 +26,9 @@ import os
 import sys
 import distutils.spawn as spawn
 
+from typing import List
 from ..commonPy.utility import panic
+from ..commonPy.cleanupNodes import SetOfBadTypenames
 
 
 def Version():
@@ -37,7 +39,7 @@ def Version():
 # the second param is not asnFile, it is asnFiles
 
 
-def OnStartup(unused_modelingLanguage, asnFiles, outputDir, unused_badTypes):
+def OnStartup(unused_modelingLanguage: str, asnFiles: List[str], outputDir: str, unused_badTypes: SetOfBadTypenames) -> None:  # pylint: disable=invalid-sequence-index
     # print "Use ASN1SCC to generate the structures for '%s'" % asnFile
     asn1SccPath = spawn.find_executable('asn1.exe')
     if not asn1SccPath:

dmt/A_mappers/c_A_mapper.py

@@ -25,18 +25,20 @@ code generator A.'''
 import os
 import sys
 import distutils.spawn as spawn
+from typing import List
 
 from ..commonPy.utility import panic
+from ..commonPy.cleanupNodes import SetOfBadTypenames
 
 
-def Version():
+def Version() -> None:
     print("Code generator: " + "$Id: c_A_mapper.py 2382 2012-06-22 08:35:33Z ttsiodras $")  # pragma: no cover
 
 
 # Especially for the C mapper, since we need to pass the complete ASN.1 files list to ASN1SCC,
 # the second param is not asnFile, it is asnFiles
 
-def OnStartup(unused_modelingLanguage, asnFiles, outputDir, unused_badTypes):
+def OnStartup(unused_modelingLanguage: str, asnFiles: List[str], outputDir: str, unused_badTypes: SetOfBadTypenames) -> None:  # pylint: disable=invalid-sequence-index
     # print "Use ASN1SCC to generate the structures for '%s'" % asnFile
     asn1SccPath = spawn.find_executable('asn1.exe')
     if not asn1SccPath:

dmt/A_mappers/qgenada_A_mapper.py

@@ -28,12 +28,15 @@ code generator A.'''
 import os
 import sys
 import re
+
 import distutils.spawn as spawn
+from typing import List
 
 from ..commonPy import asnParser
 from ..commonPy.utility import panic, inform
 from ..commonPy.asnAST import AsnBool, AsnInt, AsnReal, AsnString, AsnEnumerated, AsnSequence, AsnSet, AsnChoice, AsnMetaMember, AsnSequenceOf, AsnSetOf
 from ..commonPy.createInternalTypes import ScanChildren
+from ..commonPy.cleanupNodes import SetOfBadTypenames
 
 # The file written to
 g_outputFile = None
@@ -57,7 +60,7 @@ def CleanNameAsSimulinkWants(name):
 # Especially for the C mapper, since we need to pass the complete ASN.1 files list to ASN1SCC,
 # the second param is not asnFile, it is asnFiles
 
-def OnStartup(unused_modelingLanguage, asnFiles, outputDir, unused_badTypes):
+def OnStartup(unused_modelingLanguage: str, asnFiles: List[str], outputDir: str, unused_badTypes: SetOfBadTypenames) -> None:  # pylint: disable=invalid-sequence-index
     # print "Use ASN1SCC to generate the structures for '%s'" % asnFile
     asn1SccPath = spawn.find_executable('asn1.exe')
     if not asn1SccPath:

dmt/A_mappers/qgenc_A_mapper.py

@@ -23,7 +23,7 @@
 #
 import re
 
-from typing import Set  # NOQA pylint: disable=unused-import
+from typing import Set, List  # NOQA pylint: disable=unused-import
 
 from ..commonPy.utility import panic, inform
 from ..commonPy import asnParser
@@ -31,6 +31,7 @@ from ..commonPy.asnAST import (
     AsnBool, AsnInt, AsnReal, AsnString, AsnEnumerated, AsnSequence,
     AsnSet, AsnChoice, AsnMetaMember, AsnSequenceOf, AsnSetOf)
 from ..commonPy.createInternalTypes import ScanChildren
+from ..commonPy.cleanupNodes import SetOfBadTypenames
 
 # The file written to
 g_outputFile = None
@@ -51,7 +52,7 @@ def CleanNameAsSimulinkWants(name):
     return re.sub(r'[^a-zA-Z0-9_]', '_', name)
 
 
-def OnStartup(unused_modelingLanguage, unused_asnFile, outputDir, unused_badTypes):
+def OnStartup(unused_modelingLanguage: str, unused_asnFiles: List[str], outputDir: str, unused_badTypes: SetOfBadTypenames) -> None:  # pylint: disable=invalid-sequence-index
     global g_bHasStartupRunOnce
     if g_bHasStartupRunOnce:
         # Don't rerun, it has already done all the work

dmt/A_mappers/smp2_A_mapper.py

@@ -25,7 +25,7 @@ import os
 import re
 import random
 
-from typing import Set, IO, Any  # NOQA pylint: disable=unused-import
+from typing import List, Set, IO, Any  # NOQA pylint: disable=unused-import
 
 from ..commonPy.asnAST import AsnMetaMember, AsnChoice, AsnSet, AsnSequence, AsnSequenceOf, AsnSetOf
 from ..commonPy.asnParser import g_names, g_leafTypeDict, CleanNameForAST
@@ -116,7 +116,7 @@ def FixupAstForSMP2():
 g_bStartupRun = False
 
 
-def OnStartup(unused_modelingLanguage, asnFiles, outputDir, unused_badTypes):
+def OnStartup(unused_modelingLanguage: str, asnFiles: List[str], outputDir: str, unused_badTypes: SetOfBadTypenames) -> None:  # pylint: disable=invalid-sequence-index
     '''
     Smp2 cannot represent constraint changes in unnamed inner types
     e.g. this...

dmt/B_mappers/synchronousTool.py

@@ -28,6 +28,7 @@ from typing import IO, Any  # NOQA pylint: disable=unused-import
 
 from ..commonPy.utility import panic, inform, panicWithCallStack
 from ..commonPy.aadlAST import InParam, OutParam, InOutParam
+from ..commonPy.recursiveMapper import RecursiveMapper
 
 
 class SynchronousToolGlueGenerator:
@@ -35,19 +36,19 @@ class SynchronousToolGlueGenerator:
     ##############################################
     # Parts to override for each synchronous tool
 
-    def Version(self):  # pylint: disable=no-self-use
+    def Version(self) -> None:  # pylint: disable=no-self-use
         panicWithCallStack("Method undefined in a SynchronousToolGlueGenerator...")  # pragma: no cover
 
-    def FromToolToASN1SCC(self):  # pylint: disable=no-self-use
+    def FromToolToASN1SCC(self) -> RecursiveMapper:  # pylint: disable=no-self-use
         panicWithCallStack("Method undefined in a SynchronousToolGlueGenerator...")  # pragma: no cover
 
-    def FromToolToOSS(self):  # pylint: disable=no-self-use
+    def FromToolToOSS(self) -> RecursiveMapper:  # pylint: disable=no-self-use
         panicWithCallStack("Method undefined in a SynchronousToolGlueGenerator...")  # pragma: no cover
 
-    def FromASN1SCCtoTool(self):  # pylint: disable=no-self-use
+    def FromASN1SCCtoTool(self) -> RecursiveMapper:  # pylint: disable=no-self-use
         panicWithCallStack("Method undefined in a SynchronousToolGlueGenerator...")  # pragma: no cover
 
-    def FromOSStoTool(self):  # pylint: disable=no-self-use
+    def FromOSStoTool(self) -> RecursiveMapper:  # pylint: disable=no-self-use
         panicWithCallStack("Method undefined in a SynchronousToolGlueGenerator...")  # pragma: no cover
 
     def HeadersOnStartup(self, unused_modelingLanguage, unused_asnFile, unused_subProgram, unused_subProgramImplementation, unused_outputDir, unused_maybeFVname):  # pylint: disable=no-self-use

dmt/commonPy/asnParser.py

@@ -53,7 +53,7 @@ import re
 import distutils.spawn as spawn
 
 import xml.sax  # type: ignore
-from typing import Union, List, Dict, Tuple, Any  # NOQA pylint: disable=W0611
+from typing import IO, TypeVar, Type, Optional, Callable, Union, List, Dict, Tuple, Any  # NOQA pylint: disable=W0611
 
 from . import configMT
 from . import utility
@@ -104,24 +104,18 @@ lotokens = [tkn.lower() for tkn in tokens]
 
 #    'BIT': 'BIT',
 reserved = {
-    'DEFINITIONS': 'DEFINITIONS', 'APPLICATION': 'APPLICATION',
-    'TAGS': 'TAGS', 'BEGIN': 'BEGIN', 'CHOICE': 'CHOICE',
-    'SEQUENCE': 'SEQUENCE', 'SET': 'SET', 'OF': 'OF',
-    'END': 'END', 'OPTIONAL': 'OPTIONAL', 'BOOLEAN': 'BOOLEAN',
-    'INTEGER': 'INTEGER', 'REAL': 'REAL', 'OCTET': 'OCTET',
-    'STRING': 'STRING', 'UTF8String': 'UTF8STRING',
-    'AsciiString': 'ASCIISTRING', 'NumberString': 'NUMBERSTRING',
-    'VisibleString': 'VISIBLESTRING', 'PrintableString': 'PRINTABLESTRING',
-    'ENUMERATED': 'ENUMERATED', 'AUTOMATIC': 'AUTOMATIC',
-    'IMPLICIT': 'IMPLICIT', 'EXPLICIT': 'EXPLICIT', 'SIZE': 'SIZE',
-    'TRUE': 'TRUE', 'FALSE': 'FALSE', 'DEFAULT': 'DEFAULT',
-    'mantissa': 'MANTISSA', 'base': 'BASE', 'exponent': 'EXPONENT',
-    'WITH': 'WITH', 'FROM': 'FROM', 'IMPORTS': 'IMPORTS',
-    'EXPORTS': 'EXPORTS', 'ALL': 'ALL', 'COMPONENTS': 'COMPONENTS'
+    'DEFINITIONS': 'DEFINITIONS', 'APPLICATION': 'APPLICATION', 'TAGS': 'TAGS', 'BEGIN': 'BEGIN', 'CHOICE': 'CHOICE',
+    'SEQUENCE': 'SEQUENCE', 'SET': 'SET', 'OF': 'OF', 'END': 'END', 'OPTIONAL': 'OPTIONAL', 'BOOLEAN': 'BOOLEAN',
+    'INTEGER': 'INTEGER', 'REAL': 'REAL', 'OCTET': 'OCTET', 'STRING': 'STRING', 'UTF8String': 'UTF8STRING',
+    'AsciiString': 'ASCIISTRING', 'NumberString': 'NUMBERSTRING', 'VisibleString': 'VISIBLESTRING',
+    'PrintableString': 'PRINTABLESTRING', 'ENUMERATED': 'ENUMERATED', 'AUTOMATIC': 'AUTOMATIC', 'SIZE': 'SIZE',
+    'IMPLICIT': 'IMPLICIT', 'EXPLICIT': 'EXPLICIT', 'TRUE': 'TRUE', 'FALSE': 'FALSE', 'DEFAULT': 'DEFAULT',
+    'mantissa': 'MANTISSA', 'base': 'BASE', 'exponent': 'EXPONENT', 'WITH': 'WITH', 'FROM': 'FROM',
+    'IMPORTS': 'IMPORTS', 'EXPORTS': 'EXPORTS', 'ALL': 'ALL', 'COMPONENTS': 'COMPONENTS'
 }
 
 
-def KnownType(node, names):
+def KnownType(node: AsnNode, names: AST_Lookup) -> bool:
     retVal = True
     if isinstance(node, str):
         utility.panic("Referenced type (%s) does not exist!\n" % node)
@@ -134,10 +128,9 @@ def KnownType(node, names):
     elif isinstance(node, AsnMetaMember):
         retVal = KnownType(names.get(node._containedType, node._containedType), names)
     elif isinstance(node, (AsnSequenceOf, AsnSetOf)):
-        if isinstance(node._containedType, str):
-            containedType = names.get(node._containedType, node._containedType)
-        else:
-            containedType = node._containedType
+        containedType = node._containedType
+        while isinstance(containedType, str):
+            containedType = names[containedType]
         retVal = KnownType(containedType, names)
     elif isinstance(node, AsnMetaType):
         retVal = KnownType(names.get(node._containedType, node._containedType), names)
@@ -385,7 +378,7 @@ def CheckForInvalidKeywords(node_or_str: Union[str, AsnNode]) -> None:
                 CheckForInvalidKeywords(g_names[node._containedType])
 
 
-def ParseAsnFileList(listOfFilenames):
+def ParseAsnFileList(listOfFilenames: List[str]) -> None:  # pylint: disable=invalid-sequence-index
     asn1SccPath = spawn.find_executable('asn1.exe')
     if asn1SccPath is None:
         utility.panic("ASN1SCC seems not installed on your system (asn1.exe not found in PATH).\n")
@@ -430,7 +423,7 @@ def ParseAsnFileList(listOfFilenames):
                 g_names[nodeTypename]._isArtificial = True
 
 
-def Dump():
+def Dump() -> None:
     for nodeTypename in sorted(g_names.keys()):
         if g_names[nodeTypename]._isArtificial:
             continue
@@ -439,7 +432,7 @@ def Dump():
         print("::", g_names[nodeTypename], g_leafTypeDict[nodeTypename])
 
 
-def test_asn1():
+def test_asn1() -> None:
     if "-debug" in sys.argv:
         configMT.debugParser = True
         sys.argv.remove("-debug")
@@ -461,14 +454,14 @@ g_lineno = -1
 
 
 class Element:
-    def __init__(self, name, attrs):
+    def __init__(self, name: str, attrs: Dict[str, Any]) -> None:
         self._name = name
         self._attrs = attrs
         self._children = []  # type: List[Element]
 
 
 class InputFormatXMLHandler(xml.sax.ContentHandler):
-    def __init__(self, debug=False):
+    def __init__(self, debug: bool=False) -> None:
         xml.sax.ContentHandler.__init__(self)
         self._debug = False
         if debug:
@@ -477,7 +470,7 @@ class InputFormatXMLHandler(xml.sax.ContentHandler):
         self._root = Element('root', {})
         self._roots = [self._root]
 
-    def startElement(self, name, attrs):
+    def startElement(self, name: str, attrs: Dict[str, Any]) -> None:
         if self._debug:
             print(self._indent + "(", name, ")", ", ".join(list(attrs.keys())))  # pragma: no cover
             self._indent += "    "  # pragma: no cover
@@ -486,7 +479,7 @@ class InputFormatXMLHandler(xml.sax.ContentHandler):
         self._roots.append(newElement)
 
     # def endElement(self, name):
-    def endElement(self, _):
+    def endElement(self, _: Any) -> None:
         if self._debug:
             if len(self._indent) > 4:  # pragma: no cover
                 self._indent = self._indent[:len(self._indent) - 4]  # pragma: no cover
@@ -499,24 +492,27 @@ class InputFormatXMLHandler(xml.sax.ContentHandler):
 #        Travel(indent+"    ", c)
 
 
-def VisitAll(node, expectedType, Action):
+Action = Callable[[Element], Any]
+
+
+def VisitAll(node: Element, expectedType: str, action: Action) -> List[Any]:  # pylint: disable=invalid-sequence-index
     results = []  # type: List[Any]
     if node is not None:
         if node._name == expectedType:
-            results = [Action(node)]
+            results = [action(node)]
         for child in node._children:
-            results += VisitAll(child, expectedType, Action)
+            results += VisitAll(child, expectedType, action)
     return results
 
 
-def GetAttr(node, attrName):
+def GetAttr(node: Element, attrName: str) -> Optional[Any]:
     if attrName not in list(node._attrs.keys()):
         return None
     else:
         return node._attrs[attrName]
 
 
-def GetChild(node, childName):
+def GetChild(node: Element, childName: str) -> Optional[Element]:
     for x in node._children:
         if x._name == childName:
             return x
@@ -524,7 +520,7 @@ def GetChild(node, childName):
 
 
 class Pretty:
-    def __repr__(self):
+    def __repr__(self) -> str:
         result = ""  # pragma: no cover
         for i in dir(self):  # pragma: no cover
             if i != "__repr__":  # pragma: no cover
@@ -534,14 +530,29 @@ class Pretty:
         return result  # pragma: no cover
 
 
+class Module(Pretty):
+    _id = None                 # type: str
+    _asnFilename = None        # type: str
+    _exportedTypes = None      # type: List[str]
+    _exportedVariables = None  # type: List[str]
+
+    # (tuples of ModuleName, imported types, imported vars)
+    _importedModules = None    # type: List[Tuple[str, List[str], List[str]]]
+    # (tuples of Typename, AsnNode)
+    _typeAssignments = None    # type: List[Tuple[str, AsnNode]]
+
+
 # def CreateBoolean(newModule, lineNo, xmlBooleanNode):
-def CreateBoolean(newModule, lineNo, _):
+def CreateBoolean(newModule: Module, lineNo: int, _: Any) -> AsnBool:
     return AsnBool(
         asnFilename=newModule._asnFilename,
         lineno=lineNo)
 
 
-def GetRange(newModule, lineNo, nodeWithMinAndMax, valueType):
+U = TypeVar('U', int, float)
+
+
+def GetRange(newModule: Module, lineNo: int, nodeWithMinAndMax: Element, valueType: Type[U]) -> Tuple[U, U]:
     try:
         mmin = GetAttr(nodeWithMinAndMax, "Min")
         # rangel = ( mmin == "MIN" ) and -2147483648L or valueType(mmin)
@@ -557,24 +568,24 @@ def GetRange(newModule, lineNo, nodeWithMinAndMax, valueType):
         descr = {int: "integer", float: "floating point"}  # pragma: no cover
         utility.panic("Expecting %s value ranges (%s, %s)" %  # pragma: no cover
                       (descr[valueType], newModule._asnFilename, lineNo))  # pragma: no cover
-    return [rangel, rangeh]
+    return (rangel, rangeh)
 
 
-def CreateInteger(newModule, lineNo, xmlIntegerNode):
+def CreateInteger(newModule: Module, lineNo: int, xmlIntegerNode: Element) -> AsnInt:
     return AsnInt(
         asnFilename=newModule._asnFilename,
         lineno=lineNo,
         range=GetRange(newModule, lineNo, xmlIntegerNode, int))
 
 
-def CreateReal(newModule, lineNo, xmlRealNode):
+def CreateReal(newModule: Module, lineNo: int, xmlRealNode: Element) -> AsnReal:
     return AsnReal(
         asnFilename=newModule._asnFilename,
         lineno=lineNo,
         range=GetRange(newModule, lineNo, xmlRealNode, float))
 
 
-def CreateEnumerated(newModule, lineNo, xmlEnumeratedNode):
+def CreateEnumerated(newModule: Module, lineNo: int, xmlEnumeratedNode: Element) -> AsnEnumerated:
     # bSetIntValue = True
     # if GetAttr(xmlEnumeratedNode, "ValuesAutoCalculated") == "True":
     #    bSetIntValue = False
@@ -590,19 +601,19 @@ def CreateEnumerated(newModule, lineNo, xmlEnumeratedNode):
 
 
 # def CreateBitString(newModule, lineNo, xmlBitString):
-def CreateBitString(_, __, ___):
+def CreateBitString(_, __, ___):  # type: ignore
     utility.panic("BitString type is not supported by the toolchain. "  # pragma: no cover
                   "Please use SEQUENCE OF BOOLEAN")  # pragma: no cover
 
 
-def CreateOctetString(newModule, lineNo, xmlOctetString):
+def CreateOctetString(newModule: Module, lineNo: int, xmlOctetString: Element) -> AsnOctetString:
     return AsnOctetString(
         asnFilename=newModule._asnFilename,
         lineno=lineNo,
         range=GetRange(newModule, lineNo, xmlOctetString, int))
 
 
-def CreateIA5String(newModule, lineNo, xmlIA5StringNode):
+def CreateIA5String(newModule: Module, lineNo: int, xmlIA5StringNode: Element) -> AsnAsciiString:
     # utility.panic("IA5Strings are supported by ASN1SCC, but are not supported yet " # pragma: no cover
     #               "by the toolchain. Please use OCTET STRING") # pragma: no cover
     # return CreateOctetString(newModule, lineNo, xmlIA5StringNode)
@@ -612,7 +623,7 @@ def CreateIA5String(newModule, lineNo, xmlIA5StringNode):
         range=GetRange(newModule, lineNo, xmlIA5StringNode, int))
 
 
-def CreateNumericString(newModule, lineNo, xmlNumericStringNode):
+def CreateNumericString(newModule: Module, lineNo: int, xmlNumericStringNode: Element) -> AsnOctetString:
     return CreateOctetString(newModule, lineNo, xmlNumericStringNode)  # pragma: no cover
 
 
@@ -629,7 +640,7 @@ def getIntOrFloatOrNone(d: str) -> Union[int, float, None]:
             return None
 
 
-def CreateReference(newModule, lineNo, xmlReferenceNode):
+def CreateReference(newModule: Module, lineNo: int, xmlReferenceNode: Element) -> AsnMetaType:
     return AsnMetaType(
         asnFilename=newModule._asnFilename,
         lineno=lineNo,
@@ -638,7 +649,10 @@ def CreateReference(newModule, lineNo, xmlReferenceNode):
         Max=getIntOrFloatOrNone(GetAttr(xmlReferenceNode, "Max")))
 
 
-def CommonSetSeqOf(newModule, lineNo, xmlSequenceOfNode, classToCreate):
+