Merge branch 'master' of https://github.com/ttsiodras/DataModellingTools

.flake8

 [flake8]
-ignore = E501,E225,C103
+ignore = E501,E225,C103,E722,E252,W504
 max-line-length = 160

Makefile

@@ -21,7 +21,7 @@ pylint:
 
 mypy:
 	@echo Performing type analysis via mypy...
-	@mypy --disallow-untyped-defs --check-untyped-defs ${PY_SRC} || exit 1
+	@mypy --disallow-untyped-defs --check-untyped-defs --ignore-missing-imports ${PY_SRC} || exit 1
 
 coverage:
 	@echo Performing coverage checks...

README.md

@@ -3,7 +3,7 @@
 TASTE Data Modelling Tools
 ==========================
 
-These are the tools used by the European Space Agency's [TASTE toolchain](https://taste.tuxfamily.org/)
+These are the tools used by the European Space Agency's [TASTE toolchain](https://taste.tools/)
 to automate handling of the Data Modelling. They include more than two
 dozen codegenerators that automatically create the 'glue'; the run-time translation
 bridges that allow code generated by modelling tools (Simulink, SCADE, OpenGeode, etc)
@@ -13,7 +13,7 @@ For the encoders and decoders of the messages
 themselves, TASTE uses [ASN1SCC](https://github.com/ttsiodras/asn1scc) - an ASN.1
 compiler specifically engineered for safety-critical environments.
 
-For more details, visit the [TASTE site](https://taste.tuxfamily.org/).
+For more details, visit the [TASTE site](https://taste.tools/).
 
 Installation
 ------------

circle.yml

@@ -9,7 +9,7 @@ dependencies:
     - sudo rm -rf /var/cache/apt/archives && sudo ln -s ~/.apt-cache /var/cache/apt/archives && mkdir -p ~/.apt-cache/partial
     # - sudo apt-get update
     - sudo apt-get install libxslt-dev libxml2-dev python-psycopg2
-    - wget -O - -q https://github.com/ttsiodras/asn1scc/releases/download/3.3.9/asn1scc-bin-3.3.9.tar.bz2 | tar jxvf -
+    - wget -O - -q https://github.com/ttsiodras/asn1scc/releases/download/4.1b/asn1scc-bin-4.1b.tar.bz2 | tar jxvf -
     - wget -O - -q https://github.com/ttsiodras/DataModellingTools/files/335591/antlr-2.7.7.tar.gz | tar zxvf - ; cd antlr-2.7.7/lib/python ; sudo pip2 install .
     - sudo apt-get install mono-runtime libmono-system-data4.0-cil libmono-system-web4.0-cil libfsharp-core4.3-cil
 

dmt/A_mappers/Stubs.py

@@ -26,7 +26,7 @@ import copy
 import DV_Types  # pylint: disable=import-error
 from ctypes import (
     cdll, c_void_p, c_ubyte, c_double, c_uint,
-    c_longlong, c_bool, c_int, c_long
+    c_longlong, c_bool, c_int, c_long, c_char
 )
 
 # load the *getset.so in this folder
@@ -252,18 +252,19 @@ An example for SetLength:
                     'double': c_double,
                     'flag': c_bool,
                     'int': c_int,
-                    'long': c_long
+                    'long': c_long,
+                    'char': c_ubyte # char
                 }.get(resType, None)
                 if cTypesResultType is None:
                     raise AsnCoderError("Result type of %s not yet supported in the Python mapper - contact support." % resType)
             bridgeFunc = getattr(JMP, bridgeFuncName)
             bridgeFunc.restype = cTypesResultType
             retVal = bridgeFunc(self._ptr, *self._params)
-        except:
+        except Exception as e:
             oldAP = self._accessPath
             if args.get("reset", True):
                 self.Reset()
-            raise AsnCoderError("The access path you used (%s) is not valid." % oldAP)
+            raise AsnCoderError("The access path you used (%s) is not valid. (%s)" % (oldAP, str(e)))
         if args.get("reset", True):
             self.Reset()
         return retVal

dmt/A_mappers/ada_A_mapper.py

@@ -48,7 +48,7 @@ def OnStartup(unused_modelingLanguage: str, asnFiles: List[str], outputDir: str,
         panic("ASN1SCC seems to be missing from your system (asn1.exe not found in PATH).\n")  # pragma: no cover
     os.system(
         ("mono " if sys.platform.startswith('linux') else "") +
-        "\"{}\" -wordSize 8 -typePrefix asn1Scc -Ada -uPER -o \"".format(asn1SccPath) +
+        "\"{}\" -typePrefix asn1Scc -Ada -uPER -o \"".format(asn1SccPath) +
         outputDir + "\" \"" + "\" \"".join(asnFiles) + "\"")
     os.system("rm -f \"" + outputDir + "\"/*.adb")
 

dmt/A_mappers/c_A_mapper.py

@@ -47,7 +47,7 @@ def OnStartup(unused_modelingLanguage: str, asnFiles: List[str], outputDir: str,
         panic("ASN1SCC seems to be missing from your system (asn1.exe not found in PATH).\n")  # pragma: no cover
     os.system(
         ("mono " if sys.platform.startswith('linux') else "") +
-        "\"{}\" -wordSize 8 -typePrefix asn1Scc -c -uPER -o \"".format(asn1SccPath) +
+        "\"{}\" -typePrefix asn1Scc -c -uPER -o \"".format(asn1SccPath) +
         outputDir + "\" \"" + "\" \"".join(asnFiles) + "\"")
     cmd = 'rm -f '
     for i in ['real.c', 'asn1crt.c', 'acn.c']:

dmt/A_mappers/python_A_mapper.py

 #
-# (C) Semantix Information Technologies.
-#
-# Semantix Information Technologies is licensing the code of the
-# Data Modelling Tools (DMT) in the following dual-license mode:
-#
-# Commercial Developer License:
-#       The DMT Commercial Developer License is the suggested version
-# to use for the development of proprietary and/or commercial software.
-# This version is for developers/companies who do not want to comply
-# with the terms of the GNU Lesser General Public License version 2.1.
-#
-# GNU LGPL v. 2.1:
-#       This version of DMT is the one to use for the development of
-# applications, when you are willing to comply with the terms of the
-# GNU Lesser General Public License version 2.1.
-#
-# Note that in both cases, there are no charges (royalties) for the
-# generated code.
+# (C) Semantix Information Technologies,
+#     Neuropublic,
+#     European Space Agency
 #
+# The license of the Data Modelling Tools (DMT) is GPL with Runtime Exception
+
 import re
 import os
 
@@ -28,7 +15,8 @@ from ..commonPy.utility import panic, inform
 from ..commonPy.asnAST import (
     AsnBool, AsnInt, AsnReal, AsnString, isSequenceVariable, AsnEnumerated,
     AsnSequence, AsnSet, AsnChoice, AsnMetaMember, AsnSequenceOf, AsnSetOf,
-    AsnBasicNode, AsnNode, AsnSequenceOrSet, AsnSequenceOrSetOf)
+    AsnBasicNode, AsnNode, AsnSequenceOrSet, AsnSequenceOrSetOf,
+    AsnAsciiString)
 from ..commonPy.asnParser import AST_Lookup, AST_Leaftypes
 from ..commonPy.cleanupNodes import SetOfBadTypenames
 
@@ -44,7 +32,7 @@ g_bHasStartupRunOnce = False
 
 def Version() -> None:
     print("Code generator: " +
-          "$Id: python_A_mapper.py 2400 2012-09-04 10:40:19Z ttsiodras $")  # pragma: no cover
+          "$Id: python_A_mapper.py $")  # pragma: no cover
 
 
 def CleanNameAsPythonWants(name: str) -> str:
@@ -52,7 +40,7 @@ def CleanNameAsPythonWants(name: str) -> str:
 
 
 def OnStartup(unused_modelingLanguage: str, asnFile: str, outputDir: str, badTypes: SetOfBadTypenames) -> None:
-    os.system("bash -c '[ ! -f \"" + outputDir + "/" + asnFile + "\" ] && cp \"" + asnFile + "\" \"" + outputDir + "\"'")
+    os.system("cp -u \"" + asnFile + "\" \"" + outputDir + "\"")
     this_path = os.path.dirname(__file__)
     stubs = this_path + os.sep + 'Stubs.py'
     os.system('cp "{}" "{}"'.format(stubs, outputDir))
@@ -158,7 +146,7 @@ $(BDIR)/$(GRAMMAR)_getset.c:       $(GRAMMAR).asn
 
 $(BDIR)/asn1crt.c $(BDIR)/$(GRAMMAR).c $(BDIR)/real.c $(BDIR)/acn.c $(BDIR)/$(GRAMMAR).h $(BDIR)/asn1crt.h:       $(GRAMMAR).asn
 %(tab)sif [ ! -f "$(GRAMMAR).acn" ] ; then %(mono)s $(ASN1SCC) -ACND -o $(BDIR) $< ; fi
-%(tab)s%(mono)s $(ASN1SCC) -ACN -c -uPER -equal -wordSize 8 -o $(BDIR) $< $(GRAMMAR).acn
+%(tab)s%(mono)s $(ASN1SCC) -ACN -c -uPER -equal -o $(BDIR) $< $(GRAMMAR).acn
 
 $(BDIR)/DV.py:       $(GRAMMAR).asn
 %(tab)sgrep 'REQUIRED_BYTES_FOR_.*ENCODING' $(BDIR)/$(GRAMMAR).h | awk '{print $$2 " = " $$3}' > $@
@@ -182,6 +170,7 @@ clean:
     g_outputGetSetC.write('\n/* Helper functions for NATIVE encodings */\n\n')
 
     def WorkOnType(nodeTypeName: str) -> None:
+        inform("Python_A_mapper: Working on type '%s'...", nodeTypeName)
         typ = CleanNameAsPythonWants(nodeTypeName)
         g_outputGetSetH.write('void SetDataFor_%s(void *dest, void *src);\n' % typ)
         g_outputGetSetH.write("byte* MovePtrBySizeOf_%s(byte *pData);\n" % typ)
@@ -329,7 +318,6 @@ def CommonBaseImpl(comment: str,
         g_outputGetSetC.write("}\n")
 
 
-# def CommonBaseImplSequenceFixed(comment, ctype, path, params, accessPathInC, node, postfix = ""):
 def CommonBaseImplSequenceFixed(comment: str,
                                 ctype: str,
                                 path: str,
@@ -349,6 +337,31 @@ def CommonBaseImplSequenceFixed(comment: str,
     g_outputGetSetC.write("}\n")
 
 
+def CommonBaseImplIA5String(comment: str,
+                            ctype: str,
+                            path: str,
+                            params: Params,
+                            accessPathInC: str,
+                            node: AsnAsciiString) -> None:
+    g_outputGetSetH.write("\n/* %s */\n%s %s_GetLength(%s);\n" % (comment, ctype, path, params.GetDecl()))
+    g_outputGetSetC.write("\n/* %s */\n%s %s_GetLength(%s)\n" % (comment, ctype, path, params.GetDecl()))
+    g_outputGetSetC.write("{\n")
+    if not isSequenceVariable(node):
+        g_outputGetSetC.write("    return " + str(node._range[-1]) + ";\n")
+    else:
+        g_outputGetSetC.write("    return strlen((*root)" + accessPathInC + ");\n")
+    g_outputGetSetC.write("}\n")
+    g_outputGetSetH.write("\n/* %s */\nvoid %s_SetLength(%s, %s value);\n" % (comment, path, params.GetDecl(), ctype))
+    g_outputGetSetC.write("\n/* %s */\nvoid %s_SetLength(%s, %s value)\n" % (comment, path, params.GetDecl(), ctype))
+    g_outputGetSetC.write("{\n")
+    if not isSequenceVariable(node):
+        g_outputGetSetC.write("    assert(value == " + str(node._range[-1]) + ");\n")
+        # g_outputGetSetC.write("    fprintf(stderr, \"WARNING: setting length of fixed-length string\\n\");\n")
+    else:
+        g_outputGetSetC.write("    (*root)" + accessPathInC + "[value] = 0;\n")
+    g_outputGetSetC.write("}\n")
+
+
 def CreateGettersAndSetters(
         path: str,
         params: Params,
@@ -367,6 +380,13 @@ def CreateGettersAndSetters(
         CommonBaseImpl("INTEGER", "asn1SccSint", path, params, accessPathInC)
     elif isinstance(node, AsnReal):
         CommonBaseImpl("REAL", "double", path, params, accessPathInC)
+    elif isinstance(node, AsnAsciiString):
+        if not node._range:
+            panic("Python_A_mapper: IA5String (in %s) must have a SIZE constraint!\n" % node.Location())  # pragma: no cover
+        CommonBaseImplIA5String("IA5String", "long", path, params, accessPathInC, node)
+        params.AddParam('int', "iDx", leafTypeDict)
+        CommonBaseImpl("IA5String_bytes", "char", path + "_iDx", params, accessPathInC + ("[" + params._vars[-1] + "]"), "")
+        params.Pop()
     elif isinstance(node, AsnString):
         if not node._range:
             panic("Python_A_mapper: string (in %s) must have a SIZE constraint!\n" % node.Location())  # pragma: no cover
@@ -539,12 +559,13 @@ def CreateDeclarationForType(nodeTypename: str, names: AST_Lookup, leafTypeDict:
         g_outputFile.write("        print(self.GSER() + '\\n')\n\n\n")
 
     else:  # pragma: no cover
-        panic("Unexpected ASN.1 type... Send this grammar to Semantix")  # pragma: no cover
+        panic("Unexpected ASN.1 type... Send this grammar to ESA")  # pragma: no cover
 
 
 def CreateDeclarationsForAllTypes(names: AST_Lookup, leafTypeDict: AST_Leaftypes, badTypes: SetOfBadTypenames) -> None:
     for nodeTypename in names:
-        if not names[nodeTypename]._isArtificial and nodeTypename not in badTypes:
+        # Do not ignore the so called "bad types". In python, IA5Strings are supported
+        if not names[nodeTypename]._isArtificial:  # and nodeTypename not in badTypes:
             CreateDeclarationForType(nodeTypename, names, leafTypeDict)
 
 # vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4

dmt/A_mappers/qgenada_A_mapper.py

@@ -72,7 +72,7 @@ def OnStartup(unused_modelingLanguage: str, asnFiles: List[str], outputDir: str,
         panic("ASN1SCC seems to be missing from your system (asn1.exe not found in PATH).\n")  # pragma: no cover
     os.system(
         ("mono " if sys.platform.startswith('linux') else "") +
-        "\"{}\" -wordSize 8 -typePrefix asn1Scc -c -uPER -o \"".format(asn1SccPath) +
+        "\"{}\"  -typePrefix asn1Scc -c -uPER -o \"".format(asn1SccPath) +
         outputDir + "\" \"" + "\" \"".join(asnFiles) + "\"")
     os.system("rm -f \"" + outputDir + "\"/*.adb")
 

dmt/B_mappers/vhdlTemplate.py

@@ -362,7 +362,7 @@ taste.bit:      $(SRCS)
 {tab}cp "$@" ../TASTE.bit || exit 1
 
 $(SYSTEMC_GENERATED):   $(SYSTEMC_SRC)
-{tab}for i in $^ ; do if [ "`basename "$$i" | sed 's,^.*\.,,'`" = "cpp" ] ; then /c/Program\ Files/SystemCrafter/SystemCrafter\ SC/bin/craft.exe /vhdl $$i || exit 1; fi ; done
+{tab}for i in $^ ; do if [ "`basename "$$i" | sed 's,^.*\\.,,'`" = "cpp" ] ; then /c/Program\\ Files/SystemCrafter/SystemCrafter\\ SC/bin/craft.exe /vhdl $$i || exit 1; fi ; done
 
 test:
 {tab}cd .. ;  ./TASTE.exe

dmt/B_mappers/zestSC1_B_mapper.py

@@ -459,7 +459,7 @@ static void ErrorHandler(
     ZESTSC1_STATUS Status,
     const char *Msg)
 {
-    printf("**** TASTE - Function %s returned an error\\n        \\"%%s\\"\\n\\n", Function, Msg);
+    printf("**** TASTE - Function %s returned an error\\n        \\"%s\\"\\n\\n", Function, Msg);
     exit(1);
 }
 

dmt/aadl2glueC.py

@@ -166,7 +166,7 @@ of each SUBPROGRAM param.'''
                     for step in python2className.split('.')[1:]:
                         klass = getattr(klass, step)
                     o.__class__ = klass
-            except Exception as _:
+            except Exception:
                 pass
 
         patchMe(sp)
@@ -207,7 +207,7 @@ types). This used to cover Dumpable C/Ada Types and OG headers.'''
     if asnFile is not None:
         if not asn1SccPath:
             panic("ASN1SCC seems not installed on your system (asn1.exe not found in PATH).\n")  # pragma: no cover
-        os.system('mono "{}" -wordSize 8 -typePrefix asn1Scc -Ada -equal -o "{}" "{}"'
+        os.system('mono "{}" -typePrefix asn1Scc -Ada -equal -o "{}" "{}"'
                   .format(asn1SccPath, outputDir, '" "'.join([asnFile])))
 
 

dmt/asn2aadlPlus.py

@@ -89,11 +89,14 @@ def calculateForNativeAndASN1SCC(absASN1SCCpath, autosrc, names, inputFiles):
 
     # Spawn ASN1SCC.exe compiler - for MacOS define a new sh file calling mono Asn1f2.exe
     if platform.system() == "Windows" or platform.system() == "Darwin":
-        mysystem("%s -wordSize 8 -c -uPER -o \"%s\" %s %s" % (absASN1SCCpath, autosrc, acn, '"' + '" "'.join(inputFiles) + '"'))
+        mysystem("%s -c -uPER -o \"%s\" %s %s" % (absASN1SCCpath, autosrc, acn, '"' + '" "'.join(inputFiles) + '"'))
         for line in os.popen("%s -AdaUses %s" % (absASN1SCCpath, '" "'.join(inputASN1files))):
             g_AdaPackageNameOfType[line.split(':')[0]] = line.split(':')[1].rstrip()
     else:
-        mysystem("mono %s -wordSize 8 -c -uPER -o \"%s\" %s %s" % (absASN1SCCpath, autosrc, acn, '"' + '" "'.join(inputFiles) + '"'))
+        cmd = "mono %s -c -uPER -o \"%s\" %s %s" % (absASN1SCCpath, autosrc, acn, '"' + '" "'.join(inputFiles) + '"')
+        res = mysystem(cmd)
+        if res != 0:
+            panic("This command failed: %s\n" % cmd)
         for line in os.popen('mono  %s -AdaUses "%s"' % (absASN1SCCpath, '" "'.join(inputASN1files))):
             g_AdaPackageNameOfType[line.split(':')[0]] = line.split(':')[1].rstrip()
 
@@ -424,7 +427,7 @@ end Stream_Element_Buffer;
         if bAADLv2:
             o.write('    Deployment::ASN1_Module_Name => "%s";\n' % g_AdaPackageNameOfType[asnTypename].replace('_', '-'))
         if os.getenv('UPD') is None:
-            o.write('    Source_Language => ASN1;\n')
+            o.write('    Source_Language => (ASN1);\n')
         o.write('    -- Size of a buffer to cover all forms of message representation:\n')
         le_size = 0 if asnTypename not in messageSizes else messageSizes[asnTypename][0]
         o.write('    -- Real message size is %d; suggested aligned message buffer is...\n' % le_size)
@@ -487,6 +490,22 @@ end Stream_Element_Buffer;
             o.write('    Data_Model::Data_Representation => Struct;\n')
             o.write('END ' + cleanName + '_Buffer.impl;\n\n')
 
+    # Generate a SYSTEM in the DataView, otherwise Ocarina cannot parse it
+    # standalone. This allows buildsupport to get the list of ASN.1 files
+    # and modules, which is otherwise not visible unless those for which
+    # at least one type is referenced in a provided interface.
+    o.write('SYSTEM Taste_DataView\n')
+    o.write('END    Taste_DataView;\n\n')
+    o.write('SYSTEM IMPLEMENTATION Taste_DataView.others\n')
+    o.write('SUBCOMPONENTS\n')
+    for asnTypename in list(asnParser.g_names.keys()):
+        node = asnParser.g_names[asnTypename]
+        if node._isArtificial:
+            continue
+        cleanName = cleanNameAsAADLWants(asnTypename)
+        o.write('   %s : DATA %s;\n' % (cleanName, cleanName))
+    o.write('END Taste_DataView.others;\n')
+
     listOfAsn1Files = {}
     for asnTypename in list(asnParser.g_names.keys()):
         listOfAsn1Files[asnParser.g_names[asnTypename]._asnFilename] = 1
@@ -500,7 +519,7 @@ end Stream_Element_Buffer;
                 o.write('DATA ACN_' + fname + '\n')
                 o.write('PROPERTIES\n')
                 o.write('    Source_Text => ("' + possibleACN + '");\n')
-                o.write('    Source_Language => ACN;\n')
+                o.write('    Source_Language => (ACN);\n')
                 o.write('END ACN_' + fname + ';\n\n')
 
     o.write('end DataView;\n')

dmt/asn2dataModel.py

 #!/usr/bin/env python3
 # vim: set expandtab ts=8 sts=4 shiftwidth=4
 #
-# (C) Semantix Information Technologies.
+# (C) Semantix Information Technologies, Neuropublic, and European Space Agency
 #
 # Copyright 2014-2015 IB Krates <info@krates.ee>
 #       QGenc code generator integration
 #
-# Semantix Information Technologies is licensing the code of the
-# Data Modelling Tools (DMT) in the following dual-license mode:
-#
-# Commercial Developer License:
-#       The DMT Commercial Developer License is the suggested version
-# to use for the development of proprietary and/or commercial software.
-# This version is for developers/companies who do not want to comply
-# with the terms of the GNU Lesser General Public License version 2.1.
-#
-# GNU LGPL v. 2.1:
-#       This version of DMT is the one to use for the development of
-# applications, when you are willing to comply with the terms of the
-# GNU Lesser General Public License version 2.1.
-#
-# Note that in both cases, there are no charges (royalties) for the
-# generated code.
-#
+# License is GPL with runtime exception
 '''
 Model Translator
 
@@ -211,7 +195,8 @@ def main() -> None:
             names = uniqueASNfiles[asnFile][0]
             for nodeTypename in sorted(names):
                 # Check if this type must be skipped
-                if nodeTypename in badTypes:
+                if nodeTypename in badTypes and modelingLanguage.lower() != 'python':
+                    # all languages but python discard IA5Strings
                     continue
                 node = names[nodeTypename]
                 inform("Processing %s (%s)...", nodeTypename, modelingLanguage)
@@ -220,7 +205,7 @@ def main() -> None:
                 assert nodeTypename in leafTypeDict
 
                 leafType = leafTypeDict[nodeTypename]
-                if leafType in ['BOOLEAN', 'INTEGER', 'REAL', 'OCTET STRING']:
+                if leafType in ['BOOLEAN', 'INTEGER', 'REAL', 'OCTET STRING', 'AsciiString']:
                     processor = backend.OnBasic
                 elif leafType == 'SEQUENCE':
                     processor = backend.OnSequence

dmt/commonPy/asnParser.py

@@ -51,6 +51,7 @@ import copy
 import tempfile
 import re
 import distutils.spawn as spawn
+import hashlib
 
 import xml.sax  # type: ignore
 from typing import IO, TypeVar, Type, Optional, Callable, Union, List, Dict, Tuple, Any  # NOQA pylint: disable=W0611
@@ -87,11 +88,11 @@ g_modules = {}       # type: AST_Modules
 g_checkedSoFarForKeywords = {}  # type: Dict[str, int]
 
 g_invalidKeywords = [
-    "active", "adding", "all", "alternative", "and", "any", "as", "atleast", "axioms", "block", "call", "channel", "comment", "connect", "connection", "constant", "constants", "create", "dcl", "decision", "default", "else", "endalternative", "endblock", "endchannel", "endconnection", "enddecision", "endgenerator", "endmacro", "endnewtype", "endoperator", "endpackage", "endprocedure", "endprocess", "endrefinement", "endselect", "endservice", "endstate", "endsubstructure", "endsyntype", "endsystem", "env", "error", "export", "exported", "external", "fi", "finalized", "for", "fpar", "from", "gate", "generator", "if", "import", "imported", "in", "inherits", "input", "interface", "join", "literal", "literals", "macro", "macrodefinition", "macroid", "map", "mod", "nameclass", "newtype", "nextstate", "nodelay", "noequality", "none", "not", "now", "offspring", "operator", "operators", "or", "ordering", "out", "output", "package", "parent", "priority", "procedure", "process", "provided", "redefined", "referenced", "refinement", "rem", "remote", "reset", "return", "returns", "revealed", "reverse", "save", "select", "self", "sender", "service", "set", "signal", "signallist", "signalroute", "signalset", "spelling", "start", "state", "stop", "struct", "substructure", "synonym", "syntype", "system", "task", "then", "this", "timer", "to", "type", "use", "via", "view", "viewed", "virtual", "with", "xor", "end", "i", "j", "auto", "const",
+    "active", "adding", "all", "alternative", "and", "any", "as", "atleast", "axioms", "block", "call", "channel", "comment", "connect", "connection", "constant", "constants", "create", "dcl", "decision", "default", "else", "endalternative", "endblock", "endchannel", "endconnection", "enddecision", "endgenerator", "endmacro", "endnewtype", "endoperator", "endpackage", "endprocedure", "endprocess", "endrefinement", "endselect", "endservice", "endstate", "endsubstructure", "endsyntype", "endsystem", "env", "error", "export", "exported", "external", "fi", "finalized", "for", "fpar", "from", "gate", "generator", "if", "import", "imported", "in", "inherits", "input", "interface", "join", "literal", "literals", "macro", "macrodefinition", "macroid", "map", "mod", "nameclass", "newtype", "nextstate", "nodelay", "noequality", "none", "not", "now", "offspring", "operator", "operators", "or", "ordering", "out", "output", "package", "parent", "priority", "procedure", "process", "provided", "redefined", "referenced", "refinement", "rem", "remote", "reset", "return", "returns", "revealed", "reverse", "save", "select", "self", "sender", "service", "set", "signal", "signallist", "signalroute", "signalset", "spelling", "start", "state", "stop", "struct", "substructure", "synonym", "syntype", "system", "task", "then", "this", "to", "type", "use", "via", "view", "viewed", "virtual", "with", "xor", "end", "i", "j", "auto", "const",
     # From Nicolas Gillet/Astrium for SCADE
     "abstract", "activate", "and", "assume", "automaton", "bool", "case", "char", "clock", "const", "default", "div", "do", "else", "elsif", "emit", "end", "enum", "every", "false", "fby", "final", "flatten", "fold", "foldi", "foldw", "foldwi", "function", "guarantee", "group", "if", "imported", "initial", "int", "is", "last", "let", "make", "map", "mapfold", "mapi", "mapw", "mapwi", "match", "merge", "mod", "node", "not", "numeric", "of", "onreset", "open", "or", "package", "parameter", "pre", "private", "probe", "public", "real", "restart", "resume", "returns", "reverse", "sensor", "sig", "specialize", "state", "synchro", "tel", "then", "times", "transpose", "true", "type", "unless", "until", "var", "when", "where", "with", "xor",
     # From Maxime - ESA GNC Team
-    "open", "close", "flag"
+    "open", "close", "flag", "device", "range", "name"
 ]
 
 tokens = (
@@ -260,8 +261,11 @@ def VerifyAndFixAST() -> Dict[str, str]:
         # To cope with ReferenceTypes that redefine their
         # constraints (for now, ASN1SCC provides only INTEGERs)
         if isinstance(originalNode, AsnMetaType):
-            target = copy.copy(node)  # we need to keep the _asnFilename
+            target = copy.copy(node)  # type: ignore
+            # we need to keep the _asnFilename
             target._asnFilename = originalNode._asnFilename
+            if isinstance(node, AsnInt) and Min is not None and Max is not None:
+                target._range = [Min, Max]  # type: ignore
         elif isinstance(node, AsnInt) and Min is not None and Max is not None:
             target = copy.copy(node)  # we need to keep the Min/Max
             target._range = [Min, Max]
@@ -379,28 +383,59 @@ def CheckForInvalidKeywords(node_or_str: Union[str, AsnNode]) -> None:
 
 
 def ParseAsnFileList(listOfFilenames: List[str]) -> None:  # pylint: disable=invalid-sequence-index
+    # Add basic ASN.1 caching to avoid calling the ASN.1 compiler over and over
+    projectCache = os.getenv ("PROJECT_CACHE")
+    xmlAST = xmlAST2 = None
+    someFilesHaveChanged = False
+    if projectCache is not None:
+        filehash = hashlib.md5()
+        for each in sorted(listOfFilenames):
+            filehash.update(open(each).read().encode('utf-8'))
+            # also hash the file path: it is used in the AST in XML, so it is
+            # not enough to hash the content of the ASN.1 files, as two sets
+            # of files may have the same hash, that would lead to different XML
+            # content.
+            filehash.update(each.encode('utf-8'))
+        newHash = filehash.hexdigest()
+        # set the name of the XML files containing the dumped ASTs
+        xmlAST  = projectCache + os.sep + newHash + "_ast_v4.xml"
+        xmlAST2 = projectCache + os.sep + newHash + "_ast_v1.xml"
+        if not os.path.exists(xmlAST) or not os.path.exists(xmlAST2):
+            someFilesHaveChanged = True
+            print("[DMT] ASN.1 model changed, re-processing...")
+    else:
+        # no projectCache set, so xmlAST and xmlAST2 are set to None
+        someFilesHaveChanged = True
+    if not someFilesHaveChanged:
+        print("[DMT] No change in ASN.1 model.")
+
+    if not xmlAST:
+        (dummy, xmlAST) = tempfile.mkstemp()
+        os.fdopen(dummy).close()
+        xmlAST2 = xmlAST + "2"
+
     asn1SccPath = spawn.find_executable('asn1.exe')
     if asn1SccPath is None:
         utility.panic("ASN1SCC seems not installed on your system (asn1.exe not found in PATH).\n")
     else:
-        (dummy, xmlAST) = tempfile.mkstemp()
-        os.fdopen(dummy).close()
-        asn1SccDir = os.path.dirname(os.path.abspath(asn1SccPath))
-        spawnResult = os.system("mono \"" + asn1SccPath + "\" -customStg \"" + asn1SccDir + "/xml.stg:" + xmlAST + "\" -customStgAstVerion 4 \"" + "\" \"".join(listOfFilenames) + "\"")
-        if spawnResult != 0:
-            errCode = spawnResult / 256
-            if errCode == 1:
-                utility.panic("ASN1SCC reported syntax errors. Aborting...")
-            elif errCode == 2:
-                utility.panic("ASN1SCC reported semantic errors (or mono failed). Aborting...")
-            elif errCode == 3:
-                utility.panic("ASN1SCC reported internal error. Contact Semantix with this input. Aborting...")
-            elif errCode == 4:
-                utility.panic("ASN1SCC reported usage error. Aborting...")
-            else:
-                utility.panic("ASN1SCC generic error. Contact Semantix with this input. Aborting...")
+        if someFilesHaveChanged:
+            asn1SccDir = os.path.dirname(os.path.abspath(asn1SccPath))
+            spawnResult = os.system("mono \"" + asn1SccPath + "\" -customStg \"" + asn1SccDir + "/xml.stg:" + xmlAST + "\" -customStgAstVersion 4 \"" + "\" \"".join(listOfFilenames) + "\"")
+            if spawnResult != 0:
+                errCode = spawnResult / 256
+                if errCode == 1:
+                    utility.panic("ASN1SCC reported syntax errors. Aborting...")
+                elif errCode == 2:
+                    utility.panic("ASN1SCC reported semantic errors (or mono failed). Aborting...")
+                elif errCode == 3:
+                    utility.panic("ASN1SCC reported internal error. Contact ESA with this input. Aborting...")
+                elif errCode == 4:
+                    utility.panic("ASN1SCC reported usage error. Aborting...")
+                else:
+                    utility.panic("ASN1SCC generic error. Contact ESA with this input. Aborting...")
         ParseASN1SCC_AST(xmlAST)
-        os.unlink(xmlAST)
+        if projectCache is None:
+            os.unlink(xmlAST)
         g_names.update(g_names)
         g_leafTypeDict.update(g_leafTypeDict)
         g_checkedSoFarForKeywords.update(g_checkedSoFarForKeywords)
@@ -409,13 +444,15 @@ def ParseAsnFileList(listOfFilenames: List[str]) -> None:  # pylint: disable=inv
         # We also need to mark the artificial types -
         # So spawn the custom type output at level 1 (unfiltered)
         # and mark any types not inside it as artificial.
-        os.system("mono \"" + asn1SccPath + "\" -customStg \"" + asn1SccDir + "/xml.stg:" + xmlAST + "2\" -customStgAstVerion 1 \"" + "\" \"".join(listOfFilenames) + "\"")
+        if someFilesHaveChanged:
+            os.system("mono \"" + asn1SccPath + "\" -customStg \"" + asn1SccDir + "/xml.stg:" + xmlAST2 + "\" -customStgAstVersion 1 \"" + "\" \"".join(listOfFilenames) + "\"")
         realTypes = {}
-        for line in os.popen("grep  'ExportedType\>' \"" + xmlAST + "2\"").readlines():  # pylint: disable=anomalous-backslash-in-string
+        for line in os.popen("grep  'ExportedType\>' \"" + xmlAST2 + "\"").readlines():  # flake8: noqa pylint: disable=anomalous-backslash-in-string
             line = re.sub(r'^.*Name="', '', line.strip())
             line = re.sub(r'" />$', '', line)
             realTypes[line] = 1
-        os.unlink(xmlAST + "2")
+        if projectCache is None:
+            os.unlink(xmlAST2)
         for nodeTypename in list(g_names.keys()):
             if nodeTypename not in realTypes:
                 g_names[nodeTypename]._isArtificial = True

dmt/commonPy/cleanupNodes.py

-# (C) Semantix Information Technologies.
-#
-# Semantix Information Technologies is licensing the code of the
-# Data Modelling Tools (DMT) in the following dual-license mode:
-#
-# Commercial Developer License:
-#       The DMT Commercial Developer License is the appropriate version
-# to use for the development of proprietary and/or commercial software.
-# This version is for developers/companies who do not want to share
-# the source code they develop with others or otherwise comply with the
-# terms of the GNU Lesser General Public License version 3.
-#
-# GNU LGPL v. 2.1:
-#       This version of DMT is the one to use for the development of
-# non-commercial applications, when you are willing to comply
-# with the terms of the GNU Lesser General Public License version 3.
-#
-# The features of the two licenses are summarized below:
-#
-#                       Commercial
-#                       Developer               LGPL
-#                       License
-#
-# License cost          License fee charged     No license fee
-#
-# Must provide source
-# code changes to DMT   No, modifications can   Yes, all source code
-#                       be closed               must be provided back
-#
-# Can create            Yes, that is,           No, applications are subject
-# proprietary           no source code needs    to the LGPL and all source code
-# applications          to be disclosed         must be made available
-#
-# Support               Yes, 12 months of       No, but available separately
-#                       premium technical       for purchase
-#                       support
-#
-# Charge for Runtimes   None                    None
+# (C)  Semantix Information Technologies,
+#      Neuropublic,
+#      European Space Agency
 #
+# The license of the Data Modelling Tools (DMT) is GPL with Runtime Exception
+
 '''
 Rules to gather the list of types that must be skipped
 '''

requirements.txt

 coverage>=3.7.1
-flake8>=2.6.0
-mypy>=0.530
+flake8==2.6.0
+mypy==0.530
 pyflakes>=1.2.3
 pylint>=1.7.0
 pytest>=2.6.3

setup.py

@@ -11,7 +11,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='dmt',
-    version="2.1.10",
+    version="2.1.11",
     packages=find_packages(),
     author='Thanassis Tsiodras',
     author_email='Thanassis.Tsiodras@esa.int',

tests-coverage/DataTypesSimulink.asn

@@ -107,9 +107,34 @@ myVar MyInt ::= 4
 ConfigString ::= IA5String (SIZE(1..20))
 myStrVar ConfigString ::= "This is a test"
 
-T-TypeThatMustNotBeMapped ::= SEQUENCE {
+FixedLenConfigString ::= IA5String (SIZE (5))
+myStrFixed FixedLenConfigString ::= "Hello"
+
+T-TypeThatMustNotBeMappedExceptInPython ::= SEQUENCE {
     config  ConfigString,
-    param   INTEGER (0..10)
+    param   INTEGER (0..10),
+   fixstr  FixedLenConfigString DEFAULT "Hello"
+}
+E ::= INTEGER (0..255|1299)(5)
+TypeWithOptional ::= SEQUENCE {
+   a BOOLEAN OPTIONAL,
+   b INTEGER (0..255|1299),
+   c INTEGER (0..255),
+   d T-TypeThatMustNotBeMappedExceptInPython DEFAULT {config "Config", param 5, fixstr "World"}
+}
+
+push-it T-TypeThatMustNotBeMappedExceptInPython ::= {config "Config", param 5, fixstr "World"}
+
+SubTypeWithOptional ::= TypeWithOptional (WITH COMPONENTS {a ABSENT, c (42), b (1299)})
+
+SuperChoice ::= CHOICE {
+   first-choice BOOLEAN,
+   second-choice INTEGER (0..255),
+   third-choice TypeWithOptional