Commit 68130ef2 authored by Thanassis Tsiodras's avatar Thanassis Tsiodras

Merge remote-tracking branch 'dmt-github/master'

parents 154e290b 7e2d0e95
......@@ -9,3 +9,7 @@ tests-coverage/Simulink_DataView_asn.m
*swp
antlr-2.7.7
asn1scc
tests-sqlalchemy/sql.log
tests-sqlalchemy/test.db
tests-sqlalchemy/verifier/LotsOfDataTypes.asn
tests-sqlalchemy/verifier/asn2dataModel/
......@@ -8,7 +8,8 @@ PY_SRC:=$(filter-out dmt/B_mappers/antlr.main.py dmt/A_mappers/Stubs.py, ${PY_SR
TYPING_FOLDER:=$(shell pip3 show typing | grep ^Location | sed 's,^.*: ,,')
export PYTHONPATH=${TYPING_FOLDER}
all: flake8 pylint mypy coverage
# all: flake8 pylint mypy coverage testDB
all: flake8 mypy coverage testDB
flake8:
@echo Performing syntax checks via flake8...
......@@ -26,4 +27,10 @@ coverage:
@echo Performing coverage checks...
@$(MAKE) -C tests-coverage || exit 1
testDB:
@echo Installing DMT for local user...
@pip3 install .
@echo Performing database tests...
@$(MAKE) -C tests-sqlalchemy || exit 1
.PHONY: flake8 pylint mypy coverage
......@@ -7,9 +7,9 @@ dependencies:
- "~/.apt-cache"
pre:
- sudo rm -rf /var/cache/apt/archives && sudo ln -s ~/.apt-cache /var/cache/apt/archives && mkdir -p ~/.apt-cache/partial
- sudo apt-get update
- sudo apt-get install libxslt-dev libxml2-dev
- wget -O - -q https://github.com/ttsiodras/asn1scc/releases/download/3.2.81/asn1scc-bin-3.2.81.tar.gz | tar zxvf -
# - sudo apt-get update
- sudo apt-get install libxslt-dev libxml2-dev python-psycopg2
- wget -O - -q https://github.com/ttsiodras/asn1scc/releases/download/3.3.9/asn1scc-bin-3.3.9.tar.bz2 | tar jxvf -
- wget -O - -q https://github.com/ttsiodras/DataModellingTools/files/335591/antlr-2.7.7.tar.gz | tar zxvf - ; cd antlr-2.7.7/lib/python ; sudo pip2 install .
- sudo apt-get install mono-complete
......@@ -18,4 +18,5 @@ dependencies:
test:
override:
- sudo pip2 install SQLAlchemy psycopg2
- PATH=$PATH:$(pwd)/asn1scc make
......@@ -18,6 +18,8 @@
# Note that in both cases, there are no charges (royalties) for the
# generated code.
#
from __future__ import absolute_import
import os
import re
import copy
......@@ -272,7 +274,7 @@ An example for SetLength:
bridgeFunc = getattr(JMP, Clean(self._nodeTypeName) + "_" + self._Caccessor + "_Set" + args.get("postfix", ""))
if isinstance(value, float):
ctypesValue = c_double(value)
elif isinstance(value, int):
elif isinstance(value, (int, long)):
ctypesValue = c_longlong(value)
else:
ctypesValue = value
......
......@@ -141,9 +141,9 @@ def OnStartup(unused_modelingLanguage: str, asnFile: str, outputDir: str, badTyp
# and used during comparisons of incoming TMs (For MSCs)
# mono_exe = "mono " if sys.argv[0].endswith('.py') and sys.platform.startswith('linux') else ""
mono_exe = ""
mono_exe = "mono"
makefile_text = '''\
ASN1SCC:=asn1.exe
ASN1SCC:=$(shell which asn1.exe)
ASN2DATAMODEL:=asn2dataModel
GRAMMAR := %(origGrammarBase)s
BASEGRAMMAR := %(base)s
......@@ -157,8 +157,8 @@ $(BDIR)/$(GRAMMAR)_getset.c: $(GRAMMAR).asn
%(tab)s$(ASN2DATAMODEL) -toPython -o $(BDIR) $<
$(BDIR)/asn1crt.c $(BDIR)/$(GRAMMAR).c $(BDIR)/real.c $(BDIR)/acn.c $(BDIR)/$(GRAMMAR).h $(BDIR)/asn1crt.h: $(GRAMMAR).asn
%(tab)sif [ ! -f "$(GRAMMAR).acn" ] ; then %(mono)s$(ASN1SCC) -ACND -o $(BDIR) $< ; fi
%(tab)s%(mono)s$(ASN1SCC) -ACN -c -uPER -equal -wordSize 8 -o $(BDIR) $< $(GRAMMAR).acn
%(tab)sif [ ! -f "$(GRAMMAR).acn" ] ; then %(mono)s $(ASN1SCC) -ACND -o $(BDIR) $< ; fi
%(tab)s%(mono)s $(ASN1SCC) -ACN -c -uPER -equal -wordSize 8 -o $(BDIR) $< $(GRAMMAR).acn
$(BDIR)/DV.py: $(GRAMMAR).asn
%(tab)sgrep 'REQUIRED_BYTES_FOR_.*ENCODING' $(BDIR)/$(GRAMMAR).h | awk '{print $$2 " = " $$3}' > $@
......@@ -495,7 +495,7 @@ def DumpTypeDumper(
lines.append(codeIndent + "state = self.GetState()")
lines.append(codeIndent + "length = %s.GetLength()" % variableName)
lines.append(codeIndent + "self.Reset(state)")
lines.append(codeIndent + "map(partial(emitElem, %s), xrange(length))" % variableName)
lines.append(codeIndent + "map(partial(emitElem, %s), range(length))" % variableName)
lines.append(codeIndent + 'self.Reset(state)')
lines.append(codeIndent + 'lines.append("}")')
......
......@@ -142,6 +142,7 @@ def FixupAstForSQL() -> None:
if not neededToAddPseudoType():
break
g_bStartupRun = False
......@@ -289,6 +290,7 @@ def CreateSequenceOf(nodeTypename: str, node: AsnSequenceOrSetOf, unused_leafTyp
def CreateChoice(nodeTypename: str, node: AsnChoice, leafTypeDict: AST_Leaftypes) -> None:
CommonSeqSetChoice(nodeTypename, node, leafTypeDict, True)
g_bShutdownRun = False
......
......@@ -500,7 +500,7 @@ class {cleanTypename}_SQL(Base):
def __init__(self, pyObj):
self._children = []
state = pyObj.GetState()
for i in xrange(pyObj.GetLength()):
for i in range(pyObj.GetLength()):
pyObj.Reset(state)
newIndex = {cleanTypename}_indexes_SQL()
newIndex.idx = i
......@@ -522,6 +522,7 @@ class {cleanTypename}_SQL(Base):
def CreateChoice(nodeTypename: str, node: AsnChoice, _: AST_Leaftypes) -> None:
CommonSeqSetChoice(nodeTypename, node, _, isChoice=True)
g_bShutdownRun = False
......
......@@ -176,6 +176,7 @@ void TeleCmds::CreateControls()
''')
enumFieldNames = {} # type: Dict[str, int]
......@@ -972,6 +973,7 @@ def WriteCodeForAction(nodeTypename: str,
g_MyAction.write(" }\n")
g_MyAction.write("}\n")
g_SPs = [] # type: List[str]
g_bBraceOpen = False
......
......@@ -71,10 +71,16 @@ def OnStartup(modelingLanguage: str,
global g_PythonFile
if g_PythonFile is None:
g_PythonFile = open(outputDir + "python/PythonController.py", "w")
g_headerPython.append("from __future__ import absolute_import\n\n")
g_headerPython.append("import threading, time, sys, os, ctypes\n")
# g_headerPython.append("from PythonAccess import *")
g_headerPython.append("import DV")
g_headerPython.append('PythonAccess = ctypes.cdll.LoadLibrary("./PythonAccess.so")')
g_headerPython.append('try:')
g_headerPython.append(' PythonAccess = ctypes.cdll.LoadLibrary("./PythonAccess.so")')
g_headerPython.append('except OSError:')
g_headerPython.append(' folder = os.path.dirname(os.path.abspath(__file__))')
g_headerPython.append(' PythonAccess = ctypes.cdll.LoadLibrary(folder + "/PythonAccess.so")')
g_headerPython.append(' sys.path.append(folder)')
g_headerPython.append('OpenMsgQueueForReading = PythonAccess.OpenMsgQueueForReading')
g_headerPython.append('OpenMsgQueueForReading.restype = ctypes.c_int')
g_headerPython.append('CloseMsgQueue = PythonAccess.CloseMsgQueue')
......@@ -138,7 +144,7 @@ def OnStartup(modelingLanguage: str,
if modelingLanguage.lower() == "gui_pi":
# We have telemetry, we need a thread polling the /xyz_PI_queue (xyz: g_maybeFVname)
g_headerPython.append("import " + g_asn_name + "_asn")
g_headerPython.append("from . import " + g_asn_name + "_asn")
g_bodyPython.append("class Poll_" + cleanFVname + "(threading.Thread):")
g_bodyPython.append(" def run(self):")
g_bodyPython.append(' self._bDie = False')
......
#!/usr/bin/env python
#!/usr/bin/env python3
"""
ASN.1 Importer
......@@ -86,14 +86,14 @@ def calculateForNativeAndASN1SCC(absASN1SCCpath, autosrc, names, inputFiles):
acn = " -ACN " if any(x.lower().endswith(".acn") for x in inputFiles) else ""
inputASN1files = [x for x in inputFiles if not x.lower().endswith('.acn')]
# Spawn ASN1SCC.exe compiler
if platform.system() == "Windows":
# Spawn ASN1SCC.exe compiler - for MacOS define a new sh file calling mono Asn1f2.exe
if platform.system() == "Windows" or platform.system() == "Darwin":
mysystem("%s -wordSize 8 -c -uPER -o \"%s\" %s %s" % (absASN1SCCpath, autosrc, acn, '"' + '" "'.join(inputFiles) + '"'))
for line in os.popen("%s -AdaUses %s" % (absASN1SCCpath, '" "'.join(inputASN1files))):
g_AdaPackageNameOfType[line.split(':')[0]] = line.split(':')[1].rstrip()
else:
mysystem("mono %s -wordSize 8 -c -uPER -o \"%s\" %s %s" % (absASN1SCCpath, autosrc, acn, '"' + '" "'.join(inputFiles) + '"'))
for line in os.popen('mono %s -AdaUses "%s"' % (absASN1SCCpath, '" "'.join(inputASN1files))):
for line in os.popen('mono %s -AdaUses "%s"' % (absASN1SCCpath, '" "'.join(inputASN1files))):
g_AdaPackageNameOfType[line.split(':')[0]] = line.split(':')[1].rstrip()
msgEncoderFile = open(autosrc + os.sep + base + ".stats.c", 'w')
......@@ -129,7 +129,6 @@ def calculateForNativeAndASN1SCC(absASN1SCCpath, autosrc, names, inputFiles):
msgEncoderFile.close()
# Code generation - asn1c part
# Create a dictionary to lookup the asn-types from their corresponding c-type
namesDict = {}
for asnTypename in list(names.keys()):
......@@ -144,8 +143,8 @@ def calculateForNativeAndASN1SCC(absASN1SCCpath, autosrc, names, inputFiles):
pipe = Popen("find-supported-compilers", stdout=PIPE).stdout
g_platformCompilers = pipe.read().splitlines()
except OSError as err:
print('Not running in a TASTE environment: {}\nUsing GCC only for computing sizeofs'.format(str(err)))
g_platformCompilers = ['gcc']
print('Not running in a TASTE Environment: {}\nUsing GCC only for computing sizeofs'.format(str(err)))
g_platformCompilers = ['gcc'.encode()]
# Get the maximum size of each asn1type from all platform compilers
messageSizes = {}
for cc in g_platformCompilers:
......@@ -155,17 +154,19 @@ def calculateForNativeAndASN1SCC(absASN1SCCpath, autosrc, names, inputFiles):
path_to_compiler = spawn.find_executable(cc.decode('utf-8'))
if path_to_compiler is None:
continue
for cfile in os.listdir("."):
if cfile.endswith(".c"):
if mysystem('%s -c -std=c99 -I. "%s" 2>"%s.stats.err"' % (path_to_compiler, cfile, base)) != 0:
panic("Compilation of generated sources failed - is %s installed?\n"
"(report inside '%s')\n" % (cc, os.path.join(autosrc, base + ".stats.err")))
os.chdir(pwd)
# Receive the size information for each value from the compiled object file
for line in os.popen("nm --print-size " + autosrc + os.sep + base + ".stats.o").readlines():
if platform.system() == "Darwin":
nm = "gnm"
else:
nm = "nm"
for line in os.popen( nm + " --print-size " + autosrc + os.sep + base + ".stats.o").readlines():
try:
(dummy, size, dummy2, msg) = line.split()
except ValueError:
......
......@@ -241,6 +241,7 @@ def main() -> None:
if 'OnShutdown' in dir(backend):
backend.OnShutdown(badTypes)
if __name__ == "__main__":
if "-pdb" in sys.argv:
sys.argv.remove("-pdb") # pragma: no cover
......
......@@ -385,10 +385,8 @@ def ParseAsnFileList(listOfFilenames: List[str]) -> None: # pylint: disable=inv
else:
(dummy, xmlAST) = tempfile.mkstemp()
os.fdopen(dummy).close()
# spawnResult = os.system("mono \""+asn1SccPath+"\" -ast \""+xmlAST+"\" \"" + "\" \"".join(listOfFilenames) + "\"")
asn1SccDir = os.path.dirname(os.path.abspath(asn1SccPath))
mono = "mono " if sys.argv[0].endswith('.py') and sys.platform.startswith('linux') else ""
spawnResult = os.system(mono + "\"" + asn1SccPath + "\" -customStg \"" + asn1SccDir + "/xml.stg:" + xmlAST + "\" -customStgAstVerion 4 \"" + "\" \"".join(listOfFilenames) + "\"")
spawnResult = os.system("mono \"" + asn1SccPath + "\" -customStg \"" + asn1SccDir + "/xml.stg:" + xmlAST + "\" -customStgAstVerion 4 \"" + "\" \"".join(listOfFilenames) + "\"")
if spawnResult != 0:
errCode = spawnResult / 256
if errCode == 1:
......@@ -411,7 +409,7 @@ def ParseAsnFileList(listOfFilenames: List[str]) -> None: # pylint: disable=inv
# We also need to mark the artificial types -
# So spawn the custom type output at level 1 (unfiltered)
# and mark any types not inside it as artificial.
os.system(mono + "\"" + asn1SccPath + "\" -customStg \"" + asn1SccDir + "/xml.stg:" + xmlAST + "2\" -customStgAstVerion 1 \"" + "\" \"".join(listOfFilenames) + "\"")
os.system("mono \"" + asn1SccPath + "\" -customStg \"" + asn1SccDir + "/xml.stg:" + xmlAST + "2\" -customStgAstVerion 1 \"" + "\" \"".join(listOfFilenames) + "\"")
realTypes = {}
for line in os.popen("grep 'ExportedType\>' \"" + xmlAST + "2\"").readlines(): # pylint: disable=anomalous-backslash-in-string
line = re.sub(r'^.*Name="', '', line.strip())
......@@ -815,7 +813,7 @@ def VisitAsn1Module(xmlAsn1File: Element, xmlModule: Element, modules: List[Modu
def ParseASN1SCC_AST(filename: str) -> None:
parser = xml.sax.make_parser()
parser = xml.sax.make_parser([])
handler = InputFormatXMLHandler()
parser.setContentHandler(handler)
# parser.setFeature("http://xml.org/sax/features/validation", True)
......@@ -986,6 +984,7 @@ def test_xml() -> None:
print("\nRe-created grammar:\n\n")
PrintGrammarFromASTtoStdOut()
if __name__ == "__main__":
if "-testXML" in sys.argv:
sys.argv.remove("-testXML")
......
coverage==3.7.1
flake8==2.6.0
mypy-lang==0.4.2
pyflakes==1.2.3
pylint==1.5.6
pytest==2.6.3
lxml==3.6.0
astroid==1.4.6
pycodestyle==2.0.0
typing==3.5.2.2
coverage>=3.7.1
flake8>=2.6.0
mypy-lang>=0.4.2
pyflakes>=1.2.3
pylint>=1.7.0
pytest>=2.6.3
lxml>=3.6.0
astroid>=1.4.6
pycodestyle>=2.0.0
typing>=3.5.2.2
......@@ -26,15 +26,15 @@ setup(
'Programming Language :: Python :: 3.5'
],
install_requires=[
'coverage==3.7.1',
'flake8==2.6.0',
'mypy-lang==0.4.2',
'pyflakes==1.2.3',
'pylint==1.5.6',
'pytest==2.6.3',
'astroid==1.4.6',
'pycodestyle==2.0.0',
'typing==3.5.2.2',
'coverage>=3.7.1',
'flake8>=2.6.0',
'mypy-lang>=0.4.2',
'pyflakes>=1.2.3',
'pylint>=1.7.0',
'pytest>=2.6.3',
'astroid>=1.4.6',
'pycodestyle>=2.0.0',
'typing>=3.5.2.2',
],
entry_points={
'console_scripts': [
......
MY-MODULE DEFINITIONS AUTOMATIC TAGS ::= BEGIN
-- Original definition by Alain
--T-POS ::= BOOLEAN
MyInt ::= INTEGER (0 .. 20)
My2ndInt ::= MyInt ( 1 .. 18)
MySeq ::= SEQUENCE {
anInt MyInt,
anotherInt My2ndInt
}
MyChoice ::= CHOICE {
anInt MyInt,
aReal REAL (0.0 .. 10.0)
}
AType ::= SEQUENCE {
blArray SEQUENCE (SIZE(10)) OF BOOLEAN
}
MySuperSeq ::= SEQUENCE {
aSeq MySeq,
anotherSeq MySeq,
anArray T-ARR3,
aString OCTET STRING (SIZE(1..10))
}
My2ndAType ::= AType
TypeEnumerated ::= ENUMERATED {
red(0),
green(1),
blue(2)
}
My2ndEnumerated ::= TypeEnumerated
TypeNested ::= SEQUENCE {
intVal INTEGER(0..10),
int2Val INTEGER(-10..10),
int3Val MyInt (10..12),
intArray SEQUENCE (SIZE (10)) OF INTEGER (0..3),
realArray SEQUENCE (SIZE (10)) OF REAL (0.1 .. 3.14),
octStrArray SEQUENCE (SIZE (10)) OF OCTET STRING (SIZE(1..10)),
boolArray SEQUENCE (SIZE (10)) OF T-BOOL,
enumArray SEQUENCE (SIZE (10)) OF TypeEnumerated,
enumValue TypeEnumerated,
enumValue2 ENUMERATED {
truism(0),
falsism(1)
},
label OCTET STRING (SIZE(10..40)),
bAlpha T-BOOL,
bBeta BOOLEAN,
sString T-STRING,
arr T-ARR,
arr2 T-ARR2
}
My2ndTypeNested ::= TypeNested
T-POS ::= CHOICE {
longitude REAL(-180.0..180.0),
latitude REAL(-90.0..90.0),
height REAL(30000.0..45000.0),
subTypeArray SEQUENCE (SIZE(10..15)) OF TypeNested,
label OCTET STRING (SIZE(50)),
intArray T-ARR,
myIntSet T-SET,
myIntSetOf T-SETOF,
anInt My2ndInt
}
T-POS-SET ::= SET {
longitude REAL(-180.0..180.0),
latitude REAL(-90.0..90.0),
height REAL(30000.0..45000.0),
subTypeArray SEQUENCE (SIZE(10..15)) OF TypeNested,
label OCTET STRING (SIZE(20..50)),
intArray T-ARR,
myIntSet T-SET,
myIntSetOf T-SETOF
}
T-ARR ::= SEQUENCE (SIZE (5..6)) OF INTEGER (0..32764)
My2ndArr ::= T-ARR
T-ARR2 ::= SEQUENCE (SIZE (5..6)) OF REAL (0.1 .. 4.2)
T-ARR3 ::= SEQUENCE (SIZE (5..6)) OF SEQUENCE (SIZE(7)) OF REAL (0.1 .. 39.2)
T-ARR4 ::= SEQUENCE (SIZE (5..6)) OF SEQUENCE (SIZE(7)) OF REAL (0.1 .. 4.3)
T-SET ::= SET {
data1 INTEGER(0..131071),
data2 REAL(-100.0 .. 10.0),
data3 INTEGER(-1024..1024),
data4 INTEGER(-1310720..131071)
}
T-SETOF ::= SET (SIZE (5..6)) OF INTEGER (0..4)
T-BOOL ::= BOOLEAN
My2ndBool ::= T-BOOL
T-INT ::= INTEGER (0..50)
T-REAL ::= REAL (-3.14..3.14)
My2ndReal ::= T-REAL
T-STRING ::= OCTET STRING (SIZE(10..15))
My2ndString ::= T-STRING
T-FIXEDSTRING ::= OCTET STRING (SIZE(15))
T-META ::= T-POS
END