Removed duplicate code from XML parser (#376)

* Added: handling of Byte, SByte and DateTime convertion from ua to python

* log nodeid in addressspace

* Updated: Removed some duplicate xml parsing.

_parse_value was parsing the child of a node as a constant.
Making it parse a node directly and not its child allows recursive calls
to parse ListOf elements.
_parse_containted_value as been added as a wrapper to obtain former
behavior.
Now ua_type_to_python is simply a wrapper to ua_utils.string_to_val and
_to_bool a wrapper to ua_type_to_python.
All value parsing is performed by ua_utils.string_to_value.
Missing types have been listed as FIXME.

* Added: tests for XML parsing.

Added some tests in tests/tests_xml.py for datatype DateTime (+list),
QualifiedName (+list), ListOfGuid and ListOfExtensionObjects.
Few tweaks in xmlimporter: (i) DateTime is now parsed in string_to_val
and double parsing was causing an error, (ii) single Guid were converted
to UUID while lists were not by default list case.
Another possibility would be either to make the conversion to UUID in xmlparser
or to make the _add_variable_value also recursive but I'm not sure of
other possible consequences.
Finaly it seems according to specs that all DateTime should be timezone
aware so I forced UTC when a timezone was not precised.
Maybe an exception should be raised?

* Added: pytz in requirements

* Added: pytz in travis requirements

* Added: pytz in travis requirements

.travis.yml

@@ -2,18 +2,21 @@ language: python
 python:
   - "2.7"
   - "3.4"
-  - "pypy"  
+  - "pypy"
 # command to install dependencies
 install:
   - pip install python-dateutil
   - if [[ $TRAVIS_PYTHON_VERSION == '3.4' ]]; then pip install cryptography ; fi
+  - if [[ $TRAVIS_PYTHON_VERSION == '3.4' ]]; then pip install pytz ; fi
   - if [[ $TRAVIS_PYTHON_VERSION == '2.7' ]]; then pip install futures ; fi
   - if [[ $TRAVIS_PYTHON_VERSION == '2.7' ]]; then pip install cryptography ; fi
   - if [[ $TRAVIS_PYTHON_VERSION == '2.7' ]]; then pip install trollius ; fi
   - if [[ $TRAVIS_PYTHON_VERSION == '2.7' ]]; then pip install enum34 ; fi
+  - if [[ $TRAVIS_PYTHON_VERSION == '2.7' ]]; then pip install pytz ; fi
     #- if [[ $TRAVIS_PYTHON_VERSION == 'pypy3' ]]; then pip install cryptography ; fi
   - if [[ $TRAVIS_PYTHON_VERSION == 'pypy' ]]; then pip install futures ; fi
   - if [[ $TRAVIS_PYTHON_VERSION == 'pypy' ]]; then pip install trollius ; fi
   - if [[ $TRAVIS_PYTHON_VERSION == 'pypy' ]]; then pip install enum34 ; fi
+  - if [[ $TRAVIS_PYTHON_VERSION == 'pypy' ]]; then pip install pytz ; fi
 # command to run tests
 script: ./run-tests.sh

opcua/common/ua_utils.py

@@ -79,14 +79,14 @@ def string_to_val(string, vtype):
         else:
             val = False
     elif vtype in (ua.VariantType.Int16, ua.VariantType.Int32, ua.VariantType.Int64):
-            val = int(string)
+        val = int(string)
     elif vtype in (ua.VariantType.UInt16, ua.VariantType.UInt32, ua.VariantType.UInt64):
         val = int(string)
     elif vtype in (ua.VariantType.Float, ua.VariantType.Double):
         val = float(string)
     elif vtype in (ua.VariantType.String, ua.VariantType.XmlElement):
         val = string
-    elif vtype in (ua.VariantType.SByte, ua.VariantType.ByteString):
+    elif vtype in (ua.VariantType.Byte, ua.VariantType.SByte, ua.VariantType.ByteString):
         val = string.encode("utf-8")
     elif vtype in (ua.VariantType.NodeId, ua.VariantType.ExpandedNodeId):
         val = ua.NodeId.from_string(string)

opcua/common/xmlimporter.py

@@ -233,6 +233,10 @@ class XmlImporter(object):
                 extobj = self._make_ext_obj(ext)
                 values.append(extobj)
             return values
+        elif obj.valuetype == 'ListOfGuid':
+            return ua.Variant([
+                uuid.UUID(guid) for guid in obj.value
+            ], getattr(ua.VariantType, obj.valuetype[6:]))
         elif obj.valuetype.startswith("ListOf"):
             vtype = obj.valuetype[6:]
             if hasattr(ua.ua_binary.Primitives, vtype):
@@ -242,8 +246,6 @@ class XmlImporter(object):
         elif obj.valuetype == 'ExtensionObject':
             extobj = self._make_ext_obj(obj.value)
             return ua.Variant(extobj, getattr(ua.VariantType, obj.valuetype))
-        elif obj.valuetype == 'DateTime':
-            return ua.Variant(dateutil.parser.parse(obj.value), getattr(ua.VariantType, obj.valuetype))
         elif obj.valuetype == 'Guid':
             return ua.Variant(uuid.UUID(obj.value), getattr(ua.VariantType, obj.valuetype))
         elif obj.valuetype == 'LocalizedText':

opcua/common/xmlparser.py

@@ -2,32 +2,28 @@
 parse xml file from opcua-spec
 """
 import logging
+from pytz import utc
+import uuid
 import re
 import sys
 
 import xml.etree.ElementTree as ET
 
+from opcua.common import ua_utils
+from opcua import ua
+
+
+def ua_type_to_python(val, uatype_as_str):
+    """
+    Converts a string value to a python value according to ua_utils.
+    """
+    return ua_utils.string_to_val(val, getattr(ua.VariantType, uatype_as_str))
 
 def _to_bool(val):
-    return val in ("True", "true", "on", "On", "1")
-
-
-def ua_type_to_python(val, uatype):
-    if uatype.startswith("Int") or uatype.startswith("UInt"):
-        return int(val)
-    elif uatype.lower().startswith("bool"):
-        return _to_bool(val)
-    elif uatype in ("Double", "Float"):
-        return float(val)
-    elif uatype == "String":
-        return val
-    elif uatype in ("Bytes", "Bytes", "ByteString", "ByteArray"):
-        if sys.version_info.major > 2:
-            return bytes(val, 'utf8')
-        else:
-            return val
-    else:
-        raise Exception("uatype nopt handled", uatype, " for val ", val)
+    """
+    Easy access to boolean conversion.
+    """
+    return ua_type_to_python(val, "Boolean")
 
 
 class NodeData(object):
@@ -197,7 +193,7 @@ class XMLParser(object):
         elif tag == "References":
             self._parse_refs(el, obj)
         elif tag == "Value":
-            self._parse_value(el, obj)
+            self._parse_contained_value(el, obj)
         elif tag == "InverseName":
             obj.inversename = el.text
         elif tag == "Definition":
@@ -206,66 +202,77 @@ class XMLParser(object):
         else:
             self.logger.info("Not implemented tag: %s", el)
 
+    def _parse_contained_value(self, el, obj):
+        """
+        Parse the child of el as a constant.
+        """
+        val_el = el.find(".//")  # should be only one child
+        self._parse_value(val_el, obj)
+
     def _parse_value(self, val_el, obj):
-        child_el = val_el.find(".//")  # should be only one child
-        if child_el is not None:
-            ntag = self._retag.match(child_el.tag).groups()[1]
+        """
+        Parse the node val_el as a constant.
+        """
+        if val_el is not None:
+            ntag = self._retag.match(val_el.tag).groups()[1]
         else:
             ntag = "Null"
-        obj.valuetype = ntag
 
-        if ntag in ("Int8", "UInt8", "Int16", "UInt16", "Int32", "UInt32", "Int64", "UInt64"):
-            obj.value = int(child_el.text)
-        elif ntag in ("Float", "Double"):
-            obj.value = float(child_el.text)
-        elif ntag == "Boolean":
-            obj.value = _to_bool(child_el.text)
+        obj.valuetype = ntag
+        if ntag == "Null":
+            obj.value = None
+        elif hasattr(ua.ua_binary.Primitives1, ntag):
+            # Elementary types have their parsing directly relying on ua_type_to_python.
+            obj.value = ua_type_to_python(val_el.text, ntag)
+        elif ntag == "DateTime":
+            obj.value = ua_type_to_python(val_el.text, ntag)
+            # According to specs, DateTime should be either UTC or with a timezone.
+            if obj.value.tzinfo is None or obj.value.tzinfo.utcoffset(obj.value) is None:
+                utc.localize(obj.value) # FIXME Forcing to UTC if unaware, maybe should raise?
         elif ntag in ("ByteString", "String"):
-            mytext = child_el.text
-            if mytext is None:  # support importing null strings
+            mytext = ua_type_to_python(val_el.text, ntag)
+            if mytext is None:
+                # Support importing null strings.
                 mytext = ""
             mytext = mytext.replace('\n', '').replace('\r', '')
             obj.value = mytext
-        elif ntag == "DateTime":
-            obj.value = child_el.text
         elif ntag == "Guid":
-            self._parse_value(child_el, obj)
-            obj.valuetype = obj.datatype  # override parsed string type to guid
-        elif ntag == "LocalizedText":
-            obj.value = self._parse_body(child_el)
+            self._parse_contained_value(val_el, obj)
+            # Override parsed string type to guid.
+            obj.valuetype = ntag
         elif ntag == "NodeId":
-            id_el = child_el.find("uax:Identifier", self.ns)
+            id_el = val_el.find("uax:Identifier", self.ns)
             if id_el is not None:
                 obj.value = id_el.text
-        elif ntag == "ListOfExtensionObject":
-            obj.value = self._parse_list_of_extension_object(child_el)
+        elif ntag == "ExtensionObject":
+            obj.value = self._parse_ext_obj(val_el)
+        elif ntag == "LocalizedText":
+            obj.value = self._parse_body(val_el)
         elif ntag == "ListOfLocalizedText":
-            obj.value = self._parse_list_of_localized_text(child_el)
+            obj.value = self._parse_list_of_localized_text(val_el)
+        elif ntag == "ListOfExtensionObject":
+            obj.value = self._parse_list_of_extension_object(val_el)
         elif ntag.startswith("ListOf"):
-            obj.value = self._parse_list(child_el)
-        elif ntag == "ExtensionObject":
-            obj.value = self._parse_ext_obj(child_el)
-        elif ntag == "Null":
-            obj.value = None
+            # Default case for "ListOf" types.
+            # Should stay after particular cases (e.g.: "ListOfLocalizedText").
+            obj.value = []
+            for val_el in val_el:
+                tmp = NodeData()
+                self._parse_value(val_el, tmp)
+                obj.value.append(tmp.value)
         else:
+            # Missing according to string_to_val: XmlElement, ExpandedNodeId,
+            # QualifiedName, StatusCode.
+            # Missing according to ua.VariantType (also missing in string_to_val):
+            # DataValue, Variant, DiagnosticInfo.
             self.logger.warning("Parsing value of type '%s' not implemented", ntag)
 
     def _get_text(self, el):
         txtlist = [txt.strip() for txt in el.itertext()]
         return "".join(txtlist)
 
-    def _parse_list(self, el):
-        value = []
-        for val_el in el:
-            ntag = self._retag.match(val_el.tag).groups()[1]
-            if ntag.startswith("ListOf"):
-                val = self._parse_list(val_el)
-            else:
-                val = ua_type_to_python(val_el.text, ntag)
-            value.append(val)
-        return value
-
     def _parse_list_of_localized_text(self, el):
+        # FIXME Why not calling parse_body as for LocalizedText without list?
         value = []
         for localized_text in el:
             ntag = self._retag.match(localized_text.tag).groups()[1]

setup.py

@@ -3,9 +3,9 @@ from setuptools import setup, find_packages
 import sys
 
 if sys.version_info[0] < 3:
-    install_requires = ["python-dateutil", "enum34", "trollius", "futures"]
+    install_requires = ["python-dateutil", "enum34", "trollius", "futures", "pytz"]
 else:
-    install_requires = ["python-dateutil"]
+    install_requires = ["python-dateutil", "pytz"]
 
 setup(name="freeopcua",
       version="0.90.0",

tests/tests_xml.py

 import uuid
+import datetime, pytz
 import logging
 
 from opcua import ua
@@ -218,6 +219,30 @@ class XmlTests(object):
         o = self.opc.nodes.objects.add_variable(2, "xmlguid", uuid.uuid4())
         self._test_xml_var_type(o, "guid")
 
+    def test_xml_guid_array(self):
+        o = self.opc.nodes.objects.add_variable(2, "xmlguid", [uuid.uuid4(), uuid.uuid4()])
+        self._test_xml_var_type(o, "guid_array")
+
+    def test_xml_datetime(self):
+        o = self.opc.nodes.objects.add_variable(3, "myxmlvar-dt", datetime.datetime.now(), ua.VariantType.DateTime)
+        self._test_xml_var_type(o, "datetime")
+
+    def test_xml_datetime_array(self):
+        o = self.opc.nodes.objects.add_variable(3, "myxmlvar-array", [
+            datetime.datetime.now(),
+            datetime.datetime.utcnow(),
+            datetime.datetime.now(pytz.timezone("Asia/Tokyo"))
+        ], ua.VariantType.DateTime)
+        self._test_xml_var_type(o, "datetime_array")
+
+    #def test_xml_qualifiedname(self):
+    #    o = self.opc.nodes.objects.add_variable(2, "xmlltext", ua.QualifiedName("mytext", 5))
+    #    self._test_xml_var_type(o, "qualified_name")
+
+    #def test_xml_qualifiedname_array(self):
+    #    o = self.opc.nodes.objects.add_variable(2, "xmlltext_array", [ua.QualifiedName("erert", 5), ua.QualifiedName("erert33", 6)])
+    #    self._test_xml_var_type(o, "qualified_name_array")
+
     def test_xml_localizedtext(self):
         o = self.opc.nodes.objects.add_variable(2, "xmlltext", ua.LocalizedText("mytext"))
         self._test_xml_var_type(o, "localized_text")
@@ -246,6 +271,31 @@ class XmlTests(object):
         self.assertEqual(arg.Description, arg2.Description)
         self.assertEqual(arg.DataType, arg2.DataType)
 
+    def test_xml_ext_obj_array(self):
+        arg = ua.Argument()
+        arg.DataType = ua.NodeId(ua.ObjectIds.Float)
+        arg.Description = ua.LocalizedText(b"Nice description")
+        arg.ArrayDimensions = [1, 2, 3]
+        arg.Name = "MyArg"
+
+        arg2 = ua.Argument()
+        arg2.DataType = ua.NodeId(ua.ObjectIds.Int32)
+        arg2.Description = ua.LocalizedText(b"Nice description2")
+        arg2.ArrayDimensions = [4, 5, 6]
+        arg2.Name = "MyArg2"
+
+        args = [arg, arg2]
+
+        node = self.opc.nodes.objects.add_variable(2, "xmlexportobj2", args)
+        node2 = self._test_xml_var_type(node, "ext_obj_array", test_equality=False)
+        readArgs = node2.get_value()
+
+        for i,arg in enumerate(readArgs):
+            self.assertEqual(args[i].Name, readArgs[i].Name)
+            self.assertEqual(args[i].ArrayDimensions, readArgs[i].ArrayDimensions)
+            self.assertEqual(args[i].Description, readArgs[i].Description)
+            self.assertEqual(args[i].DataType, readArgs[i].DataType)
+
     def test_xml_enum(self):
         o = self.opc.nodes.objects.add_variable(2, "xmlenum", 0, varianttype=ua.VariantType.Int32, datatype=ua.ObjectIds.ApplicationType)
         self._test_xml_var_type(o, "enum")