Harding xml import (#1259)

* add enum/struct import for xml

* remove warning for doc elements

* respect datatype in insert order

directly import structs/enums

* fix unittest

the new import need all variables a hand . So register the the additional types before

* fix case in path

* prevent regreistration of class

* fix test because of changed import order

* Add nodeset via git submodule

* handle alias

* use submodule

* Add "Opc.Ua.PlasticsRubber.GeneralTypes.NodeSet2.xml

* prevent endlessloop in sort nodes

* Add dependabot

* correct merge error

.github/workflows/python-package.yml

@@ -19,6 +19,8 @@ jobs:
 
     steps:
     - uses: actions/checkout@v2
+      with:
+        submodules: true
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v2
       with:

.gitmodules

+[submodule "nodeset"]
+	path = nodeset
+	url = https://github.com/OPCFoundation/UA-Nodeset.git

asyncua/common/structures104.py

@@ -381,6 +381,20 @@ async def load_custom_struct(node: Node) -> Any:
     return env[name]
 
 
+async def load_custom_struct_xml_import(node_id: ua.NodeId, attrs: ua.DataTypeAttributes):
+    """
+    This function is used to load custom structs from xmlimporter
+    """
+    name = attrs.DisplayName.Text
+    if hasattr(ua, name):
+        return getattr(ua, name)
+    sdef = attrs.DataTypeDefinition
+    env = await _generate_object(name, sdef, data_type=node_id)
+    struct = env[name]
+    ua.register_extension_object(name, sdef.DefaultEncodingId, struct, node_id)
+    return env[name]
+
+
 async def _recursive_parse_basedatatypes(server, base_node, parent_datatype, new_alias) -> Any:
 
     for desc in await base_node.get_children_descriptions(refs=ua.ObjectIds.HasSubtype):
@@ -394,6 +408,20 @@ async def _recursive_parse_basedatatypes(server, base_node, parent_datatype, new
         await _recursive_parse_basedatatypes(server, server.get_node(desc.NodeId), name, new_alias)
 
 
+async def load_basetype_alias_xml_import(server, name, nodeid, parent_datatype_nid):
+    '''
+    Insert alias for a datatype used for xml import
+    '''
+    if hasattr(ua, name):
+        return getattr(ua, name)
+    parent = server.get_node(parent_datatype_nid)
+    bname = await parent.read_browse_name()
+    parent_datatype = clean_name(bname.Name)
+    env = make_basetype_code(name, parent_datatype)
+    ua.register_basetype(name, nodeid, env[name])
+    return env[name]
+
+
 def make_basetype_code(name, parent_datatype):
     """
     alias basetypes
@@ -521,3 +549,15 @@ async def load_enums(server: Union["Server", "Client"], base_node: Node = None,
         ua.register_enum(name, desc.NodeId, env[name])
         new_enums[name] = env[name]
     return new_enums
+
+
+async def load_enum_xml_import(node_id: ua.NodeId, attrs: ua.DataTypeAttributes, option_set: bool):
+    """
+    This function is used to load enums from xmlimporter
+    """
+    name = attrs.DisplayName.Text
+    if hasattr(ua, name):
+        return getattr(ua, name)
+    env = await _generate_object(name, attrs.DataTypeDefinition, enum=True, option_set=option_set)
+    ua.register_enum(name, node_id, env[name])
+    return env[name]

asyncua/common/xmlimporter.py

@@ -6,7 +6,7 @@ import logging
 import uuid
 from typing import Union, Dict, List, Tuple
 from dataclasses import fields, is_dataclass
-
+from asyncua.common.structures104 import load_custom_struct_xml_import, load_enum_xml_import, load_basetype_alias_xml_import
 from asyncua import ua
 from asyncua.ua.uatypes import type_is_union, types_from_union, type_is_list, type_from_list
 from .xmlparser import XMLParser, ua_type_to_python
@@ -131,7 +131,7 @@ class XmlImporter:
         dnodes = self.parser.get_node_datas()
         dnodes = self.make_objects(dnodes)
         self._add_missing_parents(dnodes)
-        nodes_parsed = self._sort_nodes_by_parentid(dnodes)
+        nodes_parsed = self._sort_nodes(dnodes)
         nodes = []
         for nodedata in nodes_parsed:  # self.parser:
             try:
@@ -346,6 +346,8 @@ class XmlImporter:
         node.NodeAttributes = attrs
         res = await self._get_server().add_nodes([node])
         await self._add_refs(obj)
+        # do not verify these nodes because some nodesets contain invalid elements
+        if obj.displayname != 'Default Binary' and obj.displayname != 'Default XML' and obj.displayname != 'Default Json':
             res[0].StatusCode.check()
         return res[0].AddedNodeId
 
@@ -368,6 +370,7 @@ class XmlImporter:
         if obj.desc:
             attrs.Description = ua.LocalizedText(obj.desc)
         attrs.DisplayName = ua.LocalizedText(obj.displayname)
+        if obj.datatype is not None:
             attrs.DataType = obj.datatype
         if obj.value is not None:
             attrs.Value = await self._add_variable_value(obj, )
@@ -414,6 +417,8 @@ class XmlImporter:
                     val,
                 )
             for attname, v in val:
+                if attname != 'EncodingMask':
+                    # Skip Encoding Mask used for optional types
                     atttype = self._get_val_type(extclass, attname)
                     self._set_attr(atttype, args, attname, v)
         return extclass(**args)
@@ -457,6 +462,8 @@ class XmlImporter:
         elif is_dataclass(atttype):
             subargs = {}
             for attname2, v2 in val:
+                if attname2 != 'EncodingMask':
+                    # Skip Encoding Mask used for optional types
                     sub_atttype = self._get_val_type(atttype, attname2)
                     self._set_attr(sub_atttype, subargs, attname2, v2)
             if "Encoding" in subargs:
@@ -508,9 +515,10 @@ class XmlImporter:
         if obj.desc:
             attrs.Description = ua.LocalizedText(obj.desc)
         attrs.DisplayName = ua.LocalizedText(obj.displayname)
+        if obj.datatype is not None:
             attrs.DataType = obj.datatype
-        if obj.value and len(obj.value) == 1:
-            attrs.Value = obj.value[0]
+        if obj.value:
+            attrs.Value = await self._add_variable_value(obj, )
         if obj.rank:
             attrs.ValueRank = obj.rank
         if obj.abstract:
@@ -566,6 +574,10 @@ class XmlImporter:
         return res[0].AddedNodeId
 
     async def add_datatype(self, obj, no_namespace_migration=False):
+        is_enum = False
+        is_struct = False
+        is_option_set = False
+        is_alias = False
         node = self._get_add_node_item(obj, no_namespace_migration)
         attrs = ua.DataTypeAttributes()
         if obj.desc:
@@ -576,20 +588,25 @@ class XmlImporter:
         else:
             attrs.IsAbstract = False
         if not obj.definitions:
-            pass
+            is_alias = True
         else:
             if obj.parent == self.session.nodes.enum_data_type.nodeid:
                 attrs.DataTypeDefinition = self._get_edef(obj)
+                is_enum = True
             elif obj.parent == self.session.nodes.base_structure_type.nodeid:
                 attrs.DataTypeDefinition = self._get_sdef(obj)
+                is_struct = True
             else:
                 parent_node = self.session.get_node(obj.parent)
                 path = await parent_node.get_path()
                 if self.session.nodes.option_set_type in path:
                     # nodes below option_set_type are enums, not structs
                     attrs.DataTypeDefinition = self._get_edef(obj)
+                    is_enum = True
+                    is_option_set = True
                 elif self.session.nodes.base_structure_type in path:
                     attrs.DataTypeDefinition = self._get_sdef(obj)
+                    is_struct = True
                 else:
                     _logger.warning(
                         "%s has datatypedefinition and path %s"
@@ -601,6 +618,13 @@ class XmlImporter:
         res = await self._get_server().add_nodes([node])
         res[0].StatusCode.check()
         await self._add_refs(obj)
+        if is_struct:
+            await load_custom_struct_xml_import(node.RequestedNewNodeId, attrs)
+        if is_enum:
+            await load_enum_xml_import(node.RequestedNewNodeId, attrs, is_option_set)
+        if is_alias:
+            if node.ParentNodeId != ua.NodeId(ua.ObjectIds.Structure):
+                await load_basetype_alias_xml_import(self.session, node.BrowseName.Name, node.RequestedNewNodeId, node.ParentNodeId)
         return res[0].AddedNodeId
 
     async def _add_refs(self, obj):
@@ -667,29 +691,43 @@ class XmlImporter:
             sdef.StructureType = ua.StructureType.Structure
         return sdef
 
-    def _sort_nodes_by_parentid(self, ndatas):
+    def _sort_nodes(self, ndatas):
         """
         Sort the list of nodes according their parent node in order to respect
-        the dependency between nodes.
+        the dependency between nodes. Also respect the datatypes used in variables
 
         :param nodes: list of NodeDataObjects
         :returns: list of sorted nodes
         """
 
         sorted_ndatas = []
-        sorted_nodes_ids = set()
-        all_node_ids = set(data.nodeid for data in ndatas)
-        while len(sorted_nodes_ids) < len(ndatas):
-            for ndata in ndatas:
-                if ndata.nodeid in sorted_nodes_ids:
-                    continue
-                elif (ndata.parent is None or ndata.parent not in all_node_ids):
+        all_nodes = {data.nodeid: data for data in ndatas}
+        sorted_nodes = {}
+
+        def add_to_sorted(nid, ndata):
+            # check if a node is a datatype, if fields are imported already
+            for field in ndata.definitions:
+                if field.datatype != nid and field.datatype in all_nodes:
+                    if field.datatype not in sorted_nodes:
+                        return
             sorted_ndatas.append(ndata)
-                    sorted_nodes_ids.add(ndata.nodeid)
+            sorted_nodes[nid] = ndata
+        last_len = 0
+        while len(sorted_nodes) < len(ndatas):
+            for nid, ndata in all_nodes.items():
+                if nid in sorted_nodes:
+                    continue
+                if ndata.datatype is not None and ndata.datatype in all_nodes:
+                    if ndata.datatype not in sorted_nodes:
+                        continue
+                if (ndata.parent is None or ndata.parent not in all_nodes):
+                    add_to_sorted(nid, ndata)
                 else:
                     # Check if the nodes parent is already in the list of
                     # inserted nodes
-                    if ndata.parent in sorted_nodes_ids:
-                        sorted_ndatas.append(ndata)
-                        sorted_nodes_ids.add(ndata.nodeid)
+                    if ndata.parent in sorted_nodes:
+                        add_to_sorted(nid, ndata)
+            if last_len == len(sorted_nodes):
+                # When no change is found we are in a endlessloop
+                raise ValueError('Ordering of nodes is not possible')
         return sorted_ndatas

asyncua/common/xmlparser.py

@@ -245,6 +245,9 @@ class XMLParser:
             for field in el:
                 field = self._parse_field(field)
                 obj.definitions.append(field)
+        elif tag == "Documentation" or tag == "Category":
+            # Only for documentation
+            pass
         else:
             self.logger.info("Not implemented tag: %s", el)
 
@@ -400,7 +403,6 @@ class XMLParser:
         if nsval is not None:
             ns = string_to_val(nsval.text, ua.VariantType.UInt16)
         v = ua.QualifiedName(name, ns)
-        self.logger.warning("qn: %s", v)
         return v
 
     def _parse_refs(self, el, obj):

nodeset @ 526c5e59

+Subproject commit 526c5e59e7ba9e54f9dc7848ce2baa249395d3ef

tests/test_xml.py

@@ -26,6 +26,14 @@ CUSTOM_NODES_NS_XML_PATH3 = BASE_DIR / "custom_nodesns_4.xml"
 CUSTOM_NS_META_ADD_XML_PATH = BASE_DIR / "custom_ns_meta_add.xml"
 CUSTOM_REQ_XML_PASS_PATH = BASE_DIR / "test_requirement_pass.xml"
 CUSTOM_REQ_XML_FAIL_PATH = BASE_DIR / "test_requirement_fail.xml"
+NODESET_DIR = BASE_DIR.parents[0] / "nodeset"
+NODESET_DI = NODESET_DIR / "DI" / "Opc.Ua.Di.NodeSet2.xml"
+NODESET_SCALES = NODESET_DIR / "Scales" / "Opc.Ua.Scales.NodeSet2.xml"
+NODESET_PACK = NODESET_DIR / "PackML" / "Opc.Ua.PackML.NodeSet2.xml"
+NODESET_TMC = NODESET_DIR / "TMC" / "Opc.Ua.TMC.NodeSet2.xml"
+NODESET_IA = NODESET_DIR / "IA" / "Opc.Ua.IA.NodeSet2.xml"
+NODESET_VISION = NODESET_DIR / "MachineVision" / "Opc.Ua.MachineVision.NodeSet2.xml"
+NODESET_PLASTIC_RUBBER = NODESET_DIR / "PlasticsRubber" / "GeneralTypes" / "1.03" / "Opc.Ua.PlasticsRubber.GeneralTypes.NodeSet2.xml"
 
 
 @uamethod
@@ -56,6 +64,30 @@ async def test_xml_import(opc):
         await opc.opc.delete_nodes([opc.opc.get_node(nodeid)])
 
 
+async def test_xml_import_companion_specifications(opc):
+    # if not already shift the new namespaces
+    await opc.server.register_namespace("http://placeholder.toincrease.nsindex")
+    # Test against some companion specifications
+    nodes = await opc.opc.import_xml(NODESET_DI)
+    scales_nodes = await opc.opc.import_xml(NODESET_SCALES)
+    for nodeid in scales_nodes:
+        await opc.opc.delete_nodes([opc.opc.get_node(nodeid)])
+    if opc.opc == opc.server:
+        # Only load in server testcase takes to long if the networklayer is involved
+        pack_nodes = await opc.opc.import_xml(NODESET_PACK)
+
+        # pack_nodes += await opc.opc.import_xml(NODESET_TMC) # @TODO we have a nameming colison
+        for nodeid in pack_nodes:
+            await opc.opc.delete_nodes([opc.opc.get_node(nodeid)])
+        ruber_nodes = await opc.opc.import_xml(NODESET_PLASTIC_RUBBER)
+        for nodeid in ruber_nodes:
+            await opc.opc.delete_nodes([opc.opc.get_node(nodeid)])
+        nodes += await opc.opc.import_xml(NODESET_IA)
+        nodes += await opc.opc.import_xml(NODESET_VISION)
+    for nodeid in nodes:
+        await opc.opc.delete_nodes([opc.opc.get_node(nodeid)])
+
+
 async def test_xml_import_additional_ns(opc):
     # if not already shift the new namespaces
     await opc.server.register_namespace("http://placeholder.toincrease.nsindex")
@@ -596,8 +628,8 @@ async def test_xml_struct_in_struct_with_value(opc, tmpdir):
     await opc.opc.export_xml([outer_struct, inner_struct, valnode], tmp_path, export_values=True)
     await opc.opc.delete_nodes([outer_struct, inner_struct, valnode])
     new_nodes = await opc.opc.import_xml(tmp_path)
-    imported_outer_struct = opc.opc.get_node(new_nodes[0])
-    imported_inner_struct = opc.opc.get_node(new_nodes[1])
+    imported_outer_struct = opc.opc.get_node(new_nodes[1])
+    imported_inner_struct = opc.opc.get_node(new_nodes[0])
     imported_valnode = opc.opc.get_node(new_nodes[2])
     assert outer_struct == imported_outer_struct
     assert inner_struct == imported_inner_struct

tests/util_enum_struct.py

@@ -19,6 +19,7 @@ import asyncua.ua
 setattr(asyncua.ua, 'ExampleEnum', ExampleEnum)
 
 
+
 @dataclass
 class ExampleStruct:
     IntVal1: uatypes.Int16 = 0
@@ -27,9 +28,10 @@ class ExampleStruct:
 
 async def add_server_custom_enum_struct(server: Server):
     # import some nodes from xml
-    await server.import_xml(TEST_DIR / "enum_struct_test_nodes.xml")
-    ns = await server.get_namespace_index('http://yourorganisation.org/struct_enum_example/')
+    ns = await server.register_namespace('http://yourorganisation.org/struct_enum_example/')
+    uatypes.register_enum('ExampleEnum', ua.NodeId(3002, ns), ExampleEnum)
     uatypes.register_extension_object('ExampleStruct', ua.NodeId(5001, ns), ExampleStruct)
+    await server.import_xml(TEST_DIR / "enum_struct_test_nodes.xml"),
     val = ua.ExampleStruct()
     val.IntVal1 = 242
     val.EnumVal = ua.ExampleEnum.EnumVal2