docs: merge settings docs in a separate step

The script "libnm/generate-setting-docs.py" generates property info based
on GObject introspection data.

Optionally, when creating the manual for D-Bus documentation, it would accept
an argument "--override" to merge the generated information with the information
from an XML generated by "libnm/generate-plugin-docs.xml". Change this.
Instead, let "libnm/generate-setting-docs.py" just do one thing: generate
the XML based on GObject introspection data. Then, a second script
"libnm/generate-docs-nm-settings-docs-merge.py" can merge the XMLs.

Note that currently the manual for "nm-settings-keyfile" only contains
information about properties that are explicitly mentioned for keyfile.
It think that is not right. In NetworkManager there are multiple "aspects"
about connection profiles: D-Bus, libnm, nmcli, keyfile and ifcfg-rh.
When we generate a manual page for any of these aspects, we should always
detail all properties. At least for nmcli and D-Bus. That means, we will
do the merging multiple times. Hence, keep the steps for parsing GObject
introspection data and the merging separate.

Also, "generate-setting-docs.py" and "generate-plugin-docs.pl" should
generate the same XML scheme, so that merge doesn't need special hacks.
That is currently not the case, for example, the override XML contains a
"format" attribute, while the other one contains a "type". Merging these
is a special hack. This should be unified.

Makefile.am

@@ -1579,16 +1579,8 @@ libnm/nm-property-docs.xml: libnm/generate-setting-docs.py $(libnm_docs_sources)
 		--gir $(builddir)/libnm/NM-1.0.gir \
 		--output $@
 
-libnm/nm-settings-docs.xml: libnm/generate-setting-docs.py libnm/nm-settings-docs-overrides.xml $(libnm_docs_sources) | libnm/NM-1.0.gir libnm/NM-1.0.typelib libnm/libnm.la
+libnm/nm-settings-docs.xml: libnm/nm-settings-docs-overrides.xml libnm/nm-property-docs.xml libnm/generate-docs-nm-settings-docs-merge.py
-	$(AM_V_GEN) \
+	$(AM_V_GEN) "$(PYTHON)" $(srcdir)/libnm/generate-docs-nm-settings-docs-merge.py $@ $(wordlist 1,2,$^)
-	export GI_TYPELIB_PATH=$(abs_builddir)/libnm$${GI_TYPELIB_PATH:+:$$GI_TYPELIB_PATH}; \
-	export LD_LIBRARY_PATH=$(abs_builddir)/libnm/.libs$${LD_LIBRARY_PATH:+:$$LD_LIBRARY_PATH}; \
-	$(call set_sanitizer_env,$(abs_builddir)/libnm/.libs/libnm.so); \
-	"$(PYTHON)" \
-	$(srcdir)/libnm/generate-setting-docs.py \
-		--gir $(builddir)/libnm/NM-1.0.gir \
-		--overrides $(word 2,$^) \
-		--output $@
 
 libnm/nm-settings-keyfile-docs.xml: libnm/generate-plugin-docs.pl $(libnm_docs_sources)
 	$(AM_V_GEN) $(srcdir)/libnm/generate-plugin-docs.pl keyfile $@ $(filter-out $<,$^)
@@ -1600,8 +1592,9 @@ EXTRA_DIST += $(libnm_noinst_data)
 endif
 
 EXTRA_DIST += \
-	libnm/generate-setting-docs.py \
 	libnm/generate-plugin-docs.pl \
+	libnm/generate-docs-nm-settings-docs-merge.py \
+	libnm/generate-setting-docs.py \
 	libnm/nm-enum-types.c.template \
 	libnm/nm-enum-types.h.template \
 	libnm/meson.build

libnm/generate-docs-nm-settings-docs-merge.py

@@ -0,0 +1,146 @@
+#!/usr/bin/env python
+# SPDX-License-Identifier: LGPL-2.1+
+
+from __future__ import print_function
+
+import os
+import sys
+import collections
+import xml.etree.ElementTree as ET
+
+###############################################################################
+
+_setting_name_order = [
+    "connection",
+    "6lowpan",
+    "802-1x",
+    "adsl",
+    "bluetooth",
+    "bond",
+    "bridge",
+    "bridge-port",
+    "cdma",
+    "dcb",
+    "dummy",
+    "ethtool",
+    "generic",
+    "gsm",
+    "infiniband",
+    "ipv4",
+    "ipv6",
+    "ip-tunnel",
+    "macsec",
+    "macvlan",
+    "match",
+    "802-11-olpc-mesh",
+    "ovs-bridge",
+    "ovs-dpdk",
+    "ovs-interface",
+    "ovs-patch",
+    "ovs-port",
+    "ppp",
+    "pppoe",
+    "proxy",
+    "serial",
+    "sriov",
+    "tc",
+    "team",
+    "team-port",
+    "tun",
+    "user",
+    "vlan",
+    "vpn",
+    "vrf",
+    "vxlan",
+    "wifi-p2p",
+    "wimax",
+    "802-3-ethernet",
+    "wireguard",
+    "802-11-wireless",
+    "802-11-wireless-security",
+    "wpan",
+]
+
+def _setting_name_order_idx(name):
+    try:
+        return _setting_name_order.index(name)
+    except ValueError:
+        return len(_setting_name_order)
+
+def key_fcn_setting_name(n1):
+    return (_setting_name_order_idx(n1), n1)
+
+def iter_keys_of_dicts(dicts, key = None):
+    keys = set([k for d in dicts for k in d.keys()])
+    return sorted(keys, key = key)
+
+def node_to_dict(node, tag, key_attr):
+    dictionary = collections.OrderedDict()
+    if node is not None:
+        for n in node.iter(tag):
+            k = n.get(key_attr)
+            assert(k is not None)
+            dictionary[k] = n
+    return dictionary
+
+def node_get_attr(nodes, name):
+    for n in nodes:
+        if n is None:
+            continue
+        x = n.get(name, None)
+        if x:
+            return x
+    return None
+
+def node_set_attr(dst_node, name, nodes):
+    x = node_get_attr(nodes, name)
+    if x:
+        dst_node.set(name, x)
+
+###############################################################################
+
+if len(sys.argv) < 3:
+    print("%s [OUT_FILE] [SETTING_XML [...]]" % (sys.argv[0]))
+    exit(1)
+
+output_xml_file = sys.argv[1]
+
+xml_roots = list([ET.parse(f).getroot() for f in sys.argv[2:]])
+
+assert(all([root.tag.startswith('nm-') for root in xml_roots]))
+
+settings_roots = list([node_to_dict(root, 'setting', 'name') for root in xml_roots])
+
+root_node = ET.Element('nm-setting-docs')
+
+for setting_name in iter_keys_of_dicts(settings_roots, key_fcn_setting_name):
+
+    settings = list([d.get(setting_name) for d in settings_roots])
+
+    setting_node = ET.SubElement(root_node, 'setting')
+
+    setting_node.set('name', setting_name)
+
+    node_set_attr(setting_node, 'description', settings)
+    node_set_attr(setting_node, 'name_upper', settings)
+
+    properties = list([node_to_dict(s, 'property', 'name') for s in settings])
+
+    for property_name in iter_keys_of_dicts(properties):
+
+        properties_attrs = list([p.get(property_name) for p in properties])
+
+        property_node = ET.SubElement(setting_node, 'property')
+        property_node.set('name', property_name)
+        property_node.set('name_upper', property_name.upper().replace('-', '_'))
+
+        x = node_get_attr(properties_attrs, 'format')
+        if x:
+            property_node.set('type', x)
+        else:
+            node_set_attr(property_node, 'type', properties_attrs)
+
+        node_set_attr(property_node, 'default', properties_attrs)
+        node_set_attr(property_node, 'description', properties_attrs)
+
+ET.ElementTree(root_node).write(output_xml_file)

libnm/generate-setting-docs.py

@@ -172,7 +172,6 @@ def usage():
 parser = argparse.ArgumentParser()
 parser.add_argument('-l', '--lib-path', metavar='PATH', action='append', help='path to scan for shared libraries')
 parser.add_argument('-g', '--gir', metavar='FILE', help='NM-1.0.gir file')
-parser.add_argument('-x', '--overrides', metavar='FILE', help='documentation overrides file')
 parser.add_argument('-o', '--output', metavar='FILE', help='output file')
 
 args = parser.parse_args()
@@ -195,9 +194,6 @@ settings = sorted(settings, key=settings_sort_key)
 
 init_constants(girxml, settings)
 
-if args.overrides is not None:
-    overrides = ET.parse(args.overrides).getroot()
-
 outfile.write("""<?xml version=\"1.0\"?>
 <!DOCTYPE nm-setting-docs [
 <!ENTITY quot "&#34;">
@@ -218,48 +214,31 @@ for settingxml in settings:
     outfile.write("  <setting name=\"%s\" description=\"%s\" name_upper=\"%s\" >\n" % (setting.props.name, class_desc, get_setting_name_define (settingxml)))
 
     setting_properties = { prop.name: prop for prop in GObject.list_properties(setting) if prop.name != 'name' }
-    if args.overrides is None:
-        setting_overrides = {}
-    else:
-        setting_overrides = { override.attrib['name']: override for override in overrides.findall('./setting[@name="%s"]/property' % setting.props.name) }
 
-    properties = sorted(set.union(set(setting_properties.keys()), set(setting_overrides.keys())))
+    for prop in sorted(setting_properties):
+        pspec = setting_properties[prop]
 
-    for prop in properties:
+        propxml = settingxml.find('./gi:property[@name="%s"]' % pspec.name, ns_map)
-        value_type = None
+        if propxml is None:
-        value_desc = None
+            propxml = basexml.find('./gi:property[@name="%s"]' % pspec.name, ns_map)
-        default_value = None
+        if propxml is None:
+            propxml = ipxml.find('./gi:property[@name="%s"]' % pspec.name, ns_map)
 
-        if prop in setting_properties:
+        value_type = get_prop_type(setting, pspec)
-            pspec = setting_properties[prop]
+        value_desc = get_docs(propxml)
-            propxml = settingxml.find('./gi:property[@name="%s"]' % pspec.name, ns_map)
+        default_value = get_default_value(setting, pspec, propxml)
-            if propxml is None:
-                propxml = basexml.find('./gi:property[@name="%s"]' % pspec.name, ns_map)
-            if propxml is None:
-                propxml = ipxml.find('./gi:property[@name="%s"]' % pspec.name, ns_map)
-
-            value_type = get_prop_type(setting, pspec)
-            value_desc = get_docs(propxml)
-            default_value = get_default_value(setting, pspec, propxml)
-
-        if prop in setting_overrides:
-            override = setting_overrides[prop]
-            if override.attrib['format'] != '':
-                value_type = override.attrib['format']
-            if override.attrib['description'] != '':
-                value_desc = override.attrib['description']
 
         prop_upper = prop.upper().replace('-', '_')
 
         if value_desc is None:
             raise Exception("%s.%s needs a documentation description" % (setting.props.name, prop))
 
+        default_value_as_xml = ''
         if default_value is not None:
-            outfile.write("    <property name=\"%s\" name_upper=\"%s\" type=\"%s\" default=\"%s\" description=\"%s\" />\n" %
+            default_value_as_xml = (' default=\"%s\"' % (escape(default_value)))
-                          (prop, prop_upper, value_type, escape(default_value), escape(value_desc)))
+
-        else:
+        outfile.write("    <property name=\"%s\" name_upper=\"%s\" type=\"%s\"%s description=\"%s\" />\n" %
-            outfile.write("    <property name=\"%s\" name_upper=\"%s\" type=\"%s\" description=\"%s\" />\n" %
+                      (prop, prop_upper, value_type, default_value_as_xml, escape(value_desc)))
-                          (prop, prop_upper, value_type, escape(value_desc)))
 
     outfile.write("  </setting>\n")
 

libnm/meson.build

@@ -240,8 +240,6 @@ if enable_introspection
     )
   endif
 
-  generate_setting_docs = join_paths(meson.current_source_dir(), 'generate-setting-docs.py')
-
   gi_typelib_path = run_command('printenv', 'GI_TYPELIB_PATH').stdout()
   if gi_typelib_path != ''
     gi_typelib_path = ':' + gi_typelib_path
@@ -265,16 +263,29 @@ if enable_introspection
     name,
     input: libnm_gir[0],
     output: name,
-    command: [generate_setting_docs_env, python.path(), generate_setting_docs, '--lib-path', meson.current_build_dir(), '--gir', '@INPUT@', '--output', '@OUTPUT@'],
+    command: [
+      generate_setting_docs_env,
+      python.path(),
+      join_paths(meson.current_source_dir(), 'generate-setting-docs.py'),
+      '--lib-path', meson.current_build_dir(),
+      '--gir', '@INPUT@',
+      '--output', '@OUTPUT@'
+    ],
     depends: libnm_gir,
   )
 
   name = 'nm-settings-docs.xml'
   nm_settings_docs = custom_target(
     name,
-    input: [libnm_gir[0], nm_settings_docs_overrides],
+    input: [nm_property_docs, nm_settings_docs_overrides],
     output: name,
-    command: [generate_setting_docs_env, python.path(), generate_setting_docs, '--lib-path', meson.current_build_dir(), '--gir', '@INPUT0@', '--overrides', '@INPUT1@', '--output', '@OUTPUT@'],
+    command: [
+      python.path(),
+      join_paths(meson.current_source_dir(), 'generate-docs-nm-settings-docs-merge.py'),
+      '@OUTPUT@',
+      nm_settings_docs_overrides,
+      nm_property_docs,
+    ],
     depends: libnm_gir,
   )
 endif