Implement netplan get/set CLI and DBus API (#163)

Implements the netplan get [nested.key] / set nested.key=value CLI and io.netplan.Netplan.Get() / Set(String configDelta, String originHint) DBus API.

It allows to get/set values from/to the merged netplan configuration as scalar, sequence or mappings.

Validation of settings, added via netplan set ..., is done by calling into the libnetplan parser using Python ctypes. Whereas the netplan get [...] command makes use of netplan's Python parser (config_manager). This is to keep it simple for now, unifying the parsers will be for another time...

Examples
netplan get
netplan get ethernets.eth0.dhcp4
netplan set ethernets.eth0.dhcp4=true --origin-hint=99_snapd --root-dir=/tmp
netplan set "network.ethernets.eth0={addresses: [1.2.3.4/24], dhcp4: true}"

busctl call io.netplan.Netplan /io/netplan/Netplan io.netplan.Netplan Get
busctl call io.netplan.Netplan /io/netplan/Netplan io.netplan.Netplan Set ss "network.ethernets.eth3={addresses: [5.6.7.8/24], dhcp4: no}" ""

Commits:
* cli:set: initial skeleton for 'netplan set' command
* cli:set: allow to set sequences
* Initial draft for 'netplan get'
* cli:get:set: migrate tests to (quicker) nosetests3
* Makefile: fix missing glib symbols in libnetplan
* parse: avoid segfault, if GError is NULL
* parse: add netplan_clear_netdefs API
* cli:set: use libnetplan for settings validation
* cli:test: improve libnetplan testing
* tests: cleanup
* dbus: initial Get() api
* cli:set: allow to set mappings/subtrees
* dbus: initial Set() api
* dbus:set: allow optional arguments as 'a{ss}'
* dbus: optional 'root-dir' argument for Get()
* test:configmanager: fix warning with nosetests3
* configManager: add modems & tunnels to python parser
* cli:get: cleanup and allow escaped dots in keys/interface-ids
* cli:set: cleanup & allow escaped dots in keys/interface-ids
* configmanager: add openvswitch/version/renderer keys to python parser
* configmanager: allow to access the full, cleaned/striped YAML tree
* cli:get: print the full YAML tree, if no key is specified
* cli:set: cleanup strip_tree
* dbus: adopt Get() api, according to spec
* cli:set: allow to specify optional 'network.' prefix
* dbus: adopt Set() api, according to spec
* cli:set: catch empty origin-hint
* cli:get/set: cleanup
* cli:set: avoid unnecessary cleanup of tempfile
* cli:set: improve description/comment of merge()
* cli:set: improve origin-hint help text
* cli:set: use path variable everywhere, remove unused argument
* cli:get: improve prefix detection
* cli:set: improve delete (key=NULL) and prefix handling
* cli:set: catch invalid input exception

5 files changed, 273 insertions, 2 deletions

diff --git a/netplan/cli/commands/__init__.py b/netplan/cli/commands/__init__.py
index 07a2a59..0a5a229 100644
--- a/netplan/cli/commands/__init__.py
+++ b/netplan/cli/commands/__init__.py
@@ -21,6 +21,8 @@ from netplan.cli.commands.ip import NetplanIp
 from netplan.cli.commands.migrate import NetplanMigrate
 from netplan.cli.commands.try_command import NetplanTry
 from netplan.cli.commands.info import NetplanInfo
+from netplan.cli.commands.set import NetplanSet
+from netplan.cli.commands.get import NetplanGet
 
 __all__ = [
     'NetplanApply',
@@ -29,4 +31,6 @@ __all__ = [
     'NetplanMigrate',
     'NetplanTry',
     'NetplanInfo',
+    'NetplanSet',
+    'NetplanGet',
 ]
diff --git a/netplan/cli/commands/get.py b/netplan/cli/commands/get.py
new file mode 100644
index 0000000..17603aa
--- /dev/null
+++ b/netplan/cli/commands/get.py
@@ -0,0 +1,67 @@
+#!/usr/bin/python3
+#
+# Copyright (C) 2020 Canonical, Ltd.
+# Author: Lukas Märdian <lukas.maerdian@canonical.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 3.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+'''netplan get command line'''
+
+import yaml
+import re
+
+import netplan.cli.utils as utils
+from netplan.configmanager import ConfigManager
+
+
+class NetplanGet(utils.NetplanCommand):
+
+    def __init__(self):
+        super().__init__(command_id='get',
+                         description='Get a setting by specifying a nested key like "ethernets.eth0.addresses", or "all"',
+                         leaf=True)
+
+    def run(self):
+        self.parser.add_argument('key', type=str, nargs='?', default='all', help='The nested key in dotted format')
+        self.parser.add_argument('--root-dir', default='/',
+                                 help='Read configuration files from this root directory instead of /')
+
+        self.func = self.command_get
+
+        self.parse_args()
+        self.run_command()
+
+    def command_get(self):
+        config_manager = ConfigManager(prefix=self.root_dir)
+        config_manager.parse()
+        tree = config_manager.tree
+
+        if self.key != 'all':
+            # The 'network.' prefix is optional for netsted keys, its always assumed to be there
+            if not self.key.startswith('network.'):
+                self.key = 'network.' + self.key
+            # Split at '.' but not at '\.' via negative lookbehind expression
+            for k in re.split(r'(?<!\\)\.', self.key):
+                k = k.replace('\\.', '.')  # Unescape interface-ids, containing dots
+                if k in tree.keys():
+                    tree = tree[k]
+                    if not isinstance(tree, dict):
+                        break
+                else:
+                    tree = None
+                    break
+
+        out = yaml.dump(tree, default_flow_style=False)[:-1]  # Remove trailing '\n'
+        if not isinstance(tree, dict) and not isinstance(tree, list):
+            out = out[:-4]  # Remove yaml.dump's '\n...' on primitive values
+        print(out)
diff --git a/netplan/cli/commands/set.py b/netplan/cli/commands/set.py
new file mode 100644
index 0000000..173cb4c
--- /dev/null
+++ b/netplan/cli/commands/set.py
@@ -0,0 +1,123 @@
+#!/usr/bin/python3
+#
+# Copyright (C) 2020 Canonical, Ltd.
+# Author: Lukas Märdian <lukas.maerdian@canonical.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 3.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+'''netplan set command line'''
+
+import os
+import yaml
+import tempfile
+import re
+
+import netplan.cli.utils as utils
+from netplan.configmanager import ConfigManager
+
+
+class NetplanSet(utils.NetplanCommand):
+
+    def __init__(self):
+        super().__init__(command_id='set',
+                         description='Add new setting by specifying a dotted key=value pair like ethernets.eth0.dhcp4=true',
+                         leaf=True)
+
+    def run(self):
+        self.parser.add_argument('key_value', type=str,
+                                 help='The nested key=value pair in dotted format. Value can be NULL to delete a key.')
+        self.parser.add_argument('--origin-hint', type=str, default='90-netplan-set',
+                                 help='Can be used to help choose a name for the overwrite YAML file. \
+                                       A .yaml suffix will be appended automatically.')
+        self.parser.add_argument('--root-dir', default='/',
+                                 help='Overwrite configuration files in this root directory instead of /')
+
+        self.func = self.command_set
+
+        self.parse_args()
+        self.run_command()
+
+    def command_set(self):
+        if len(self.origin_hint) == 0:
+            raise Exception('Invalid/empty origin-hint')
+        split = self.key_value.split('=', 1)
+        if len(split) != 2:
+            raise Exception('Invalid value specified')
+        key, value = split
+        set_tree = self.parse_key(key, yaml.safe_load(value))
+        self.write_file(set_tree, self.origin_hint + '.yaml', self.root_dir)
+
+    def parse_key(self, key, value):
+        # The 'network.' prefix is optional for netsted keys, its always assumed to be there
+        if not key.startswith('network.'):
+            key = 'network.' + key
+        # Split at '.' but not at '\.' via negative lookbehind expression
+        split = re.split(r'(?<!\\)\.', key)
+        tree = {}
+        i = 1
+        t = tree
+        for part in split:
+            part = part.replace('\\.', '.')  # Unescape interface-ids, containing dots
+            val = {}
+            if i == len(split):
+                val = value
+            t = t.setdefault(part, val)
+            i += 1
+        return tree
+
+    def merge(self, a, b, path=None):
+        """
+        Merges tree/dict 'b' into tree/dict 'a'
+        """
+        if path is None:
+            path = []
+        for key in b:
+            if key in a:
+                if isinstance(a[key], dict) and isinstance(b[key], dict):
+                    self.merge(a[key], b[key], path + [str(key)])
+                elif b[key] is None:
+                    del a[key]
+                else:
+                    # Overwrite existing key with new key/value from 'set' command
+                    a[key] = b[key]
+            else:
+                a[key] = b[key]
+        return a
+
+    def write_file(self, set_tree, name, rootdir='/'):
+        tmproot = tempfile.TemporaryDirectory(prefix='netplan-set_')
+        path = os.path.join('etc', 'netplan')
+        os.makedirs(os.path.join(tmproot.name, path))
+
+        config = {'network': {}}
+        absp = os.path.join(rootdir, path, name)
+        if os.path.isfile(absp):
+            with open(absp, 'r') as f:
+                config = yaml.safe_load(f)
+
+        new_tree = self.merge(config, set_tree)
+        stripped = ConfigManager.strip_tree(new_tree)
+        if 'network' in stripped:
+            tmpp = os.path.join(tmproot.name, path, name)
+            with open(tmpp, 'w+') as f:
+                new_yaml = yaml.dump(stripped, indent=2, default_flow_style=False)
+                f.write(new_yaml)
+            # Validate the newly created file, by parsing it via libnetplan
+            utils.netplan_parse(tmpp)
+            # Valid, move it to final destination
+            os.replace(tmpp, absp)
+        elif os.path.isfile(absp):
+            # Clear file if the last/only key got removed
+            os.remove(absp)
+        else:
+            raise Exception('Invalid input: {}'.format(set_tree))
diff --git a/netplan/cli/utils.py b/netplan/cli/utils.py
index f076c25..27ad9ae 100644
--- a/netplan/cli/utils.py
+++ b/netplan/cli/utils.py
@@ -24,11 +24,34 @@ import argparse
 import subprocess
 import netifaces
 import re
+import ctypes
+import ctypes.util
 
 NM_SERVICE_NAME = 'NetworkManager.service'
 NM_SNAP_SERVICE_NAME = 'snap.network-manager.networkmanager.service'
 
 
+class _GError(ctypes.Structure):
+    _fields_ = [("domain", ctypes.c_uint32), ("code", ctypes.c_int), ("message", ctypes.c_char_p)]
+
+
+lib = ctypes.CDLL('libnetplan.so')
+lib.netplan_parse_yaml.argtypes = [ctypes.c_char_p, ctypes.POINTER(ctypes.POINTER(_GError))]
+
+
+def netplan_parse(path):
+    # Clear old NetplanNetDefinitions from libnetplan memory
+    lib.netplan_clear_netdefs()
+    err = ctypes.POINTER(_GError)()
+    ret = bool(lib.netplan_parse_yaml(path.encode(), ctypes.byref(err)))
+    if not ret:
+        raise Exception(err.contents.message.decode('utf-8'))
+    lib.netplan_finish_parse(ctypes.byref(err))
+    if err:
+        raise Exception(err.contents.message.decode('utf-8'))
+    return True
+
+
 def get_generator_path():
     return os.environ.get('NETPLAN_GENERATE_PATH', '/lib/netplan/generate')
 
diff --git a/netplan/configmanager.py b/netplan/configmanager.py
index ce050a2..8f451b7 100644
--- a/netplan/configmanager.py
+++ b/netplan/configmanager.py
@@ -46,9 +46,11 @@ class ConfigManager(object):
         interfaces = {}
         interfaces.update(self.ovs_ports)
         interfaces.update(self.ethernets)
+        interfaces.update(self.modems)
         interfaces.update(self.wifis)
         interfaces.update(self.bridges)
         interfaces.update(self.bonds)
+        interfaces.update(self.tunnels)
         interfaces.update(self.vlans)
         return interfaces
 
@@ -56,6 +58,7 @@ class ConfigManager(object):
     def physical_interfaces(self):
         interfaces = {}
         interfaces.update(self.ethernets)
+        interfaces.update(self.modems)
         interfaces.update(self.wifis)
         return interfaces
 
@@ -64,10 +67,18 @@ class ConfigManager(object):
         return self.network['ovs_ports']
 
     @property
+    def openvswitch(self):
+        return self.network['openvswitch']
+
+    @property
     def ethernets(self):
         return self.network['ethernets']
 
     @property
+    def modems(self):
+        return self.network['modems']
+
+    @property
     def wifis(self):
         return self.network['wifis']
 
@@ -80,9 +91,36 @@ class ConfigManager(object):
         return self.network['bonds']
 
     @property
+    def tunnels(self):
+        return self.network['tunnels']
+
+    @property
     def vlans(self):
         return self.network['vlans']
 
+    @property
+    def version(self):
+        return self.network['version']
+
+    @property
+    def renderer(self):
+        return self.network['renderer']
+
+    @property
+    def tree(self):
+        return self.strip_tree(self.config)
+
+    @staticmethod
+    def strip_tree(data):
+        '''clear empty branches'''
+        new_data = {}
+        for k, v in data.items():
+            if isinstance(v, dict):
+                v = ConfigManager.strip_tree(v)
+            if v not in (u'', None, {}):
+                new_data[k] = v
+        return new_data
+
     def parse(self, extra_config=[]):
         """
         Parse all our config files to return an object that describes the system's
@@ -107,11 +145,16 @@ class ConfigManager(object):
 
         self.config['network'] = {
             'ovs_ports': {},
+            'openvswitch': {},
             'ethernets': {},
+            'modems': {},
             'wifis': {},
             'bridges': {},
             'bonds': {},
-            'vlans': {}
+            'tunnels': {},
+            'vlans': {},
+            'version': None,
+            'renderer': None
         }
         for yaml_file in files:
             self._merge_yaml_config(yaml_file)
@@ -119,7 +162,7 @@ class ConfigManager(object):
         for yaml_file in extra_config:
             self.new_interfaces |= self._merge_yaml_config(yaml_file)
 
-        logging.debug("Merged config:\n{}".format(yaml.dump(self.config, default_flow_style=False)))
+        logging.debug("Merged config:\n{}".format(yaml.dump(self.tree, default_flow_style=False)))
 
     def add(self, config_dict):
         for config_file in config_dict:
@@ -230,9 +273,13 @@ class ConfigManager(object):
                     if 'openvswitch' in network:
                         new = self._merge_ovs_ports_config(self.ovs_ports, network.get('openvswitch'))
                         new_interfaces |= new
+                        self.network['openvswitch'] = network.get('openvswitch')
                     if 'ethernets' in network:
                         new = self._merge_interface_config(self.ethernets, network.get('ethernets'))
                         new_interfaces |= new
+                    if 'modems' in network:
+                        new = self._merge_interface_config(self.modems, network.get('modems'))
+                        new_interfaces |= new
                     if 'wifis' in network:
                         new = self._merge_interface_config(self.wifis, network.get('wifis'))
                         new_interfaces |= new
@@ -242,9 +289,16 @@ class ConfigManager(object):
                     if 'bonds' in network:
                         new = self._merge_interface_config(self.bonds, network.get('bonds'))
                         new_interfaces |= new
+                    if 'tunnels' in network:
+                        new = self._merge_interface_config(self.tunnels, network.get('tunnels'))
+                        new_interfaces |= new
                     if 'vlans' in network:
                         new = self._merge_interface_config(self.vlans, network.get('vlans'))
                         new_interfaces |= new
+                    if 'version' in network:
+                        self.network['version'] = network.get('version')
+                    if 'renderer' in network:
+                        self.network['renderer'] = network.get('renderer')
             return new_interfaces
         except (IOError, yaml.YAMLError):  # pragma: nocover (filesystem failures/invalid YAML)
             logging.error('Error while loading {}, aborting.'.format(yaml_file))