buildman: Add helper functions for updating .config files

At present the only straightforward way to write tests that need a
slightly different configuration is to create a new board with its own
configuration. This is cumbersome.

It would be useful if buildman could adjust the configuration of a build
on the fly. In preparation for this, add a utility library which can
modify a .config file according to various parameters passed to it.

Signed-off-by: Simon Glass <sjg@chromium.org>

tools/buildman/cfgutil.py

@@ -0,0 +1,235 @@
+# SPDX-License-Identifier: GPL-2.0+
+# Copyright 2022 Google LLC
+# Written by Simon Glass <sjg@chromium.org>
+#
+
+"""Utility functions for dealing with Kconfig .confing files"""
+
+import re
+
+from patman import tools
+
+RE_LINE = re.compile(r'(# )?CONFIG_([A-Z0-9_]+)(=(.*)| is not set)')
+RE_CFG = re.compile(r'(~?)(CONFIG_)?([A-Z0-9_]+)(=.*)?')
+
+def make_cfg_line(opt, adj):
+    """Make a new config line for an option
+
+    Args:
+        opt (str): Option to process, without CONFIG_ prefix
+        adj (str): Adjustment to make (C is config option without prefix):
+             C to enable C
+             ~C to disable C
+             C=val to set the value of C (val must have quotes if C is
+                 a string Kconfig)
+
+    Returns:
+        str: New line to use, one of:
+            CONFIG_opt=y               - option is enabled
+            # CONFIG_opt is not set    - option is disabled
+            CONFIG_opt=val             - option is getting a new value (val is
+                in quotes if this is a string)
+    """
+    if adj[0] == '~':
+        return f'# CONFIG_{opt} is not set'
+    if '=' in adj:
+        return f'CONFIG_{adj}'
+    return f'CONFIG_{opt}=y'
+
+def adjust_cfg_line(line, adjust_cfg, done=None):
+    """Make an adjustment to a single of line from a .config file
+
+    This processes a .config line, producing a new line if a change for this
+    CONFIG is requested in adjust_cfg
+
+    Args:
+        line (str): line to process, e.g. '# CONFIG_FRED is not set' or
+            'CONFIG_FRED=y' or 'CONFIG_FRED=0x123' or 'CONFIG_FRED="fred"'
+        adjust_cfg (dict of str): Changes to make to .config file before
+                building:
+             key: str config to change, without the CONFIG_ prefix, e.g.
+                 FRED
+             value: str change to make (C is config option without prefix):
+                 C to enable C
+                 ~C to disable C
+                 C=val to set the value of C (val must have quotes if C is
+                     a string Kconfig)
+        done (set of set): Adds the config option to this set if it is changed
+            in some way. This is used to track which ones have been processed.
+            None to skip.
+
+    Returns:
+        tuple:
+            str: New string for this line (maybe unchanged)
+            str: Adjustment string that was used
+    """
+    out_line = line
+    m_line = RE_LINE.match(line)
+    adj = None
+    if m_line:
+        _, opt, _, _ = m_line.groups()
+        adj = adjust_cfg.get(opt)
+        if adj:
+            out_line = make_cfg_line(opt, adj)
+            if done is not None:
+                done.add(opt)
+
+    return out_line, adj
+
+def adjust_cfg_lines(lines, adjust_cfg):
+    """Make adjustments to a list of lines from a .config file
+
+    Args:
+        lines (list of str): List of lines to process
+        adjust_cfg (dict of str): Changes to make to .config file before
+                building:
+             key: str config to change, without the CONFIG_ prefix, e.g.
+                 FRED
+             value: str change to make (C is config option without prefix):
+                 C to enable C
+                 ~C to disable C
+                 C=val to set the value of C (val must have quotes if C is
+                     a string Kconfig)
+
+    Returns:
+        list of str: New list of lines resulting from the processing
+    """
+    out_lines = []
+    done = set()
+    for line in lines:
+        out_line, _ = adjust_cfg_line(line, adjust_cfg, done)
+        out_lines.append(out_line)
+
+    for opt in adjust_cfg:
+        if opt not in done:
+            adj = adjust_cfg.get(opt)
+            out_line = make_cfg_line(opt, adj)
+            out_lines.append(out_line)
+
+    return out_lines
+
+def adjust_cfg_file(fname, adjust_cfg):
+    """Make adjustments to a .config file
+
+    Args:
+        fname (str): Filename of .config file to change
+        adjust_cfg (dict of str): Changes to make to .config file before
+                building:
+             key: str config to change, without the CONFIG_ prefix, e.g.
+                 FRED
+             value: str change to make (C is config option without prefix):
+                 C to enable C
+                 ~C to disable C
+                 C=val to set the value of C (val must have quotes if C is
+                     a string Kconfig)
+    """
+    lines = tools.ReadFile(fname, binary=False).splitlines()
+    out_lines = adjust_cfg_lines(lines, adjust_cfg)
+    out = '\n'.join(out_lines) + '\n'
+    tools.WriteFile(fname, out, binary=False)
+
+def convert_list_to_dict(adjust_cfg_list):
+    """Convert a list of config changes into the dict used by adjust_cfg_file()
+
+    Args:
+        adjust_cfg_list (list of str): List of changes to make to .config file
+            before building. Each is one of (where C is the config option with
+            or without the CONFIG_ prefix)
+
+                C to enable C
+                ~C to disable C
+                C=val to set the value of C (val must have quotes if C is
+                    a string Kconfig
+
+    Returns:
+        dict of str: Changes to make to .config file before building:
+             key: str config to change, without the CONFIG_ prefix, e.g. FRED
+             value: str change to make (C is config option without prefix):
+                 C to enable C
+                 ~C to disable C
+                 C=val to set the value of C (val must have quotes if C is
+                     a string Kconfig)
+
+    Raises:
+        ValueError: if an item in adjust_cfg_list has invalid syntax
+    """
+    result = {}
+    for cfg in adjust_cfg_list or []:
+        m_cfg = RE_CFG.match(cfg)
+        if not m_cfg:
+            raise ValueError(f"Invalid CONFIG adjustment '{cfg}'")
+        negate, _, opt, val = m_cfg.groups()
+        result[opt] = f'%s{opt}%s' % (negate or '', val or '')
+
+    return result
+
+def check_cfg_lines(lines, adjust_cfg):
+    """Check that lines do not conflict with the requested changes
+
+    If a line enables a CONFIG which was requested to be disabled, etc., then
+    this is an error. This function finds such errors.
+
+    Args:
+        lines (list of str): List of lines to process
+        adjust_cfg (dict of str): Changes to make to .config file before
+                building:
+             key: str config to change, without the CONFIG_ prefix, e.g.
+                 FRED
+             value: str change to make (C is config option without prefix):
+                 C to enable C
+                 ~C to disable C
+                 C=val to set the value of C (val must have quotes if C is
+                     a string Kconfig)
+
+    Returns:
+        list of tuple: list of errors, each a tuple:
+            str: cfg adjustment requested
+            str: line of the config that conflicts
+    """
+    bad = []
+    done = set()
+    for line in lines:
+        out_line, adj = adjust_cfg_line(line, adjust_cfg, done)
+        if out_line != line:
+            bad.append([adj, line])
+
+    for opt in adjust_cfg:
+        if opt not in done:
+            adj = adjust_cfg.get(opt)
+            out_line = make_cfg_line(opt, adj)
+            bad.append([adj, f'Missing expected line: {out_line}'])
+
+    return bad
+
+def check_cfg_file(fname, adjust_cfg):
+    """Check that a config file has been adjusted according to adjust_cfg
+
+    Args:
+        fname (str): Filename of .config file to change
+        adjust_cfg (dict of str): Changes to make to .config file before
+                building:
+             key: str config to change, without the CONFIG_ prefix, e.g.
+                 FRED
+             value: str change to make (C is config option without prefix):
+                 C to enable C
+                 ~C to disable C
+                 C=val to set the value of C (val must have quotes if C is
+                     a string Kconfig)
+
+    Returns:
+        str: None if OK, else an error string listing the problems
+    """
+    lines = tools.ReadFile(fname, binary=False).splitlines()
+    bad_cfgs = check_cfg_lines(lines, adjust_cfg)
+    if bad_cfgs:
+        out = [f'{cfg:20}  {line}' for cfg, line in bad_cfgs]
+        content = '\\n'.join(out)
+        return f'''
+Some CONFIG adjustments did not take effect. This may be because
+the request CONFIGs do not exist or conflict with others.
+
+Failed adjustments:
+
+{content}
+'''
+    return None

tools/buildman/func_test.py

@@ -182,11 +182,11 @@ class TestFunctional(unittest.TestCase):
         self._buildman_pathname = sys.argv[0]
         self._buildman_dir = os.path.dirname(os.path.realpath(sys.argv[0]))
         command.test_result = self._HandleCommand
+        bsettings.Setup(None)
+        bsettings.AddFile(settings_data)
         self.setupToolchains()
         self._toolchains.Add('arm-gcc', test=False)
         self._toolchains.Add('powerpc-gcc', test=False)
-        bsettings.Setup(None)
-        bsettings.AddFile(settings_data)
         self._boards = board.Boards()
         for brd in boards:
             self._boards.AddBoard(board.Board(*brd))

tools/buildman/test.py

@@ -12,6 +12,7 @@ import unittest
 from buildman import board
 from buildman import bsettings
 from buildman import builder
+from buildman import cfgutil
 from buildman import control
 from buildman import toolchain
 from patman import commit
@@ -624,5 +625,127 @@ class TestBuild(unittest.TestCase):
         expected = set([os.path.join(base_dir, f) for f in to_remove])
         self.assertEqual(expected, result)
 
+    def test_adjust_cfg_nop(self):
+        """check various adjustments of config that are nops"""
+        # enable an enabled CONFIG
+        self.assertEqual(
+            'CONFIG_FRED=y',
+            cfgutil.adjust_cfg_line('CONFIG_FRED=y', {'FRED':'FRED'})[0])
+
+        # disable a disabled CONFIG
+        self.assertEqual(
+            '# CONFIG_FRED is not set',
+            cfgutil.adjust_cfg_line(
+                '# CONFIG_FRED is not set', {'FRED':'~FRED'})[0])
+
+        # use the adjust_cfg_lines() function
+        self.assertEqual(
+            ['CONFIG_FRED=y'],
+            cfgutil.adjust_cfg_lines(['CONFIG_FRED=y'], {'FRED':'FRED'}))
+        self.assertEqual(
+            ['# CONFIG_FRED is not set'],
+            cfgutil.adjust_cfg_lines(['CONFIG_FRED=y'], {'FRED':'~FRED'}))
+
+        # handling an empty line
+        self.assertEqual('#', cfgutil.adjust_cfg_line('#', {'FRED':'~FRED'})[0])
+
+    def test_adjust_cfg(self):
+        """check various adjustments of config"""
+        # disable a CONFIG
+        self.assertEqual(
+            '# CONFIG_FRED is not set',
+            cfgutil.adjust_cfg_line('CONFIG_FRED=1' , {'FRED':'~FRED'})[0])
+
+        # enable a disabled CONFIG
+        self.assertEqual(
+            'CONFIG_FRED=y',
+            cfgutil.adjust_cfg_line(
+                '# CONFIG_FRED is not set', {'FRED':'FRED'})[0])
+
+        # enable a CONFIG that doesn't exist
+        self.assertEqual(
+            ['CONFIG_FRED=y'],
+            cfgutil.adjust_cfg_lines([], {'FRED':'FRED'}))
+
+        # disable a CONFIG that doesn't exist
+        self.assertEqual(
+            ['# CONFIG_FRED is not set'],
+            cfgutil.adjust_cfg_lines([], {'FRED':'~FRED'}))
+
+        # disable a value CONFIG
+        self.assertEqual(
+            '# CONFIG_FRED is not set',
+            cfgutil.adjust_cfg_line('CONFIG_FRED="fred"' , {'FRED':'~FRED'})[0])
+
+        # setting a value CONFIG
+        self.assertEqual(
+            'CONFIG_FRED="fred"',
+            cfgutil.adjust_cfg_line('# CONFIG_FRED is not set' ,
+                                    {'FRED':'FRED="fred"'})[0])
+
+        # changing a value CONFIG
+        self.assertEqual(
+            'CONFIG_FRED="fred"',
+            cfgutil.adjust_cfg_line('CONFIG_FRED="ernie"' ,
+                                    {'FRED':'FRED="fred"'})[0])
+
+        # setting a value for a CONFIG that doesn't exist
+        self.assertEqual(
+            ['CONFIG_FRED="fred"'],
+            cfgutil.adjust_cfg_lines([], {'FRED':'FRED="fred"'}))
+
+    def test_convert_adjust_cfg_list(self):
+        """Check conversion of the list of changes into a dict"""
+        self.assertEqual({}, cfgutil.convert_list_to_dict(None))
+
+        expect = {
+            'FRED':'FRED',
+            'MARY':'~MARY',
+            'JOHN':'JOHN=0x123',
+            'ALICE':'ALICE="alice"',
+            'AMY':'AMY',
+            'ABE':'~ABE',
+            'MARK':'MARK=0x456',
+            'ANNA':'ANNA="anna"',
+            }
+        actual = cfgutil.convert_list_to_dict(
+            ['FRED', '~MARY', 'JOHN=0x123', 'ALICE="alice"',
+             'CONFIG_AMY', '~CONFIG_ABE', 'CONFIG_MARK=0x456',
+             'CONFIG_ANNA="anna"'])
+        self.assertEqual(expect, actual)
+
+    def test_check_cfg_file(self):
+        """Test check_cfg_file detects conflicts as expected"""
+        # Check failure to disable CONFIG
+        result = cfgutil.check_cfg_lines(['CONFIG_FRED=1'], {'FRED':'~FRED'})
+        self.assertEqual([['~FRED', 'CONFIG_FRED=1']], result)
+
+        result = cfgutil.check_cfg_lines(
+            ['CONFIG_FRED=1', 'CONFIG_MARY="mary"'], {'FRED':'~FRED'})
+        self.assertEqual([['~FRED', 'CONFIG_FRED=1']], result)
+
+        result = cfgutil.check_cfg_lines(
+            ['CONFIG_FRED=1', 'CONFIG_MARY="mary"'], {'MARY':'~MARY'})
+        self.assertEqual([['~MARY', 'CONFIG_MARY="mary"']], result)
+
+        # Check failure to enable CONFIG
+        result = cfgutil.check_cfg_lines(
+            ['# CONFIG_FRED is not set'], {'FRED':'FRED'})
+        self.assertEqual([['FRED', '# CONFIG_FRED is not set']], result)
+
+        # Check failure to set CONFIG value
+        result = cfgutil.check_cfg_lines(
+            ['# CONFIG_FRED is not set', 'CONFIG_MARY="not"'],
+            {'MARY':'MARY="mary"', 'FRED':'FRED'})
+        self.assertEqual([
+            ['FRED', '# CONFIG_FRED is not set'],
+            ['MARY="mary"', 'CONFIG_MARY="not"']], result)
+
+        # Check failure to add CONFIG value
+        result = cfgutil.check_cfg_lines([], {'MARY':'MARY="mary"'})
+        self.assertEqual([
+            ['MARY="mary"', 'Missing expected line: CONFIG_MARY="mary"']], result)
+
+
 if __name__ == "__main__":
     unittest.main()