nixos/tests/test-driver: normalise test driver entrypoint(s)

Previously the driver was configured exclusively through convoluted
environment variables.

Now the driver's defaults are configured through env variables.

Some additional concerns are in the github comments of this PR.
This commit is contained in:
David Arnold 2021-06-06 12:00:12 -05:00 committed by David Arnold
parent 077b2825cd
commit 926fb93968
No known key found for this signature in database
GPG Key ID: AB15A6AF1101390D
4 changed files with 89 additions and 63 deletions

View File

@ -5,7 +5,7 @@ when developing or debugging a test:
```ShellSession
$ nix-build nixos/tests/login.nix -A driverInteractive
$ ./result/bin/nixos-test-driver
$ ./result/bin/nixos-test-driver --interactive
starting VDE switch for network 1
>
```
@ -24,20 +24,11 @@ back into the test driver command line upon its completion. This allows
you to inspect the state of the VMs after the test (e.g. to debug the
test script).
To just start and experiment with the VMs, run:
```ShellSession
$ nix-build nixos/tests/login.nix -A driverInteractive
$ ./result/bin/nixos-run-vms
```
The script `nixos-run-vms` starts the virtual machines defined by test.
You can re-use the VM states coming from a previous run by setting the
`--keep-vm-state` flag.
```ShellSession
$ ./result/bin/nixos-run-vms --keep-vm-state
$ ./result/bin/nixos-test-driver --interactive --keep-vm-state
```
The machine state is stored in the `$TMPDIR/vm-state-machinename`

View File

@ -6,7 +6,7 @@
</para>
<programlisting>
$ nix-build nixos/tests/login.nix -A driverInteractive
$ ./result/bin/nixos-test-driver
$ ./result/bin/nixos-test-driver --interactive
starting VDE switch for network 1
&gt;
</programlisting>
@ -25,23 +25,12 @@ starting VDE switch for network 1
completion. This allows you to inspect the state of the VMs after
the test (e.g. to debug the test script).
</para>
<para>
To just start and experiment with the VMs, run:
</para>
<programlisting>
$ nix-build nixos/tests/login.nix -A driverInteractive
$ ./result/bin/nixos-run-vms
</programlisting>
<para>
The script <literal>nixos-run-vms</literal> starts the virtual
machines defined by test.
</para>
<para>
You can re-use the VM states coming from a previous run by setting
the <literal>--keep-vm-state</literal> flag.
</para>
<programlisting>
$ ./result/bin/nixos-run-vms --keep-vm-state
$ ./result/bin/nixos-test-driver --interactive --keep-vm-state
</programlisting>
<para>
The machine state is stored in the

101
nixos/lib/test-driver/test-driver.py Normal file → Executable file
View File

@ -24,7 +24,6 @@ import sys
import telnetlib
import tempfile
import time
import traceback
import unicodedata
CHAR_TO_KEY = {
@ -930,29 +929,16 @@ def join_all() -> None:
machine.wait_for_shutdown()
def test_script() -> None:
exec(os.environ["testScript"])
def run_tests() -> None:
def run_tests(interactive: bool = False) -> None:
global machines
tests = os.environ.get("tests", None)
if tests is not None:
with log.nested("running the VM test script"):
try:
exec(tests, globals())
except Exception as e:
eprint("error: ")
traceback.print_exc()
sys.exit(1)
if interactive:
ptpython.repl.embed(globals(), locals())
else:
ptpython.repl.embed(locals(), globals())
# TODO: Collect coverage data
for machine in machines:
if machine.is_up():
machine.execute("sync")
test_script()
# TODO: Collect coverage data
for machine in machines:
if machine.is_up():
machine.execute("sync")
def serial_stdout_on() -> None:
@ -965,6 +951,31 @@ def serial_stdout_off() -> None:
log._print_serial_logs = False
class EnvDefault(argparse.Action):
"""An argpars Action that takes values from the specified
environment variable as the flags default value.
"""
def __init__(self, envvar, required=False, default=None, nargs=None, **kwargs): # type: ignore
if not default and envvar:
if envvar in os.environ:
if nargs is not None and (nargs.isdigit() or nargs in ["*", "+"]):
default = os.environ[envvar].split()
else:
default = os.environ[envvar]
kwargs["help"] = (
kwargs["help"] + f" (default from environment: {default})"
)
if required and default:
required = False
super(EnvDefault, self).__init__(
default=default, required=required, nargs=nargs, **kwargs
)
def __call__(self, parser, namespace, values, option_string=None): # type: ignore
setattr(namespace, self.dest, values)
@contextmanager
def subtest(name: str) -> Iterator[None]:
with log.nested(name):
@ -986,18 +997,52 @@ if __name__ == "__main__":
help="re-use a VM state coming from a previous run",
action="store_true",
)
(cli_args, vm_scripts) = arg_parser.parse_known_args()
arg_parser.add_argument(
"-I",
"--interactive",
help="drop into a python repl and run the tests interactively",
action="store_true",
)
arg_parser.add_argument(
"--start-scripts",
metavar="START-SCRIPT",
action=EnvDefault,
envvar="startScripts",
nargs="*",
help="start scripts for participating virtual machines",
)
arg_parser.add_argument(
"--vlans",
metavar="VLAN",
action=EnvDefault,
envvar="vlans",
nargs="*",
help="vlans to span by the driver",
)
arg_parser.add_argument(
"testscript",
action=EnvDefault,
envvar="testScript",
help="the test script to run",
type=pathlib.Path,
)
args = arg_parser.parse_args()
global test_script
def test_script() -> None:
with log.nested("running the VM test script"):
exec(pathlib.Path(args.testscript).read_text(), globals())
log = Logger()
vlan_nrs = list(dict.fromkeys(os.environ.get("VLANS", "").split()))
vde_sockets = [create_vlan(v) for v in vlan_nrs]
vde_sockets = [create_vlan(v) for v in args.vlans]
for nr, vde_socket, _, _ in vde_sockets:
os.environ["QEMU_VDE_SOCKET_{}".format(nr)] = vde_socket
machines = [
create_machine({"startCommand": s, "keepVmState": cli_args.keep_vm_state})
for s in vm_scripts
create_machine({"startCommand": s, "keepVmState": args.keep_vm_state})
for s in args.start_scripts
]
machine_eval = [
"{0} = machines[{1}]".format(m.name, idx) for idx, m in enumerate(machines)
@ -1017,6 +1062,6 @@ if __name__ == "__main__":
log.close()
tic = time.time()
run_tests()
run_tests(args.interactive)
toc = time.time()
print("test script finished in {:.2f}s".format(toc - tic))

View File

@ -83,7 +83,10 @@ rec {
''
mkdir -p $out
LOGFILE=/dev/null tests='exec(os.environ["testScript"])' ${driver}/bin/nixos-test-driver
# effectively mute the XMLLogger
export LOGFILE=/dev/null
${driver}/bin/nixos-test-driver
'';
passthru = driver.passthru // {
@ -161,7 +164,10 @@ rec {
''
mkdir -p $out/bin
vmStartScripts=($(for i in ${toString vms}; do echo $i/bin/run-*-vm; done))
echo -n "$testScript" > $out/test-script
ln -s ${testDriver}/bin/nixos-test-driver $out/bin/nixos-test-driver
${lib.optionalString (!skipLint) ''
PYFLAKES_BUILTINS="$(
echo -n ${lib.escapeShellArg (lib.concatStringsSep "," nodeHostNames)},
@ -169,17 +175,12 @@ rec {
)" ${python3Packages.pyflakes}/bin/pyflakes $out/test-script
''}
ln -s ${testDriver}/bin/nixos-test-driver $out/bin/
vms=($(for i in ${toString vms}; do echo $i/bin/run-*-vm; done))
# set defaults through environment
# see: ./test-driver/test-driver.py argparse implementation
wrapProgram $out/bin/nixos-test-driver \
--add-flags "''${vms[*]}" \
--run "export testScript=\"\$(${coreutils}/bin/cat $out/test-script)\"" \
--set VLANS '${toString vlans}'
ln -s ${testDriver}/bin/nixos-test-driver $out/bin/nixos-run-vms
wrapProgram $out/bin/nixos-run-vms \
--add-flags "''${vms[*]}" \
--set tests 'start_all(); join_all();' \
--set VLANS '${toString vlans}'
--set startScripts "''${vmStartScripts[*]}" \
--set testScript "$out/test-script" \
--set vlans '${toString vlans}'
'');
# Make a full-blown test