colin/nixpkgs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

nixos/testing: Move entrypoint to nixos/lib + doc

Browse Source
This commit is contained in:
Robert Hensing 2022-06-15 16:59:21 +02:00
parent 5727fd3e6f
commit b0c781cc41
5 changed files with 256 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

73
nixos/doc/manual/development/writing-nixos-tests.section.md
View File

@ -1,9 +1,9 @@
# Writing Tests {#sec-writing-nixos-tests}
A NixOS test is a Nix expression that has the following structure:
A NixOS test is a module that has the following structure:
```nix
import ./make-test-python.nix {
{
# One or more machines:
nodes =
@ -21,7 +21,10 @@ import ./make-test-python.nix {
}
```
The attribute `testScript` is a bit of Python code that executes the
We refer to the whole test above as a test module, whereas the values
in `nodes.<name>` are NixOS modules. (A NixOS configuration is a module.)
The option `testScript` is a bit of Python code that executes the
test (described below). During the test, it will start one or more
virtual machines, the configuration of which is described by
the attribute `nodes`.
@ -34,7 +37,64 @@ when switching between consoles, and so on. An interesting multi-node test is
[`nfs/simple.nix`](https://github.com/NixOS/nixpkgs/blob/master/nixos/tests/nfs/simple.nix).
It uses two client nodes to test correct locking across server crashes.
There are a few special NixOS configuration options for test VMs:
## Calling a test {#sec-calling-nixos-tests}
Tests are invoked a bit differently depending on whether the test lives in NixOS or in another project.
### Testing within NixOS {#sec-call-nixos-test-in-nixos}
Test modules can be instantiated into derivations in multiple ways.
Tests that are part of NixOS are added to [`nixos/tests/all-tests.nix`](https://github.com/NixOS/nixpkgs/blob/master/nixos/tests/all-tests.nix).
```nix
hostname = runTest ./hostname.nix;
```
Overrides can be added by defining an anonymous module in `all-tests.nix`.
For the purpose of constructing a test matrix, use the `matrix` options instead.
```nix
hostname = runTest { imports = [ ./hostname.nix ]; defaults.networking.firewall.enable = false; };
```
You can run a test with attribute name `mytest` in `all-tests.nix` by invoking:
```shell
nix-build -A nixosTests.mytest
```
### Testing outside the NixOS project {#sec-call-nixos-test-outside-nixos}
Outside the `nixpkgs` repository, you can instantiate the test by first acquiring the NixOS library,
```nix
# regular nix
let nixos-lib = import (nixpkgs + "/nixos/lib") { };
in
```
```nix
# flake
let nixos-lib = nixpkgs.lib.nixos;
in
```
... and then invoking `runTest`, for example:
```nix
nixos-lib.runTest {
imports = [ ./test.nix ];
hostPkgs = pkgs; # the Nixpkgs package set used outside the VMs
defaults.services.foo.package = mypkg;
}
```
`runTest` returns a derivation that runs the test.
## Configuring the nodes {#sec-nixos-test-nodes}
There are a few special NixOS options for test VMs:
`virtualisation.memorySize`
@ -304,7 +364,7 @@ For faster dev cycles it\'s also possible to disable the code-linters
(this shouldn\'t be commited though):
```nix
import ./make-test-python.nix {
{
skipLint = true;
nodes.machine =
{ config, pkgs, ... }:
@ -336,7 +396,7 @@ Similarly, the type checking of test scripts can be disabled in the following
way:
```nix
import ./make-test-python.nix {
{
skipTypeCheck = true;
nodes.machine =
{ config, pkgs, ... }:
@ -400,7 +460,6 @@ added using the parameter `extraPythonPackages`. For example, you could add
`numpy` like this:
```nix
import ./make-test-python.nix
{
extraPythonPackages = p: [ p.numpy ];

217
nixos/doc/manual/from_md/development/writing-nixos-tests.section.xml
View File

@ -1,10 +1,10 @@
<section xmlns="http://docbook.org/ns/docbook" xmlns:xlink="http://www.w3.org/1999/xlink" xml:id="sec-writing-nixos-tests">
<title>Writing Tests</title>
<para>
A NixOS test is a Nix expression that has the following structure:
A NixOS test is a module that has the following structure:
</para>
<programlisting language="bash">
import ./make-test-python.nix {
{
# One or more machines:
nodes =
@ -22,7 +22,12 @@ import ./make-test-python.nix {
}
</programlisting>
<para>
The attribute <literal>testScript</literal> is a bit of Python code
We refer to the whole test above as a test module, whereas the
values in <literal>nodes.&lt;name&gt;</literal> are NixOS modules.
(A NixOS configuration is a module.)
</para>
<para>
The option <literal>testScript</literal> is a bit of Python code
that executes the test (described below). During the test, it will
start one or more virtual machines, the configuration of which is
described by the attribute <literal>nodes</literal>.
@ -38,78 +43,149 @@ import ./make-test-python.nix {
It uses two client nodes to test correct locking across server
crashes.
</para>
<para>
There are a few special NixOS configuration options for test VMs:
</para>
<variablelist>
<varlistentry>
<term>
<literal>virtualisation.memorySize</literal>
</term>
<listitem>
<para>
The memory of the VM in megabytes.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term>
<literal>virtualisation.vlans</literal>
</term>
<listitem>
<para>
The virtual networks to which the VM is connected. See
<link xlink:href="https://github.com/NixOS/nixpkgs/blob/master/nixos/tests/nat.nix"><literal>nat.nix</literal></link>
for an example.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term>
<literal>virtualisation.writableStore</literal>
</term>
<listitem>
<para>
By default, the Nix store in the VM is not writable. If you
enable this option, a writable union file system is mounted on
top of the Nix store to make it appear writable. This is
necessary for tests that run Nix operations that modify the
store.
</para>
</listitem>
</varlistentry>
</variablelist>
<para>
For more options, see the module
<link xlink:href="https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/virtualisation/qemu-vm.nix"><literal>qemu-vm.nix</literal></link>.
</para>
<para>
The test script is a sequence of Python statements that perform
various actions, such as starting VMs, executing commands in the
VMs, and so on. Each virtual machine is represented as an object
stored in the variable <literal>name</literal> if this is also the
identifier of the machine in the declarative config. If you
specified a node <literal>nodes.machine</literal>, the following
example starts the machine, waits until it has finished booting,
then executes a command and checks that the output is more-or-less
correct:
</para>
<programlisting language="python">
<section xml:id="sec-calling-nixos-tests">
<title>Calling a test</title>
<para>
Tests are invoked a bit differently depending on whether the test
lives in NixOS or in another project.
</para>
<section xml:id="sec-call-nixos-test-in-nixos">
<title>Testing within NixOS</title>
<para>
Test modules can be instantiated into derivations in multiple
ways.
</para>
<para>
Tests that are part of NixOS are added to
<link xlink:href="https://github.com/NixOS/nixpkgs/blob/master/nixos/tests/all-tests.nix"><literal>nixos/tests/all-tests.nix</literal></link>.
</para>
<programlisting language="bash">
hostname = runTest ./hostname.nix;
</programlisting>
<para>
Overrides can be added by defining an anonymous module in
<literal>all-tests.nix</literal>. For the purpose of
constructing a test matrix, use the <literal>matrix</literal>
options instead.
</para>
<programlisting language="bash">
hostname = runTest { imports = [ ./hostname.nix ]; defaults.networking.firewall.enable = false; };
</programlisting>
<para>
You can run a test with attribute name <literal>mytest</literal>
in <literal>all-tests.nix</literal> by invoking:
</para>
<programlisting>
nix-build -A nixosTests.mytest
</programlisting>
</section>
<section xml:id="sec-call-nixos-test-outside-nixos">
<title>Testing outside the NixOS project</title>
<para>
Outside the <literal>nixpkgs</literal> repository, you can
instantiate the test by first acquiring the NixOS library,
</para>
<programlisting language="bash">
# regular nix
let nixos-lib = import (nixpkgs + &quot;/nixos/lib&quot;) { };
in
</programlisting>
<programlisting language="bash">
# flake
let nixos-lib = nixpkgs.lib.nixos;
in
</programlisting>
<para>
… and then invoking <literal>runTest</literal>, for example:
</para>
<programlisting language="bash">
nixos-lib.runTest {
imports = [ ./test.nix ];
hostPkgs = pkgs; # the Nixpkgs package set used outside the VMs
defaults.services.foo.package = mypkg;
}
</programlisting>
<para>
<literal>runTest</literal> returns a derivation that runs the
test.
</para>
</section>
</section>
<section xml:id="sec-nixos-test-nodes">
<title>Configuring the nodes</title>
<para>
There are a few special NixOS options for test VMs:
</para>
<variablelist>
<varlistentry>
<term>
<literal>virtualisation.memorySize</literal>
</term>
<listitem>
<para>
The memory of the VM in megabytes.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term>
<literal>virtualisation.vlans</literal>
</term>
<listitem>
<para>
The virtual networks to which the VM is connected. See
<link xlink:href="https://github.com/NixOS/nixpkgs/blob/master/nixos/tests/nat.nix"><literal>nat.nix</literal></link>
for an example.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term>
<literal>virtualisation.writableStore</literal>
</term>
<listitem>
<para>
By default, the Nix store in the VM is not writable. If you
enable this option, a writable union file system is mounted
on top of the Nix store to make it appear writable. This is
necessary for tests that run Nix operations that modify the
store.
</para>
</listitem>
</varlistentry>
</variablelist>
<para>
For more options, see the module
<link xlink:href="https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/virtualisation/qemu-vm.nix"><literal>qemu-vm.nix</literal></link>.
</para>
<para>
The test script is a sequence of Python statements that perform
various actions, such as starting VMs, executing commands in the
VMs, and so on. Each virtual machine is represented as an object
stored in the variable <literal>name</literal> if this is also the
identifier of the machine in the declarative config. If you
specified a node <literal>nodes.machine</literal>, the following
example starts the machine, waits until it has finished booting,
then executes a command and checks that the output is more-or-less
correct:
</para>
<programlisting language="python">
machine.start()
machine.wait_for_unit(&quot;default.target&quot;)
if not &quot;Linux&quot; in machine.succeed(&quot;uname&quot;):
raise Exception(&quot;Wrong OS&quot;)
</programlisting>
<para>
The first line is technically unnecessary; machines are implicitly
started when you first execute an action on them (such as
<literal>wait_for_unit</literal> or <literal>succeed</literal>). If
you have multiple machines, you can speed up the test by starting
them in parallel:
</para>
<programlisting language="python">
<para>
The first line is technically unnecessary; machines are implicitly
started when you first execute an action on them (such as
<literal>wait_for_unit</literal> or <literal>succeed</literal>).
If you have multiple machines, you can speed up the test by
starting them in parallel:
</para>
<programlisting language="python">
start_all()
</programlisting>
</section>
<section xml:id="ssec-machine-objects">
<title>Machine objects</title>
<para>
@ -563,7 +639,7 @@ machine.wait_for_unit(&quot;xautolock.service&quot;, &quot;x-session-user&quot;)
code-linters (this shouldn't be commited though):
</para>
<programlisting language="bash">
import ./make-test-python.nix {
{
skipLint = true;
nodes.machine =
{ config, pkgs, ... }:
@ -595,7 +671,7 @@ import ./make-test-python.nix {
the following way:
</para>
<programlisting language="bash">
import ./make-test-python.nix {
{
skipTypeCheck = true;
nodes.machine =
{ config, pkgs, ... }:
@ -669,7 +745,6 @@ def foo_running():
<literal>numpy</literal> like this:
</para>
<programlisting language="bash">
import ./make-test-python.nix
{
extraPythonPackages = p: [ p.numpy ];

8
nixos/lib/default.nix
View File

@ -21,6 +21,8 @@ let
seqAttrsIf = cond: a: lib.mapAttrs (_: v: seqIf cond a v);
eval-config-minimal = import ./eval-config-minimal.nix { inherit lib; };
testing-lib = import ./testing/default.nix { inherit lib; };
in
/*
This attribute set appears as lib.nixos in the flake, or can be imported
@ -30,4 +32,10 @@ in
inherit (seqAttrsIf (!featureFlags?minimalModules) minimalModulesWarning eval-config-minimal)
evalModules
;
inherit (testing-lib)
evalTest
runTest
;
}

31
nixos/lib/testing-python.nix
View File

@ -12,6 +12,10 @@
with pkgs;
let
nixos-lib = import ./default.nix { inherit (pkgs) lib; };
in
rec {
inherit pkgs;
@ -166,26 +170,15 @@ rec {
${lib.optionalString (interactive) "--add-flags --interactive"}
'');
evalTest = module: lib.evalModules { modules = testModules ++ [ module ]; };
runTest = module: (evalTest module).config.run;
evalTest = module: nixos-lib.evalTest { imports = [ extraTestModule module ]; };
runTest = module: nixos-lib.runTest { imports = [ extraTestModule module ]; };
testModules = [
./testing/driver.nix
./testing/interactive.nix
./testing/legacy.nix
./testing/meta.nix
./testing/name.nix
./testing/network.nix
./testing/nodes.nix
./testing/pkgs.nix
./testing/run.nix
./testing/testScript.nix
{
config = {
hostPkgs = pkgs;
};
}
];
extraTestModule = {
config = {
hostPkgs = pkgs;
minimalResult = hydra;
};
};
# Make a full-blown test
makeTest =

24
nixos/lib/testing/default.nix Normal file
View File

@ -0,0 +1,24 @@
{ lib }:
let
evalTest = module: lib.evalModules { modules = testModules ++ [ module ]; };
runTest = module: (evalTest module).config.result;
testModules = [
./call-test.nix
./driver.nix
./interactive.nix
./legacy.nix
./meta.nix
./name.nix
./network.nix
./nodes.nix
./pkgs.nix
./run.nix
./testScript.nix
];
in
{
inherit evalTest runTest testModules;
}