colin/nixpkgs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

nixos/testing: Move entrypoint to nixos/lib + doc

Browse Source
This commit is contained in:
Robert Hensing 2022-06-15 16:59:21 +02:00
parent 5727fd3e6f
commit b0c781cc41
5 changed files with 256 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

73
nixos/doc/manual/development/writing-nixos-tests.section.md
View File

@ -1,9 +1,9 @@
# Writing Tests {#sec-writing-nixos-tests}
A NixOS test is a Nix expression that has the following structure:
A NixOS test is a module that has the following structure:
```nix
import ./make-test-python.nix {
{
# One or more machines:
nodes =
@ -21,7 +21,10 @@ import ./make-test-python.nix {
}
```
The attribute `testScript` is a bit of Python code that executes the
We refer to the whole test above as a test module, whereas the values
in `nodes.<name>` are NixOS modules. (A NixOS configuration is a module.)
The option `testScript` is a bit of Python code that executes the
test (described below). During the test, it will start one or more
virtual machines, the configuration of which is described by
the attribute `nodes`.
@ -34,7 +37,64 @@ when switching between consoles, and so on. An interesting multi-node test is
[`nfs/simple.nix`](https://github.com/NixOS/nixpkgs/blob/master/nixos/tests/nfs/simple.nix).
It uses two client nodes to test correct locking across server crashes.
There are a few special NixOS configuration options for test VMs:
## Calling a test {#sec-calling-nixos-tests}
Tests are invoked a bit differently depending on whether the test lives in NixOS or in another project.
### Testing within NixOS {#sec-call-nixos-test-in-nixos}
Test modules can be instantiated into derivations in multiple ways.
Tests that are part of NixOS are added to [`nixos/tests/all-tests.nix`](https://github.com/NixOS/nixpkgs/blob/master/nixos/tests/all-tests.nix).
```nix
hostname = runTest ./hostname.nix;
```
Overrides can be added by defining an anonymous module in `all-tests.nix`.
For the purpose of constructing a test matrix, use the `matrix` options instead.
```nix
hostname = runTest { imports = [ ./hostname.nix ]; defaults.networking.firewall.enable = false; };
```
You can run a test with attribute name `mytest` in `all-tests.nix` by invoking:
```shell
nix-build -A nixosTests.mytest
```
### Testing outside the NixOS project {#sec-call-nixos-test-outside-nixos}
Outside the `nixpkgs` repository, you can instantiate the test by first acquiring the NixOS library,
```nix
# regular nix
let nixos-lib = import (nixpkgs + "/nixos/lib") { };
in
```
```nix
# flake
let nixos-lib = nixpkgs.lib.nixos;
in
```
... and then invoking `runTest`, for example:
```nix
nixos-lib.runTest {
imports = [ ./test.nix ];
hostPkgs = pkgs; # the Nixpkgs package set used outside the VMs
defaults.services.foo.package = mypkg;
}
```
`runTest` returns a derivation that runs the test.
## Configuring the nodes {#sec-nixos-test-nodes}
There are a few special NixOS options for test VMs:
`virtualisation.memorySize`
@ -304,7 +364,7 @@ For faster dev cycles it\'s also possible to disable the code-linters
(this shouldn\'t be commited though):
```nix
import ./make-test-python.nix {
{
skipLint = true;
nodes.machine =
{ config, pkgs, ... }:
@ -336,7 +396,7 @@ Similarly, the type checking of test scripts can be disabled in the following
way:
```nix
import ./make-test-python.nix {
{
skipTypeCheck = true;
nodes.machine =
{ config, pkgs, ... }:
@ -400,7 +460,6 @@ added using the parameter `extraPythonPackages`. For example, you could add
`numpy` like this:
```nix
import ./make-test-python.nix
{
extraPythonPackages = p: [ p.numpy ];

99
nixos/doc/manual/from_md/development/writing-nixos-tests.section.xml
View File

@ -1,10 +1,10 @@
<section xmlns="http://docbook.org/ns/docbook" xmlns:xlink="http://www.w3.org/1999/xlink" xml:id="sec-writing-nixos-tests">
<title>Writing Tests</title>
<para>
A NixOS test is a Nix expression that has the following structure:
A NixOS test is a module that has the following structure:
</para>
<programlisting language="bash">
import ./make-test-python.nix {
{
# One or more machines:
nodes =
@ -22,7 +22,12 @@ import ./make-test-python.nix {
}
</programlisting>
<para>
The attribute <literal>testScript</literal> is a bit of Python code
We refer to the whole test above as a test module, whereas the
values in <literal>nodes.&lt;name&gt;</literal> are NixOS modules.
(A NixOS configuration is a module.)
</para>
<para>
The option <literal>testScript</literal> is a bit of Python code
that executes the test (described below). During the test, it will
start one or more virtual machines, the configuration of which is
described by the attribute <literal>nodes</literal>.
@ -38,8 +43,78 @@ import ./make-test-python.nix {
It uses two client nodes to test correct locking across server
crashes.
</para>
<section xml:id="sec-calling-nixos-tests">
<title>Calling a test</title>
<para>
There are a few special NixOS configuration options for test VMs:
Tests are invoked a bit differently depending on whether the test
lives in NixOS or in another project.
</para>
<section xml:id="sec-call-nixos-test-in-nixos">
<title>Testing within NixOS</title>
<para>
Test modules can be instantiated into derivations in multiple
ways.
</para>
<para>
Tests that are part of NixOS are added to
<link xlink:href="https://github.com/NixOS/nixpkgs/blob/master/nixos/tests/all-tests.nix"><literal>nixos/tests/all-tests.nix</literal></link>.
</para>
<programlisting language="bash">
hostname = runTest ./hostname.nix;
</programlisting>
<para>
Overrides can be added by defining an anonymous module in
<literal>all-tests.nix</literal>. For the purpose of
constructing a test matrix, use the <literal>matrix</literal>
options instead.
</para>
<programlisting language="bash">
hostname = runTest { imports = [ ./hostname.nix ]; defaults.networking.firewall.enable = false; };
</programlisting>
<para>
You can run a test with attribute name <literal>mytest</literal>
in <literal>all-tests.nix</literal> by invoking:
</para>
<programlisting>
nix-build -A nixosTests.mytest
</programlisting>
</section>
<section xml:id="sec-call-nixos-test-outside-nixos">
<title>Testing outside the NixOS project</title>
<para>
Outside the <literal>nixpkgs</literal> repository, you can
instantiate the test by first acquiring the NixOS library,
</para>
<programlisting language="bash">
# regular nix
let nixos-lib = import (nixpkgs + &quot;/nixos/lib&quot;) { };
in
</programlisting>
<programlisting language="bash">
# flake
let nixos-lib = nixpkgs.lib.nixos;
in
</programlisting>
<para>
… and then invoking <literal>runTest</literal>, for example:
</para>
<programlisting language="bash">
nixos-lib.runTest {
imports = [ ./test.nix ];
hostPkgs = pkgs; # the Nixpkgs package set used outside the VMs
defaults.services.foo.package = mypkg;
}
</programlisting>
<para>
<literal>runTest</literal> returns a derivation that runs the
test.
</para>
</section>
</section>
<section xml:id="sec-nixos-test-nodes">
<title>Configuring the nodes</title>
<para>
There are a few special NixOS options for test VMs:
</para>
<variablelist>
<varlistentry>
@ -71,8 +146,8 @@ import ./make-test-python.nix {
<listitem>
<para>
By default, the Nix store in the VM is not writable. If you
enable this option, a writable union file system is mounted on
top of the Nix store to make it appear writable. This is
enable this option, a writable union file system is mounted
on top of the Nix store to make it appear writable. This is
necessary for tests that run Nix operations that modify the
store.
</para>
@ -103,13 +178,14 @@ if not &quot;Linux&quot; in machine.succeed(&quot;uname&quot;):
<para>
The first line is technically unnecessary; machines are implicitly
started when you first execute an action on them (such as
<literal>wait_for_unit</literal> or <literal>succeed</literal>). If
you have multiple machines, you can speed up the test by starting
them in parallel:
<literal>wait_for_unit</literal> or <literal>succeed</literal>).
If you have multiple machines, you can speed up the test by
starting them in parallel:
</para>
<programlisting language="python">
start_all()
</programlisting>
</section>
<section xml:id="ssec-machine-objects">
<title>Machine objects</title>
<para>
@ -563,7 +639,7 @@ machine.wait_for_unit(&quot;xautolock.service&quot;, &quot;x-session-user&quot;)
code-linters (this shouldn't be commited though):
</para>
<programlisting language="bash">
import ./make-test-python.nix {
{
skipLint = true;
nodes.machine =
{ config, pkgs, ... }:
@ -595,7 +671,7 @@ import ./make-test-python.nix {
the following way:
</para>
<programlisting language="bash">
import ./make-test-python.nix {
{
skipTypeCheck = true;
nodes.machine =
{ config, pkgs, ... }:
@ -669,7 +745,6 @@ def foo_running():
<literal>numpy</literal> like this:
</para>
<programlisting language="bash">
import ./make-test-python.nix
{
extraPythonPackages = p: [ p.numpy ];

8
nixos/lib/default.nix
View File

@ -21,6 +21,8 @@ let
seqAttrsIf = cond: a: lib.mapAttrs (_: v: seqIf cond a v);
eval-config-minimal = import ./eval-config-minimal.nix { inherit lib; };
testing-lib = import ./testing/default.nix { inherit lib; };
in
/*
This attribute set appears as lib.nixos in the flake, or can be imported
@ -30,4 +32,10 @@ in
inherit (seqAttrsIf (!featureFlags?minimalModules) minimalModulesWarning eval-config-minimal)
evalModules
;
inherit (testing-lib)
evalTest
runTest
;
}

25
nixos/lib/testing-python.nix
View File

@ -12,6 +12,10 @@
with pkgs;
let
nixos-lib = import ./default.nix { inherit (pkgs) lib; };
in
rec {
inherit pkgs;
@ -166,26 +170,15 @@ rec {
${lib.optionalString (interactive) "--add-flags --interactive"}
'');
evalTest = module: lib.evalModules { modules = testModules ++ [ module ]; };
runTest = module: (evalTest module).config.run;
evalTest = module: nixos-lib.evalTest { imports = [ extraTestModule module ]; };
runTest = module: nixos-lib.runTest { imports = [ extraTestModule module ]; };
testModules = [
./testing/driver.nix
./testing/interactive.nix
./testing/legacy.nix
./testing/meta.nix
./testing/name.nix
./testing/network.nix
./testing/nodes.nix
./testing/pkgs.nix
./testing/run.nix
./testing/testScript.nix
{
extraTestModule = {
config = {
hostPkgs = pkgs;
minimalResult = hydra;
};
};
}
];
# Make a full-blown test
makeTest =

24
nixos/lib/testing/default.nix Normal file
View File

@ -0,0 +1,24 @@
{ lib }:
let
evalTest = module: lib.evalModules { modules = testModules ++ [ module ]; };
runTest = module: (evalTest module).config.result;
testModules = [
./call-test.nix
./driver.nix
./interactive.nix
./legacy.nix
./meta.nix
./name.nix
./network.nix
./nodes.nix
./pkgs.nix
./run.nix
./testScript.nix
];
in
{
inherit evalTest runTest testModules;
}