Add bcachefs type with encryption and multi-disk support

This update introduces a bcachefs type with encryption support and advanced formatting options.
It includes a new example (`examples/bcachefs-multi-disk.nix`) to demonstrate multi-disk setups and available options.

Key changes:

- Deterministic UUID generation.
- Addressed limitations with multi-disk root setups due to bcachefs and systemd issues.
- Provided a systemd-mount alternative for fileSystems configuration.
- Added subvolume support and updated scripts for clarity and functionality.

---------

Co-authored-by: Jonas Heinrich <onny@project-insanity.org>
Co-authored-by: Jörg Thalheim <Mic92@users.noreply.github.com>
Co-authored-by: Kyle Petryszak <6314611+ProjectInitiative@users.noreply.github.com>

Update
* Add examples
* Improve descriptions

Remove debugging

Remove comment

Use `unique` to dedup lists
This commit is contained in:
nothingnesses
2025-04-21 14:38:25 +01:00
committed by Jörg Thalheim
parent c5140c6079
commit ca27b88c88
6 changed files with 698 additions and 14 deletions

View File

@@ -1,15 +1,15 @@
{
disko.devices = {
disk = {
main = {
device = "/dev/disk/by-path/pci-0000:02:00.0-nvme-1";
vdb = {
device = "/dev/vdb";
type = "disk";
content = {
type = "gpt";
partitions = {
ESP = {
end = "500M";
vdb1 = {
type = "EF00";
size = "100M";
content = {
type = "filesystem";
format = "vfat";
@@ -17,18 +17,232 @@
mountOptions = [ "umask=0077" ];
};
};
root = {
name = "root";
end = "-0";
vdb2 = {
size = "100%";
content = {
type = "filesystem";
format = "bcachefs";
mountpoint = "/";
type = "bcachefs";
# This refers to a filesystem in the `bcachefs_filesystems` attrset below.
filesystem = "unmounted_subvolumes_in_multi";
label = "group_a.vdb2";
extraFormatArgs = [
"--discard"
];
};
};
};
};
};
vdc = {
device = "/dev/vdc";
type = "disk";
content = {
type = "gpt";
partitions = {
vdc1 = {
size = "100%";
content = {
type = "bcachefs";
filesystem = "unmounted_subvolumes_in_multi";
label = "group_a.vdc1";
extraFormatArgs = [
"--discard"
];
};
};
};
};
};
vdd = {
device = "/dev/vdd";
type = "disk";
content = {
type = "gpt";
partitions = {
vdd1 = {
size = "100%";
content = {
type = "bcachefs";
filesystem = "unmounted_subvolumes_in_multi";
label = "group_b.vdd1";
extraFormatArgs = [
"--force"
];
};
};
};
};
};
vde = {
device = "/dev/vde";
type = "disk";
content = {
type = "gpt";
partitions = {
vde1 = {
size = "100%";
content = {
type = "bcachefs";
filesystem = "mounted_subvolumes_in_multi";
label = "group_a.vde1";
extraFormatArgs = [
"--discard"
];
};
};
};
};
};
vdf = {
device = "/dev/vdf";
type = "disk";
content = {
type = "gpt";
partitions = {
vdf1 = {
size = "100%";
content = {
type = "bcachefs";
filesystem = "mounted_subvolumes_in_multi";
label = "group_a.vdf1";
extraFormatArgs = [
"--discard"
];
};
};
};
};
};
vdg = {
device = "/dev/vdg";
type = "disk";
content = {
type = "gpt";
partitions = {
vdd1 = {
size = "100%";
content = {
type = "bcachefs";
filesystem = "mounted_subvolumes_in_multi";
label = "group_b.vdg1";
extraFormatArgs = [
"--force"
];
};
};
};
};
};
vdh = {
device = "/dev/vdh";
type = "disk";
content = {
type = "gpt";
partitions = {
vdd1 = {
size = "100%";
content = {
type = "bcachefs";
filesystem = "no_reliance_on_external_subvolume";
label = "group_a.vdh1";
};
};
};
};
};
vdi = {
device = "/dev/vdi";
type = "disk";
content = {
type = "gpt";
partitions = {
vdd1 = {
size = "100%";
content = {
type = "bcachefs";
filesystem = "relies_on_external_subvolume";
label = "group_a.vdi1";
};
};
};
};
};
};
bcachefs_filesystems = {
# Example showing unmounted subvolumes in a multi-disk configuration.
unmounted_subvolumes_in_multi = {
type = "bcachefs_filesystem";
passwordFile = "/tmp/secret.key";
extraFormatArgs = [
"--compression=lz4"
"--background_compression=lz4"
];
mountOptions = [
"verbose"
];
mountpoint = "/";
subvolumes = {
"subvolumes/rootfs" = { };
"subvolumes/home" = { };
"subvolumes/home/user" = { };
"subvolumes/nix" = { };
"subvolumes/test" = { };
};
};
# # Example showing mounted subvolumes in a multi-disk configuration (not yet working).
# mounted_subvolumes_in_multi = {
# type = "bcachefs_filesystem";
# passwordFile = "/tmp/secret.key";
# extraFormatArgs = [
# "--compression=lz4"
# "--background_compression=lz4"
# ];
# mountOptions = [
# "verbose"
# ];
# subvolumes = {
# # Subvolume name is different from mountpoint
# "foo" = {
# mountpoint = "/bar";
# };
# # Subvolume name is the same as the mountpoint
# "home" = {
# mountpoint = "/home";
# };
# # Sub(sub)volume doesn't need a mountpoint as its parent is mounted
# "home/user" = {
# };
# # Parent is not mounted so the mountpoint must be set
# "nix" = {
# mountpoint = "/nix";
# };
# # This subvolume will be created but not mounted
# "test" = {
# };
# };
# };
# Example showing another bcachefs filesystem.
no_reliance_on_external_subvolume = {
type = "bcachefs_filesystem";
mountpoint = "/sometestdir";
};
# # Example showing another bcachefs filesystem that relies on a subvolume
# # in another filesystem being mounted (not yet working).
# relies_on_external_subvolume = {
# type = "bcachefs_filesystem";
# mountpoint = "/home/somedir/vdf1";
# };
};
};
}

View File

@@ -44,6 +44,7 @@ let
# option for valid contents of partitions (basically like devices, but without tables)
_partitionTypes = {
inherit (diskoLib.types)
bcachefs
btrfs
filesystem
zfs
@@ -69,6 +70,7 @@ let
# option for valid contents of devices
_deviceTypes = {
inherit (diskoLib.types)
bcachefs
table
gpt
btrfs
@@ -611,6 +613,7 @@ let
let
devices = {
inherit (cfg.config)
bcachefs_filesystems
disk
mdadm
zpool
@@ -621,6 +624,11 @@ let
in
{
options = {
bcachefs_filesystems = lib.mkOption {
type = lib.types.attrsOf diskoLib.types.bcachefs_filesystem;
default = { };
description = "bcachefs filesystem";
};
disk = lib.mkOption {
type = lib.types.attrsOf diskoLib.types.disk;
default = { };
@@ -687,6 +695,7 @@ let
throw "No disks defined, did you forget to import your disko config?"
else
v;
# @todo Do we need to add bcachefs-tools or not?
destroyDependencies = with pkgs; [
util-linux
e2fsprogs

100
lib/types/bcachefs.nix Normal file
View File

@@ -0,0 +1,100 @@
{
config,
device,
diskoLib,
lib,
options,
parent,
...
}:
{
options = {
type = lib.mkOption {
type = lib.types.enum [ "bcachefs" ];
internal = true;
description = "Type.";
};
device = lib.mkOption {
type = lib.types.str;
default = device;
description = "Device to use.";
example = "/dev/sda";
};
filesystem = lib.mkOption {
type = lib.types.str;
description = "Name of the bcachefs filesystem this partition belongs to.";
example = "main_bcachefs_filesystem";
};
# These are passed as arguments to the device corresponding to this one in the invocation of the `bcachefs format` command
# in the bcachefs_filesystem type defined in bcachefs_filesystem.nix used to format the bcachefs filesystem that this device is a part of.
extraFormatArgs = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = [ ];
description = "Extra arguments passed to the bcachefs format command.";
example = [ "--discard" ];
};
# This value is passed to the `--label` option for the device corresponding to this one in the invocation of the `bcachefs format` command
# in the bcachefs_filesystem type defined in bcachefs_filesystem.nix used to format the bcachefs filesystem that this device is a part of.
label = lib.mkOption {
type = lib.types.str;
default = "";
description = ''
Label to use for this device.
This value is passed as the `--label` argument to the `bcachefs format` command when formatting the device.
'';
example = "group_a.sda2";
};
_parent = lib.mkOption {
internal = true;
default = parent;
};
_meta = lib.mkOption {
internal = true;
readOnly = true;
type = lib.types.functionTo diskoLib.jsonType;
# Ensures that this file's `_create` will be ran for all member devices that are part of the filesystem being created,
# before the `_create` in bcachefs_filesystem.nix is ran.
default = dev: {
deviceDependencies.bcachefs_filesystems.${config.filesystem} = [ dev ];
};
};
_create = diskoLib.mkCreateOption {
inherit config options;
# The bcachefs_filesystem type defined in bcachefs_filesystem.nix will include this device when formatting and mounting the filesystem.
# The current file should not run the `bcachefs format` command. Instead, the`bcachefs format` command will be ran
# in the `_create` attribute in bcachefs_filesystem.nix, once it has collected and generated the arguments specifying the devices that should be part of the filesystem.
default = ''
# Write device arguments to temporary directory for bcachefs_filesystem.
{
printf '%s\n' '--label="${config.label}"';
${lib.concatMapStrings (args: ''printf '%s\n' '${args}';'') config.extraFormatArgs}
printf '%s\n' '${config.device}';
} >> "$disko_devices_dir/bcachefs-${lib.escapeShellArg config.filesystem}";
'';
};
_mount = diskoLib.mkMountOption {
inherit config options;
# Empty, since mounting will be handled by the bcachefs_filesystem type defined in bcachefs_filesystem.nix.
default = { };
};
_unmount = diskoLib.mkUnmountOption {
inherit config options;
# Empty, since unmounting will be handled by the bcachefs_filesystem type defined in bcachefs_filesystem.nix.
default = { };
};
_config = lib.mkOption {
internal = true;
readOnly = true;
# Empty, since NixOS configuration will be handled by the bcachefs_filesystem type defined in bcachefs_filesystem.nix.
default = { };
description = "NixOS configuration.";
};
_pkgs = lib.mkOption {
internal = true;
readOnly = true;
type = lib.types.functionTo (lib.types.listOf lib.types.package);
default = pkgs: [ ];
description = "Packages.";
};
};
}

View File

@@ -0,0 +1,331 @@
{
config,
diskoLib,
lib,
options,
parent,
rootMountPoint,
...
}:
{
options = {
name = lib.mkOption {
type = lib.types.str;
default = config._module.args.name;
description = "Name of the bcachefs filesystem.";
example = "main_bcachefs_filesystem";
};
type = lib.mkOption {
type = lib.types.enum [ "bcachefs_filesystem" ];
internal = true;
description = "Type.";
};
extraFormatArgs = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = [ ];
description = "Extra arguments passed to the `bcachefs format` command.";
example = [
"--compression=lz4"
"--background_compression=lz4"
];
};
mountOptions = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = [ "X-mount.mkdir" ];
description = ''
Options to pass to mount.
The "X-mount.mkdir" option is always automatically added.
'';
example = [
"noatime"
"verbose"
];
};
mountpoint = lib.mkOption {
type = lib.types.nullOr diskoLib.optionTypes.absolute-pathname;
default = null;
description = "Path to mount the bcachefs filesystem to.";
example = "/";
};
uuid = lib.mkOption {
type = lib.types.strMatching "[[:xdigit:]]{8}-[[:xdigit:]]{4}-[[:xdigit:]]{4}-[[:xdigit:]]{4}-[[:xdigit:]]{12}";
default =
let
# Generate a deterministic but random-looking UUID based on the filesystem name
# This avoids the need for impure access to nixpkgs at evaluation time
hash = builtins.hashString "sha256" "${config.name}";
hexChars = builtins.substring 0 32 hash;
p1 = builtins.substring 0 8 hexChars;
p2 = builtins.substring 8 4 hexChars;
p3 = builtins.substring 12 4 hexChars;
p4 = builtins.substring 16 4 hexChars;
p5 = builtins.substring 20 12 hexChars;
in
"${p1}-${p2}-${p3}-${p4}-${p5}";
defaultText = "generated deterministically based on filesystem name";
example = "809b3a2b-828a-4730-95e1-75b6343e415a";
description = ''
The UUID of the bcachefs filesystem.
If not provided, a deterministic UUID will be generated based on the filesystem name.
'';
};
passwordFile = lib.mkOption {
type = lib.types.nullOr diskoLib.optionTypes.absolute-pathname;
default = null;
description = ''
Path to the file containing the password for encryption.
Setting this option will automatically cause the `--encrypted` option to be passed to `bcachefs format` and cause the filesystem to have encryption enabled.
'';
example = "/tmp/disk.key";
};
subvolumes = lib.mkOption {
type = lib.types.attrsOf (
lib.types.submodule (
{ config, ... }:
{
options = {
name = lib.mkOption {
type = lib.types.str;
default = config._module.args.name;
description = ''
Path of the subvolume within the filesystem.
Leading forward slashes are automatically removed.
'';
example = "subvolumes/home";
};
type = lib.mkOption {
type = lib.types.enum [ "bcachefs_subvolume" ];
default = "bcachefs_subvolume";
internal = true;
description = "Type.";
};
mountOptions = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = lib.naturalSort [
"X-mount.mkdir"
"X-mount.subdir=${lib.removePrefix "/" config.name}"
];
description = ''
Options to pass to mount.
The "X-mount.mkdir" and "X-mount.subdir" options are always automatically added.
'';
};
mountpoint = lib.mkOption {
type = lib.types.nullOr diskoLib.optionTypes.absolute-pathname;
default = null;
description = ''
Path to mount the subvolume to.
DO NOT USE. Currently not working.
'';
example = "/";
};
};
}
)
);
default = { };
description = "List of subvolumes to define.";
example = {
"subvolumes/home" = { };
};
};
_parent = lib.mkOption {
internal = true;
default = parent;
};
_meta = lib.mkOption {
internal = true;
readOnly = true;
type = lib.types.functionTo diskoLib.jsonType;
default = dev: { };
description = "Metadata";
};
_create = diskoLib.mkCreateOption {
inherit config options;
# This sets a string variable containing arguments to be passed to the `bcachefs format` command.
# This string will consist of `--label` and other arguments that correspond to the values of the `label` and `extraFormatArgs` attributes, respectively,
# from each of the bcachefs devices in this filesystem specified in the configuration.
# Then, it sets the `default` attribute to a string containing shell commands that calls the `bcachefs format` command, passing in the arguments generated, as well as a `--uuid` value.
default = ''
if ! test -s "$disko_devices_dir/bcachefs-${config.name}"; then
printf "\033[31mERROR:\033[0m No devices found for bcachefs filesystem \"${config.name}\"!\nDid you forget to add some or misspell the filesystem name?\n" >&2;
exit 1;
fi;
# Create the filesystem.
(
# Empty out $@.
set --;
# Collect devices and arguments to $@.
while IFS= read -r line; do
# Append current line as a new positional parameter
set -- "$@" "$line";
done < "$disko_devices_dir/bcachefs-${config.name}";
# Format the filesystem with all devices and arguments.
if ! blkid -o export "$(blkid -lU ${config.uuid})" | grep -q 'TYPE=bcachefs' >&2 2>&1; then
bcachefs format \
"$@" \
--uuid="${config.uuid}" \
${lib.concatStringsSep " \\\n" config.extraFormatArgs} \
${
lib.optionalString (config.passwordFile != null) ''--encrypted < "${config.passwordFile}"''
};
fi;
);
# Mount the bcachefs filesystem onto a temporary directory,
# then, create the subvolumes from inside of that directory.
${lib.optionalString (config.subvolumes != { }) ''
if blkid -o export "$(blkid -lU ${config.uuid})" | grep -q 'TYPE=bcachefs' >&2 2>&1; then
${lib.concatMapStrings (subvolume: ''
(
TEMPDIR="$(mktemp -d)";
MNTPOINT="$(mktemp -d)";
${lib.optionalString (
config.passwordFile != null
) ''bcachefs unlock -k session "/dev/disk/by-uuid/${config.uuid}" < "${config.passwordFile}";''}
bcachefs mount \
-o "${lib.concatStringsSep "," (lib.unique ([ "X-mount.mkdir" ] ++ config.mountOptions))}" \
UUID="${config.uuid}" \
"$MNTPOINT";
trap 'umount "$MNTPOINT"; rm -rf "$MNTPOINT"; rm -rf "$TEMPDIR";' EXIT;
SUBVOL_ABS_PATH="$MNTPOINT/${subvolume.name}";
# Check if it's already a subvolume (using snapshot).
if ! bcachefs subvolume snapshot "$SUBVOL_ABS_PATH" "$TEMPDIR/" >&2 2>&1; then
# It's not a subvolume, now check if it's a directory.
if ! test -d "$SUBVOL_ABS_PATH"; then
# It's not a subvolume AND not a directory, so create it.
mkdir -p -- "$(dirname -- "$SUBVOL_ABS_PATH")";
bcachefs subvolume create "$SUBVOL_ABS_PATH";
fi
fi;
)
'') (lib.attrValues config.subvolumes)}
fi;
''}
'';
};
_mount = diskoLib.mkMountOption {
inherit config options;
default =
let
subvolumeMounts = diskoLib.deepMergeMap (
subvolume:
lib.optionalAttrs (subvolume.mountpoint != null) {
${subvolume.mountpoint} = ''
if ! findmnt "${rootMountPoint}${subvolume.mountpoint}" >&2 2>&1; then
# @todo Figure out why the "X-mount.mkdir" option here doesn't seem to work,
# necessitating running `mkdir` here.
mkdir -p "${rootMountPoint}${subvolume.mountpoint}";
${lib.optionalString (
config.passwordFile != null
) ''bcachefs unlock -k session "/dev/disk/by-uuid/${config.uuid}" < "${config.passwordFile}";''}
bcachefs mount \
-o "${
lib.concatStringsSep "," (
lib.unique (
[
"X-mount.mkdir"
"X-mount.subdir=${lib.removePrefix "/" subvolume.name}"
]
++ subvolume.mountOptions
)
)
}" \
UUID="${config.uuid}" \
"${rootMountPoint}${subvolume.mountpoint}";
fi;
'';
}
) (lib.attrValues config.subvolumes);
in
{
fs =
subvolumeMounts
// lib.optionalAttrs (config.mountpoint != null) {
${config.mountpoint} = ''
if ! findmnt "${rootMountPoint}${config.mountpoint}" >&2 2>&1; then
# @todo Figure out why the "X-mount.mkdir" option here doesn't seem to work,
# necessitating running `mkdir` here.
mkdir -p "${rootMountPoint}${config.mountpoint}";
${lib.optionalString (
config.passwordFile != null
) ''bcachefs unlock -k session "/dev/disk/by-uuid/${config.uuid}" < "${config.passwordFile}";''}
bcachefs mount \
-o "${lib.concatStringsSep "," (lib.unique ([ "X-mount.mkdir" ] ++ config.mountOptions))}" \
UUID="${config.uuid}" \
"${rootMountPoint}${config.mountpoint}";
fi;
'';
};
};
};
_unmount = diskoLib.mkUnmountOption {
inherit config options;
default =
let
subvolumeMounts = lib.concatMapAttrs (
_: subvolume:
lib.optionalAttrs (subvolume.mountpoint != null) {
${subvolume.mountpoint} = ''
if findmnt "UUID=${config.uuid}" "${rootMountPoint}${subvolume.mountpoint}" >&2 2>&1; then
umount "${rootMountPoint}${subvolume.mountpoint}";
fi;
'';
}
) config.subvolumes;
in
{
fs =
subvolumeMounts
// lib.optionalAttrs (config.mountpoint != null) {
${config.mountpoint} = ''
if findmnt "UUID=${config.uuid}" "${rootMountPoint}${config.mountpoint}" >&2 2>&1; then
umount "${rootMountPoint}${config.mountpoint}";
fi;
'';
};
};
};
_config = lib.mkOption {
internal = true;
readOnly = true;
# @todo Check that this implementation is correct:
default =
(lib.optional (config.mountpoint != null) {
fileSystems.${config.mountpoint} = {
device = "UUID=${config.uuid}";
fsType = "bcachefs";
options = lib.unique ([ "X-mount.mkdir" ] ++ config.mountOptions);
neededForBoot = true;
};
})
++ (map (subvolume: {
fileSystems.${subvolume.mountpoint} = {
device = "UUID=${config.uuid}";
fsType = "bcachefs";
options = lib.unique (
[
"X-mount.mkdir"
"X-mount.subdir=${lib.removePrefix "/" subvolume.name}"
]
++ subvolume.mountOptions
);
neededForBoot = true;
};
}) (lib.filter (subvolume: subvolume.mountpoint != null) (lib.attrValues config.subvolumes)));
description = "NixOS configuration.";
};
_pkgs = lib.mkOption {
internal = true;
readOnly = true;
type = lib.types.functionTo (lib.types.listOf lib.types.package);
default = pkgs: [
pkgs.bcachefs-tools
pkgs.util-linux
];
description = "Packages.";
};
};
}

View File

@@ -55,6 +55,8 @@
fs-type = lib.mkOption {
type = lib.types.nullOr (
lib.types.enum [
# @todo Check if this is needed
"bcachefs"
"btrfs"
"ext2"
"ext3"

View File

@@ -6,12 +6,40 @@ diskoLib.testLib.makeDiskoTest {
inherit pkgs;
name = "bcachefs";
disko-config = ../example/bcachefs.nix;
enableOCR = true;
extraTestScript = ''
machine.succeed("mountpoint /");
# @todo Verify all devices are part of the filesystem.
# @todo Check device labels and group assignments.
# Verify mount options were applied.
machine.succeed("mount | grep ' / ' | grep -q 'compression=lz4'");
machine.succeed("mount | grep ' / ' | grep -q 'background_compression=lz4'");
# @todo Verify mountpoint dependency order was respected.
# @todo Add tests for subvolumes.
# Print debug information.
machine.succeed("lsblk >&2");
machine.succeed("lsblk -f >&2");
machine.succeed("mount >&2");
'';
# extraSystemConfig = { pkgs, ... }: {
# # @todo Do we need to add any attributes here?
# boot = {
# supportedFilesystems = [ "bcachefs" ];
# initrd = {
# supportedFilesystems = [ "bcachefs" ];
# # systemd.enable = false;
# };
# };
# environment.systemPackages = [
# pkgs.bcachefs-tools
# pkgs.util-linux
# ];
# };
# extraInstallerConfig = {
# # @todo Do we need to add any attributes here?
# };
bootCommands = ''
machine.wait_for_text("enter passphrase for");
machine.send_chars("secretsecret\n");
'';
# so that the installer boots with a bcachefs enabled kernel
extraInstallerConfig = {
boot.supportedFilesystems = [ "bcachefs" ];
};
}