colin/nixpkgs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #4807 from wizeman/u/zfs-improvements

Browse Source 
ZFS improvements
This commit is contained in:
wmertens 2014-11-13 10:26:30 +01:00
commit 5c19521c6e
10 changed files with 296 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
lib/lists.nix
View File

@ -223,4 +223,14 @@ rec {
crossLists = f: foldl (fs: args: concatMap (f: map f args) fs) [f];
# Remove duplicate elements from the list
unique = list:
if list == [] then
[]
else
let
x = head list;
xs = unique (drop 1 list);
in [x] ++ remove x xs;
}

3
nixos/modules/installer/cd-dvd/installation-cd-base.nix
View File

@ -45,6 +45,9 @@ with lib;
# Add support for cow filesystems and their utilities
boot.supportedFilesystems = [ /* "zfs" */ "btrfs" ];
# Configure host id for ZFS to work
networking.hostId = "8425e349";
# Allow the user to log in as root without a password.
users.extraUsers.root.initialHashedPassword = "";
}

9
nixos/modules/installer/tools/nixos-generate-config.pl
View File

@ -476,6 +476,14 @@ EOF
EOF
}
# Generate a random 32-bit value to use as the host id
open my $rnd, "<", "/dev/urandom" or die $!;
read $rnd, $hostIdBin, 4;
close $rnd;
# Convert the 32-bit value to a hex string
my $hostIdHex = unpack("H*", $hostIdBin);
write_file($fn, <<EOF);
# Edit this configuration file to define what should be installed on
# your system. Help is available in the configuration.nix(5) man page
@ -491,6 +499,7 @@ EOF
$bootLoaderConfig
# networking.hostName = "nixos"; # Define your hostname.
networking.hostId = "$hostIdHex";
# networking.wireless.enable = true; # Enables wireless.
# Select internationalisation properties.

3
nixos/modules/system/boot/stage-1-init.sh
View File

@ -122,6 +122,9 @@ for o in $(cat /proc/cmdline); do
esac
done
# Set hostid before modules are loaded.
# This is needed by the spl/zfs modules.
@setHostId@
# Load the required kernel modules.
mkdir -p /lib

9
nixos/modules/system/boot/stage-1.nix
View File

@ -188,6 +188,15 @@ let
fsInfo =
let f = fs: [ fs.mountPoint (if fs.device != null then fs.device else "/dev/disk/by-label/${fs.label}") fs.fsType fs.options ];
in pkgs.writeText "initrd-fsinfo" (concatStringsSep "\n" (concatMap f fileSystems));
setHostId = optionalString (config.networking.hostId != null) ''
hi="${config.networking.hostId}"
${if pkgs.stdenv.isBigEndian then ''
echo -ne "\x''${hi:0:2}\x''${hi:2:2}\x''${hi:4:2}\x''${hi:6:2}" > /etc/hostid
'' else ''
echo -ne "\x''${hi:6:2}\x''${hi:4:2}\x''${hi:2:2}\x''${hi:0:2}" > /etc/hostid
''}
'';
};

220
nixos/modules/tasks/filesystems/zfs.nix
View File

@ -1,11 +1,10 @@
{ config, lib, pkgs, ... }:
{ config, lib, pkgs, utils, ... }:
#
# todo:
# - crontab for scrubs, etc
# - zfs tunables
# - /etc/zfs/zpool.cache handling
with utils;
with lib;
let
@ -31,6 +30,20 @@ let
zfsAutoSnap = "${autosnapPkg}/bin/zfs-auto-snapshot";
datasetToPool = x: elemAt (splitString "/" x) 0;
fsToPool = fs: datasetToPool fs.device;
zfsFilesystems = filter (x: x.fsType == "zfs") (attrValues config.fileSystems);
isRoot = fs: fs.neededForBoot || elem fs.mountPoint [ "/" "/nix" "/nix/store" "/var" "/var/log" "/var/lib" "/etc" ];
allPools = unique ((map fsToPool zfsFilesystems) ++ cfgZfs.extraPools);
rootPools = unique (map fsToPool (filter isRoot zfsFilesystems));
dataPools = unique (filter (pool: !(elem pool rootPools)) allPools);
in
{
@ -38,28 +51,73 @@ in
###### interface
options = {
boot.spl.hostid = mkOption {
default = "";
example = "0xdeadbeef";
description = ''
ZFS uses a system's hostid to determine if a storage pool (zpool) is
native to this system, and should thus be imported automatically.
Unfortunately, this hostid can change under linux from boot to boot (by
changing network adapters, for instance). Specify a unique 32 bit hostid in
hex here for zfs to prevent getting a random hostid between boots and having to
manually import pools.
'';
};
boot.zfs = {
useGit = mkOption {
type = types.bool;
default = false;
example = true;
description = ''
Use the git version of the SPL and ZFS packages.
Note that these are unreleased versions, with less testing, and therefore
may be more unstable.
'';
};
boot.zfs.useGit = mkOption {
type = types.bool;
default = false;
example = true;
description = ''
Use the git version of the SPL and ZFS packages.
Note that these are unreleased versions, with less testing, and therefore
may be more unstable.
'';
extraPools = mkOption {
type = types.listOf types.str;
default = [];
example = [ "tank" "data" ];
description = ''
Name or GUID of extra ZFS pools that you wish to import during boot.
Usually this is not necessary. Instead, you should set the mountpoint property
of ZFS filesystems to <literal>legacy</literal> and add the ZFS filesystems to
NixOS's <option>fileSystems</option> option, which makes NixOS automatically
import the associated pool.
However, in some cases (e.g. if you have many filesystems) it may be preferable
to exclusively use ZFS commands to manage filesystems. If so, since NixOS/systemd
will not be managing those filesystems, you will need to specify the ZFS pool here
so that NixOS automatically imports it on every boot.
'';
};
forceImportRoot = mkOption {
type = types.bool;
default = true;
example = false;
description = ''
Forcibly import the ZFS root pool(s) during early boot.
This is enabled by default for backwards compatibility purposes, but it is highly
recommended to disable this option, as it bypasses some of the safeguards ZFS uses
to protect your ZFS pools.
If you set this option to <literal>false</literal> and NixOS subsequently fails to
boot because it cannot import the root pool, you should boot with the
<literal>zfs_force=1</literal> option as a kernel parameter (e.g. by manually
editing the kernel params in grub during boot). You should only need to do this
once.
'';
};
forceImportAll = mkOption {
type = types.bool;
default = true;
example = false;
description = ''
Forcibly import all ZFS pool(s).
This is enabled by default for backwards compatibility purposes, but it is highly
recommended to disable this option, as it bypasses some of the safeguards ZFS uses
to protect your ZFS pools.
If you set this option to <literal>false</literal> and NixOS subsequently fails to
import your non-root ZFS pool(s), you should manually import each pool with
"zpool import -f &lt;pool-name&gt;", and then reboot. You should only need to do
this once.
'';
};
};
services.zfs.autoSnapshot = {
@ -124,12 +182,20 @@ in
config = mkMerge [
(mkIf enableZfs {
assertions = [
{
assertion = config.networking.hostId != null;
message = "ZFS requires config.networking.hostId to be set";
}
{
assertion = !cfgZfs.forceImportAll || cfgZfs.forceImportRoot;
message = "If you enable boot.zfs.forceImportAll, you must also enable boot.zfs.forceImportRoot";
}
];
boot = {
kernelModules = [ "spl" "zfs" ] ;
extraModulePackages = [ splPkg zfsPkg ];
extraModprobeConfig = mkIf (cfgSpl.hostid != "") ''
options spl spl_hostid=${cfgSpl.hostid}
'';
};
boot.initrd = mkIf inInitrd {
@ -142,50 +208,84 @@ in
cp -pdv ${zfsPkg}/lib/lib*.so* $out/lib
cp -pdv ${pkgs.zlib}/lib/lib*.so* $out/lib
'';
postDeviceCommands =
''
zpool import -f -a
'';
postDeviceCommands = concatStringsSep "\n" ([''
ZFS_FORCE="${optionalString cfgZfs.forceImportRoot "-f"}"
for o in $(cat /proc/cmdline); do
case $o in
zfs_force|zfs_force=1)
ZFS_FORCE="-f"
;;
esac
done
''] ++ (map (pool: ''
echo "importing root ZFS pool \"${pool}\"..."
zpool import -N $ZFS_FORCE "${pool}"
'') rootPools));
};
boot.loader.grub = mkIf inInitrd {
zfsSupport = true;
};
systemd.services."zpool-import" = {
description = "Import zpools";
after = [ "systemd-udev-settle.service" ];
wantedBy = [ "local-fs.target" ];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
ExecStart = "${zfsPkg}/sbin/zpool import -f -a";
};
restartIfChanged = false;
};
systemd.services."zfs-mount" = {
description = "Mount ZFS Volumes";
after = [ "zpool-import.service" ];
wantedBy = [ "local-fs.target" ];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
ExecStart = "${zfsPkg}/sbin/zfs mount -a";
ExecStop = "${zfsPkg}/sbin/zfs umount -a";
};
restartIfChanged = false;
};
environment.etc."zfs/zed.d".source = "${zfsPkg}/etc/zfs/zed.d/*";
system.fsPackages = [ zfsPkg ]; # XXX: needed? zfs doesn't have (need) a fsck
environment.systemPackages = [ zfsPkg ];
services.udev.packages = [ zfsPkg ]; # to hook zvol naming, etc.
systemd.packages = [ zfsPkg ];
systemd.services = let
getPoolFilesystems = pool:
filter (x: x.fsType == "zfs" && (fsToPool x) == pool) (attrValues config.fileSystems);
getPoolMounts = pool:
let
mountPoint = fs: escapeSystemdPath fs.mountPoint;
in
map (x: "${mountPoint x}.mount") (getPoolFilesystems pool);
createImportService = pool:
nameValuePair "zfs-import-${pool}" {
description = "Import ZFS pool \"${pool}\"";
requires = [ "systemd-udev-settle.service" ];
after = [ "systemd-udev-settle.service" "systemd-modules-load.service" ];
wantedBy = (getPoolMounts pool) ++ [ "local-fs.target" ];
before = (getPoolMounts pool) ++ [ "local-fs.target" ];
unitConfig = {
DefaultDependencies = "no";
};
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
};
script = ''
zpool_cmd="${zfsPkg}/sbin/zpool"
("$zpool_cmd" list "${pool}" >/dev/null) || "$zpool_cmd" import -N ${optionalString cfgZfs.forceImportAll "-f"} "${pool}"
'';
};
in listToAttrs (map createImportService dataPools) // {
"zfs-mount" = { after = [ "systemd-modules-load.service" ]; };
"zfs-share" = { after = [ "systemd-modules-load.service" ]; };
"zed" = { after = [ "systemd-modules-load.service" ]; };
};
systemd.targets."zfs-import" =
let
services = map (pool: "zfs-import-${pool}.service") dataPools;
in
{
requires = services;
after = services;
};
systemd.targets."zfs".wantedBy = [ "multi-user.target" ];
})
(mkIf enableAutoSnapshots {
systemd.services."zfs-snapshot-frequent" = {
description = "ZFS auto-snapshotting every 15 mins";
after = [ "zpool-import.service" ];
after = [ "zfs-import.target" ];
serviceConfig = {
Type = "oneshot";
ExecStart = "${zfsAutoSnap} frequent ${toString cfgSnapshots.frequent}";
@ -196,7 +296,7 @@ in
systemd.services."zfs-snapshot-hourly" = {
description = "ZFS auto-snapshotting every hour";
after = [ "zpool-import.service" ];
after = [ "zfs-import.target" ];
serviceConfig = {
Type = "oneshot";
ExecStart = "${zfsAutoSnap} hourly ${toString cfgSnapshots.hourly}";
@ -207,7 +307,7 @@ in
systemd.services."zfs-snapshot-daily" = {
description = "ZFS auto-snapshotting every day";
after = [ "zpool-import.service" ];
after = [ "zfs-import.target" ];
serviceConfig = {
Type = "oneshot";
ExecStart = "${zfsAutoSnap} daily ${toString cfgSnapshots.daily}";
@ -218,7 +318,7 @@ in
systemd.services."zfs-snapshot-weekly" = {
description = "ZFS auto-snapshotting every week";
after = [ "zpool-import.service" ];
after = [ "zfs-import.target" ];
serviceConfig = {
Type = "oneshot";
ExecStart = "${zfsAutoSnap} weekly ${toString cfgSnapshots.weekly}";
@ -229,7 +329,7 @@ in
systemd.services."zfs-snapshot-monthly" = {
description = "ZFS auto-snapshotting every month";
after = [ "zpool-import.service" ];
after = [ "zfs-import.target" ];
serviceConfig = {
Type = "oneshot";
ExecStart = "${zfsAutoSnap} monthly ${toString cfgSnapshots.monthly}";

50
nixos/modules/tasks/network-interfaces.nix
View File

@ -189,6 +189,10 @@ let
};
hexChars = stringToCharacters "0123456789abcdef";
isHexString = s: all (c: elem c hexChars) (stringToCharacters (toLower s));
in
{
@ -205,6 +209,20 @@ in
'';
};
networking.hostId = mkOption {
default = null;
example = "4e98920d";
type = types.nullOr types.str;
description = ''
The 32-bit host ID of the machine, formatted as 8 hexadecimal characters.
You should try to make this ID unique among your machines. You can
generate a random 32-bit ID using the following command:
<literal>head -c4 /dev/urandom | od -A none -t x4</literal>
'';
};
networking.enableIPv6 = mkOption {
default = true;
description = ''
@ -513,10 +531,15 @@ in
config = {
assertions =
flip map interfaces (i: {
(flip map interfaces (i: {
assertion = i.subnetMask == null;
message = "The networking.interfaces.${i.name}.subnetMask option is defunct. Use prefixLength instead.";
});
})) ++ [
{
assertion = cfg.hostId == null || (stringLength cfg.hostId == 8 && isHexString cfg.hostId);
message = "Invalid value given to the networking.hostId option.";
}
];
boot.kernelModules = [ ]
++ optional cfg.enableIPv6 "ipv6"
@ -872,14 +895,29 @@ in
# clear it if it's not configured in the NixOS configuration,
# since it may have been set by dhcpcd in the meantime.
system.activationScripts.hostname =
optionalString (config.networking.hostName != "") ''
hostname "${config.networking.hostName}"
optionalString (cfg.hostName != "") ''
hostname "${cfg.hostName}"
'';
system.activationScripts.domain =
optionalString (config.networking.domain != "") ''
domainname "${config.networking.domain}"
optionalString (cfg.domain != "") ''
domainname "${cfg.domain}"
'';
environment.etc = mkIf (cfg.hostId != null)
[
{
target = "hostid";
source = pkgs.runCommand "gen-hostid" {} ''
hi="${cfg.hostId}"
${if pkgs.stdenv.isBigEndian then ''
echo -ne "\x''${hi:0:2}\x''${hi:2:2}\x''${hi:4:2}\x''${hi:6:2}" > $out
'' else ''
echo -ne "\x''${hi:6:2}\x''${hi:4:2}\x''${hi:2:2}\x''${hi:0:2}" > $out
''}
'';
}
];
services.udev.extraRules =
''
KERNEL=="tun", TAG+="systemd"

39
pkgs/os-specific/linux/zfs/default.nix
View File

@ -20,34 +20,53 @@ stdenv.mkDerivation {
NIX_CFLAGS_LINK = "-lgcc_s";
preConfigure = ''
./autogen.sh
substituteInPlace ./module/zfs/zfs_ctldir.c --replace "umount -t zfs" "${utillinux}/bin/umount -t zfs"
substituteInPlace ./module/zfs/zfs_ctldir.c --replace "mount -t zfs" "${utillinux}/bin/mount -t zfs"
substituteInPlace ./lib/libzfs/libzfs_mount.c --replace "/bin/umount" "${utillinux}/bin/umount"
substituteInPlace ./lib/libzfs/libzfs_mount.c --replace "/bin/mount" "${utillinux}/bin/mount"
substituteInPlace ./udev/rules.d/* --replace "/lib/udev/vdev_id" "$out/lib/udev/vdev_id"
substituteInPlace ./cmd/ztest/ztest.c --replace "/usr/sbin/ztest" "$out/sbin/ztest"
substituteInPlace ./cmd/ztest/ztest.c --replace "/usr/sbin/zdb" "$out/sbin/zdb"
substituteInPlace ./config/user-systemd.m4 --replace "/usr/lib/modules-load.d" "$out/etc/modules-load.d"
substituteInPlace ./config/zfs-build.m4 --replace "\$sysconfdir/init.d" "$out/etc/init.d"
substituteInPlace ./etc/zfs/Makefile.am --replace "\$(sysconfdir)" "$out/etc"
substituteInPlace ./cmd/zed/Makefile.am --replace "\$(sysconfdir)" "$out/etc"
substituteInPlace ./module/zfs/zfs_ctldir.c --replace "umount -t zfs" "${utillinux}/bin/umount -t zfs"
substituteInPlace ./module/zfs/zfs_ctldir.c --replace "mount -t zfs" "${utillinux}/bin/mount -t zfs"
substituteInPlace ./lib/libzfs/libzfs_mount.c --replace "/bin/umount" "${utillinux}/bin/umount"
substituteInPlace ./lib/libzfs/libzfs_mount.c --replace "/bin/mount" "${utillinux}/bin/mount"
substituteInPlace ./udev/rules.d/* --replace "/lib/udev/vdev_id" "$out/lib/udev/vdev_id"
substituteInPlace ./cmd/ztest/ztest.c --replace "/usr/sbin/ztest" "$out/sbin/ztest"
substituteInPlace ./cmd/ztest/ztest.c --replace "/usr/sbin/zdb" "$out/sbin/zdb"
./autogen.sh
'';
configureFlags = [
"--disable-systemd"
"--enable-systemd"
"--with-linux=${kernel.dev}/lib/modules/${kernel.modDirVersion}/source"
"--with-linux-obj=${kernel.dev}/lib/modules/${kernel.modDirVersion}/build"
"--with-spl=${spl}/libexec/spl"
"--with-dracutdir=$(out)/lib/dracut"
"--with-udevdir=$(out)/lib/udev"
"--with-systemdunitdir=$(out)/etc/systemd/system"
"--with-systemdpresetdir=$(out)/etc/systemd/system-preset"
"--sysconfdir=/etc"
"--localstatedir=/var"
];
enableParallelBuilding = true;
# Remove provided services as they are buggy
postInstall = ''
rm $out/etc/systemd/system/zfs-import-*.service
sed -i '/zfs-import-scan.service/d' $out/etc/systemd/system/*
for i in $out/etc/systemd/system/*; do
substituteInPlace $i --replace "zfs-import-cache.service" "zfs-import.target"
done
'';
meta = {
description = "ZFS Filesystem Linux Kernel module";
longDescription = ''
ZFS is a filesystem that combines a logical volume manager with a
Copy-On-Write filesystem with data integrity detection and repair,
snapshotting, cloning, block devices, deduplication, and more.
snapshotting, cloning, block devices, deduplication, and more.
'';
homepage = http://zfsonlinux.org/;
license = stdenv.lib.licenses.cddl;

37
pkgs/os-specific/linux/zfs/git.nix
View File

@ -17,28 +17,47 @@ stdenv.mkDerivation {
NIX_CFLAGS_LINK = "-lgcc_s";
preConfigure = ''
./autogen.sh
substituteInPlace ./module/zfs/zfs_ctldir.c --replace "umount -t zfs" "${utillinux}/bin/umount -t zfs"
substituteInPlace ./module/zfs/zfs_ctldir.c --replace "mount -t zfs" "${utillinux}/bin/mount -t zfs"
substituteInPlace ./lib/libzfs/libzfs_mount.c --replace "/bin/umount" "${utillinux}/bin/umount"
substituteInPlace ./lib/libzfs/libzfs_mount.c --replace "/bin/mount" "${utillinux}/bin/mount"
substituteInPlace ./udev/rules.d/* --replace "/lib/udev/vdev_id" "$out/lib/udev/vdev_id"
substituteInPlace ./cmd/ztest/ztest.c --replace "/usr/sbin/ztest" "$out/sbin/ztest"
substituteInPlace ./cmd/ztest/ztest.c --replace "/usr/sbin/zdb" "$out/sbin/zdb"
substituteInPlace ./config/user-systemd.m4 --replace "/usr/lib/modules-load.d" "$out/etc/modules-load.d"
substituteInPlace ./config/zfs-build.m4 --replace "\$sysconfdir/init.d" "$out/etc/init.d"
substituteInPlace ./etc/zfs/Makefile.am --replace "\$(sysconfdir)" "$out/etc"
substituteInPlace ./cmd/zed/Makefile.am --replace "\$(sysconfdir)" "$out/etc"
substituteInPlace ./module/zfs/zfs_ctldir.c --replace "umount -t zfs" "${utillinux}/bin/umount -t zfs"
substituteInPlace ./module/zfs/zfs_ctldir.c --replace "mount -t zfs" "${utillinux}/bin/mount -t zfs"
substituteInPlace ./lib/libzfs/libzfs_mount.c --replace "/bin/umount" "${utillinux}/bin/umount"
substituteInPlace ./lib/libzfs/libzfs_mount.c --replace "/bin/mount" "${utillinux}/bin/mount"
substituteInPlace ./udev/rules.d/* --replace "/lib/udev/vdev_id" "$out/lib/udev/vdev_id"
substituteInPlace ./cmd/ztest/ztest.c --replace "/usr/sbin/ztest" "$out/sbin/ztest"
substituteInPlace ./cmd/ztest/ztest.c --replace "/usr/sbin/zdb" "$out/sbin/zdb"
./autogen.sh
'';
configureFlags = [
"--disable-systemd"
"--enable-systemd"
"--with-linux=${kernel.dev}/lib/modules/${kernel.modDirVersion}/source"
"--with-linux-obj=${kernel.dev}/lib/modules/${kernel.modDirVersion}/build"
"--with-spl=${spl_git}/libexec/spl"
"--with-dracutdir=$(out)/lib/dracut"
"--with-udevdir=$(out)/lib/udev"
"--with-systemdunitdir=$(out)/etc/systemd/system"
"--with-systemdpresetdir=$(out)/etc/systemd/system-preset"
"--sysconfdir=/etc"
"--localstatedir=/var"
];
enableParallelBuilding = true;
# Remove provided services as they are buggy
postInstall = ''
rm $out/etc/systemd/system/zfs-import-*.service
sed -i '/zfs-import-scan.service/d' $out/etc/systemd/system/*
for i in $out/etc/systemd/system/*; do
substituteInPlace $i --replace "zfs-import-cache.service" "zfs-import.target"
done
'';
meta = {
description = "ZFS Filesystem Linux Kernel module";
longDescription = ''

1
pkgs/stdenv/generic/default.nix
View File

@ -187,6 +187,7 @@ let
isArm = system == "armv5tel-linux"
|| system == "armv6l-linux"
|| system == "armv7l-linux";
isBigEndian = system == "powerpc-linux";
# For convenience, bring in the library functions in lib/ so
# packages don't have to do that themselves.