Merge pull request #230498 from Mic92/zfs-overridable

zfs: make kernel packages overridable
This commit is contained in:
Jörg Thalheim 2023-05-14 15:15:47 +01:00 committed by GitHub
commit 22f3e9411e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 295 additions and 277 deletions

View File

@ -1,270 +0,0 @@
{ pkgs, lib, stdenv, fetchFromGitHub, fetchpatch
, autoreconfHook269, util-linux, nukeReferences, coreutils
, perl, nixosTests
, configFile ? "all"
# Userspace dependencies
, zlib, libuuid, python3, attr, openssl
, libtirpc
, nfs-utils, samba
, gawk, gnugrep, gnused, systemd
, smartmontools, enableMail ? false
, sysstat, pkg-config
, curl
# Kernel dependencies
, kernel ? null
, enablePython ? true
# for determining the latest compatible linuxPackages
, linuxPackages_6_1 ? pkgs.linuxKernel.packages.linux_6_1
, linuxPackages_6_2 ? pkgs.linuxKernel.packages.linux_6_2
}:
let
inherit (lib) any optionalString optionals optional makeBinPath;
smartmon = smartmontools.override { inherit enableMail; };
buildKernel = any (n: n == configFile) [ "kernel" "all" ];
buildUser = any (n: n == configFile) [ "user" "all" ];
# XXX: You always want to build kernel modules with the same stdenv as the
# kernel was built with. However, since zfs can also be built for userspace we
# need to correctly pick between the provided/default stdenv, and the one used
# by the kernel.
# If you don't do this your ZFS builds will fail on any non-standard (e.g.
# clang-built) kernels.
stdenv' = if kernel == null then stdenv else kernel.stdenv;
common = { version
, sha256
, extraPatches ? []
, rev ? "zfs-${version}"
, isUnstable ? false
, latestCompatibleLinuxPackages
, kernelCompatible ? null }:
stdenv'.mkDerivation {
name = "zfs-${configFile}-${version}${optionalString buildKernel "-${kernel.version}"}";
src = fetchFromGitHub {
owner = "openzfs";
repo = "zfs";
inherit rev sha256;
};
patches = [
(fetchpatch {
name = "musl.patch";
url = "https://github.com/openzfs/zfs/commit/1f19826c9ac85835cbde61a7439d9d1fefe43a4a.patch";
sha256 = "XEaK227ubfOwlB2s851UvZ6xp/QOtYUWYsKTkEHzmo0=";
})
] ++ extraPatches;
postPatch = optionalString buildKernel ''
patchShebangs scripts
# The arrays must remain the same length, so we repeat a flag that is
# already part of the command and therefore has no effect.
substituteInPlace ./module/os/linux/zfs/zfs_ctldir.c \
--replace '"/usr/bin/env", "umount"' '"${util-linux}/bin/umount", "-n"' \
--replace '"/usr/bin/env", "mount"' '"${util-linux}/bin/mount", "-n"'
'' + optionalString buildUser ''
substituteInPlace ./lib/libshare/os/linux/nfs.c --replace "/usr/sbin/exportfs" "${
# We don't *need* python support, but we set it like this to minimize closure size:
# If it's disabled by default, no need to enable it, even if we have python enabled
# And if it's enabled by default, only change that if we explicitly disable python to remove python from the closure
nfs-utils.override (old: { enablePython = old.enablePython or true && enablePython; })
}/bin/exportfs"
substituteInPlace ./lib/libshare/smb.h --replace "/usr/bin/net" "${samba}/bin/net"
# Disable dynamic loading of libcurl
substituteInPlace ./config/user-libfetch.m4 --replace "curl-config --built-shared" "true"
substituteInPlace ./config/user-systemd.m4 --replace "/usr/lib/modules-load.d" "$out/etc/modules-load.d"
substituteInPlace ./config/zfs-build.m4 --replace "\$sysconfdir/init.d" "$out/etc/init.d" \
--replace "/etc/default" "$out/etc/default"
substituteInPlace ./etc/zfs/Makefile.am --replace "\$(sysconfdir)" "$out/etc"
substituteInPlace ./contrib/initramfs/hooks/Makefile.am \
--replace "/usr/share/initramfs-tools/hooks" "$out/usr/share/initramfs-tools/hooks"
substituteInPlace ./contrib/initramfs/Makefile.am \
--replace "/usr/share/initramfs-tools" "$out/usr/share/initramfs-tools"
substituteInPlace ./contrib/initramfs/scripts/Makefile.am \
--replace "/usr/share/initramfs-tools/scripts" "$out/usr/share/initramfs-tools/scripts"
substituteInPlace ./contrib/initramfs/scripts/local-top/Makefile.am \
--replace "/usr/share/initramfs-tools/scripts/local-top" "$out/usr/share/initramfs-tools/scripts/local-top"
substituteInPlace ./contrib/initramfs/scripts/Makefile.am \
--replace "/usr/share/initramfs-tools/scripts" "$out/usr/share/initramfs-tools/scripts"
substituteInPlace ./contrib/initramfs/scripts/local-top/Makefile.am \
--replace "/usr/share/initramfs-tools/scripts/local-top" "$out/usr/share/initramfs-tools/scripts/local-top"
substituteInPlace ./etc/systemd/system/Makefile.am \
--replace '$(DESTDIR)$(systemdunitdir)' "$out"'$(DESTDIR)$(systemdunitdir)'
substituteInPlace ./contrib/initramfs/conf.d/Makefile.am \
--replace "/usr/share/initramfs-tools/conf.d" "$out/usr/share/initramfs-tools/conf.d"
substituteInPlace ./contrib/initramfs/conf-hooks.d/Makefile.am \
--replace "/usr/share/initramfs-tools/conf-hooks.d" "$out/usr/share/initramfs-tools/conf-hooks.d"
substituteInPlace ./cmd/vdev_id/vdev_id \
--replace "PATH=/bin:/sbin:/usr/bin:/usr/sbin" \
"PATH=${makeBinPath [ coreutils gawk gnused gnugrep systemd ]}"
'';
nativeBuildInputs = [ autoreconfHook269 nukeReferences ]
++ optionals buildKernel (kernel.moduleBuildDependencies ++ [ perl ])
++ optional buildUser pkg-config;
buildInputs = optionals buildUser [ zlib libuuid attr libtirpc ]
++ optional buildUser openssl
++ optional buildUser curl
++ optional (buildUser && enablePython) python3;
# for zdb to get the rpath to libgcc_s, needed for pthread_cancel to work
NIX_CFLAGS_LINK = "-lgcc_s";
hardeningDisable = [ "fortify" "stackprotector" "pic" ];
configureFlags = [
"--with-config=${configFile}"
"--with-tirpc=1"
(lib.withFeatureAs (buildUser && enablePython) "python" python3.interpreter)
] ++ optionals buildUser [
"--with-dracutdir=$(out)/lib/dracut"
"--with-udevdir=$(out)/lib/udev"
"--with-systemdunitdir=$(out)/etc/systemd/system"
"--with-systemdpresetdir=$(out)/etc/systemd/system-preset"
"--with-systemdgeneratordir=$(out)/lib/systemd/system-generator"
"--with-mounthelperdir=$(out)/bin"
"--libexecdir=$(out)/libexec"
"--sysconfdir=/etc"
"--localstatedir=/var"
"--enable-systemd"
] ++ optionals buildKernel ([
"--with-linux=${kernel.dev}/lib/modules/${kernel.modDirVersion}/source"
"--with-linux-obj=${kernel.dev}/lib/modules/${kernel.modDirVersion}/build"
] ++ kernel.makeFlags);
makeFlags = optionals buildKernel kernel.makeFlags;
enableParallelBuilding = true;
installFlags = [
"sysconfdir=\${out}/etc"
"DEFAULT_INITCONF_DIR=\${out}/default"
"INSTALL_MOD_PATH=\${out}"
];
# Enabling BTF causes zfs to be build with debug symbols.
# Since zfs compress kernel modules on installation, our strip hooks skip stripping them.
# Hence we strip modules prior to compression.
postBuild = optionalString buildKernel ''
find . -name "*.ko" -print0 | xargs -0 -P$NIX_BUILD_CORES ${stdenv.cc.targetPrefix}strip --strip-debug
'';
postInstall = optionalString buildKernel ''
# Add reference that cannot be detected due to compressed kernel module
mkdir -p "$out/nix-support"
echo "${util-linux}" >> "$out/nix-support/extra-refs"
'' + optionalString buildUser ''
# Remove provided services as they are buggy
rm $out/etc/systemd/system/zfs-import-*.service
sed -i '/zfs-import-scan.service/d' $out/etc/systemd/system/*
for i in $out/etc/systemd/system/*; do
substituteInPlace $i --replace "zfs-import-cache.service" "zfs-import.target"
done
# Remove tests because they add a runtime dependency on gcc
rm -rf $out/share/zfs/zfs-tests
# Add Bash completions.
install -v -m444 -D -t $out/share/bash-completion/completions contrib/bash_completion.d/zfs
(cd $out/share/bash-completion/completions; ln -s zfs zpool)
'';
postFixup = let
path = "PATH=${makeBinPath [ coreutils gawk gnused gnugrep util-linux smartmon sysstat ]}:$PATH";
in ''
for i in $out/libexec/zfs/zpool.d/*; do
sed -i '2i${path}' $i
done
'';
outputs = [ "out" ] ++ optionals buildUser [ "dev" ];
passthru = {
inherit enableMail latestCompatibleLinuxPackages;
tests =
if isUnstable then [
nixosTests.zfs.unstable
] else [
nixosTests.zfs.installer
nixosTests.zfs.stable
];
};
meta = {
description = "ZFS Filesystem Linux Kernel module";
longDescription = ''
ZFS is a filesystem that combines a logical volume manager with a
Copy-On-Write filesystem with data integrity detection and repair,
snapshotting, cloning, block devices, deduplication, and more.
'';
homepage = "https://github.com/openzfs/zfs";
changelog = "https://github.com/openzfs/zfs/releases/tag/zfs-${version}";
license = lib.licenses.cddl;
platforms = lib.platforms.linux;
maintainers = with lib.maintainers; [ jcumming jonringer globin raitobezarius ];
mainProgram = "zfs";
# If your Linux kernel version is not yet supported by zfs, try zfsUnstable.
# On NixOS set the option boot.zfs.enableUnstable.
broken = buildKernel && (kernelCompatible != null) && !kernelCompatible;
};
};
in {
# also check if kernel version constraints in
# ./nixos/modules/tasks/filesystems/zfs.nix needs
# to be adapted
zfsStable = common {
# check the release notes for compatible kernels
kernelCompatible =
if stdenv'.isx86_64
then kernel.kernelOlder "6.3"
else kernel.kernelOlder "6.2";
latestCompatibleLinuxPackages =
if stdenv'.isx86_64
then linuxPackages_6_2
else linuxPackages_6_1;
# this package should point to the latest release.
version = "2.1.11";
sha256 = "tJLwyqUj1l5F0WKZDeMGrEFa8fc/axKqm31xtN51a5M=";
};
zfsUnstable = common {
# check the release notes for compatible kernels
# NOTE:
# zfs-2.1.9<=x<=2.1.10 is broken with aarch64-linux-6.2
# for future releases, please delete this condition.
kernelCompatible =
if stdenv'.isx86_64
then kernel.kernelOlder "6.3"
else kernel.kernelOlder "6.2";
latestCompatibleLinuxPackages =
if stdenv'.isx86_64
then linuxPackages_6_2
else linuxPackages_6_1;
# this package should point to a version / git revision compatible with the latest kernel release
# IMPORTANT: Always use a tagged release candidate or commits from the
# zfs-<version>-staging branch, because this is tested by the OpenZFS
# maintainers.
version = "2.1.12-staging-2023-04-18";
rev = "e25f9131d679692704c11dc0c1df6d4585b70c35";
sha256 = "tJLwyqUj1l5F0WKZDeMGrEFa8fc/axKqm31xtN51a5M=";
isUnstable = true;
};
}

View File

@ -0,0 +1,222 @@
{ pkgs, lib, stdenv, fetchFromGitHub, fetchpatch
, autoreconfHook269, util-linux, nukeReferences, coreutils
, perl, nixosTests
, configFile ? "all"
# Userspace dependencies
, zlib, libuuid, python3, attr, openssl
, libtirpc
, nfs-utils, samba
, gawk, gnugrep, gnused, systemd
, smartmontools, enableMail ? false
, sysstat, pkg-config
, curl
# Kernel dependencies
, kernel ? null
, enablePython ? true
, ...
}:
{ version
, sha256
, extraPatches ? []
, rev ? "zfs-${version}"
, isUnstable ? false
, latestCompatibleLinuxPackages
, kernelCompatible ? null
}:
let
inherit (lib) any optionalString optionals optional makeBinPath;
smartmon = smartmontools.override { inherit enableMail; };
buildKernel = any (n: n == configFile) [ "kernel" "all" ];
buildUser = any (n: n == configFile) [ "user" "all" ];
# XXX: You always want to build kernel modules with the same stdenv as the
# kernel was built with. However, since zfs can also be built for userspace we
# need to correctly pick between the provided/default stdenv, and the one used
# by the kernel.
# If you don't do this your ZFS builds will fail on any non-standard (e.g.
# clang-built) kernels.
stdenv' = if kernel == null then stdenv else kernel.stdenv;
in
stdenv'.mkDerivation {
name = "zfs-${configFile}-${version}${optionalString buildKernel "-${kernel.version}"}";
src = fetchFromGitHub {
owner = "openzfs";
repo = "zfs";
inherit rev sha256;
};
patches = [
(fetchpatch {
name = "musl.patch";
url = "https://github.com/openzfs/zfs/commit/1f19826c9ac85835cbde61a7439d9d1fefe43a4a.patch";
sha256 = "XEaK227ubfOwlB2s851UvZ6xp/QOtYUWYsKTkEHzmo0=";
})
] ++ extraPatches;
postPatch = optionalString buildKernel ''
patchShebangs scripts
# The arrays must remain the same length, so we repeat a flag that is
# already part of the command and therefore has no effect.
substituteInPlace ./module/os/linux/zfs/zfs_ctldir.c \
--replace '"/usr/bin/env", "umount"' '"${util-linux}/bin/umount", "-n"' \
--replace '"/usr/bin/env", "mount"' '"${util-linux}/bin/mount", "-n"'
'' + optionalString buildUser ''
substituteInPlace ./lib/libshare/os/linux/nfs.c --replace "/usr/sbin/exportfs" "${
# We don't *need* python support, but we set it like this to minimize closure size:
# If it's disabled by default, no need to enable it, even if we have python enabled
# And if it's enabled by default, only change that if we explicitly disable python to remove python from the closure
nfs-utils.override (old: { enablePython = old.enablePython or true && enablePython; })
}/bin/exportfs"
substituteInPlace ./lib/libshare/smb.h --replace "/usr/bin/net" "${samba}/bin/net"
# Disable dynamic loading of libcurl
substituteInPlace ./config/user-libfetch.m4 --replace "curl-config --built-shared" "true"
substituteInPlace ./config/user-systemd.m4 --replace "/usr/lib/modules-load.d" "$out/etc/modules-load.d"
substituteInPlace ./config/zfs-build.m4 --replace "\$sysconfdir/init.d" "$out/etc/init.d" \
--replace "/etc/default" "$out/etc/default"
substituteInPlace ./etc/zfs/Makefile.am --replace "\$(sysconfdir)" "$out/etc"
substituteInPlace ./contrib/initramfs/hooks/Makefile.am \
--replace "/usr/share/initramfs-tools/hooks" "$out/usr/share/initramfs-tools/hooks"
substituteInPlace ./contrib/initramfs/Makefile.am \
--replace "/usr/share/initramfs-tools" "$out/usr/share/initramfs-tools"
substituteInPlace ./contrib/initramfs/scripts/Makefile.am \
--replace "/usr/share/initramfs-tools/scripts" "$out/usr/share/initramfs-tools/scripts"
substituteInPlace ./contrib/initramfs/scripts/local-top/Makefile.am \
--replace "/usr/share/initramfs-tools/scripts/local-top" "$out/usr/share/initramfs-tools/scripts/local-top"
substituteInPlace ./contrib/initramfs/scripts/Makefile.am \
--replace "/usr/share/initramfs-tools/scripts" "$out/usr/share/initramfs-tools/scripts"
substituteInPlace ./contrib/initramfs/scripts/local-top/Makefile.am \
--replace "/usr/share/initramfs-tools/scripts/local-top" "$out/usr/share/initramfs-tools/scripts/local-top"
substituteInPlace ./etc/systemd/system/Makefile.am \
--replace '$(DESTDIR)$(systemdunitdir)' "$out"'$(DESTDIR)$(systemdunitdir)'
substituteInPlace ./contrib/initramfs/conf.d/Makefile.am \
--replace "/usr/share/initramfs-tools/conf.d" "$out/usr/share/initramfs-tools/conf.d"
substituteInPlace ./contrib/initramfs/conf-hooks.d/Makefile.am \
--replace "/usr/share/initramfs-tools/conf-hooks.d" "$out/usr/share/initramfs-tools/conf-hooks.d"
substituteInPlace ./cmd/vdev_id/vdev_id \
--replace "PATH=/bin:/sbin:/usr/bin:/usr/sbin" \
"PATH=${makeBinPath [ coreutils gawk gnused gnugrep systemd ]}"
'';
nativeBuildInputs = [ autoreconfHook269 nukeReferences ]
++ optionals buildKernel (kernel.moduleBuildDependencies ++ [ perl ])
++ optional buildUser pkg-config;
buildInputs = optionals buildUser [ zlib libuuid attr libtirpc ]
++ optional buildUser openssl
++ optional buildUser curl
++ optional (buildUser && enablePython) python3;
# for zdb to get the rpath to libgcc_s, needed for pthread_cancel to work
NIX_CFLAGS_LINK = "-lgcc_s";
hardeningDisable = [ "fortify" "stackprotector" "pic" ];
configureFlags = [
"--with-config=${configFile}"
"--with-tirpc=1"
(lib.withFeatureAs (buildUser && enablePython) "python" python3.interpreter)
] ++ optionals buildUser [
"--with-dracutdir=$(out)/lib/dracut"
"--with-udevdir=$(out)/lib/udev"
"--with-systemdunitdir=$(out)/etc/systemd/system"
"--with-systemdpresetdir=$(out)/etc/systemd/system-preset"
"--with-systemdgeneratordir=$(out)/lib/systemd/system-generator"
"--with-mounthelperdir=$(out)/bin"
"--libexecdir=$(out)/libexec"
"--sysconfdir=/etc"
"--localstatedir=/var"
"--enable-systemd"
] ++ optionals buildKernel ([
"--with-linux=${kernel.dev}/lib/modules/${kernel.modDirVersion}/source"
"--with-linux-obj=${kernel.dev}/lib/modules/${kernel.modDirVersion}/build"
] ++ kernel.makeFlags);
makeFlags = optionals buildKernel kernel.makeFlags;
enableParallelBuilding = true;
installFlags = [
"sysconfdir=\${out}/etc"
"DEFAULT_INITCONF_DIR=\${out}/default"
"INSTALL_MOD_PATH=\${out}"
];
# Enabling BTF causes zfs to be build with debug symbols.
# Since zfs compress kernel modules on installation, our strip hooks skip stripping them.
# Hence we strip modules prior to compression.
postBuild = optionalString buildKernel ''
find . -name "*.ko" -print0 | xargs -0 -P$NIX_BUILD_CORES ${stdenv.cc.targetPrefix}strip --strip-debug
'';
postInstall = optionalString buildKernel ''
# Add reference that cannot be detected due to compressed kernel module
mkdir -p "$out/nix-support"
echo "${util-linux}" >> "$out/nix-support/extra-refs"
'' + optionalString buildUser ''
# Remove provided services as they are buggy
rm $out/etc/systemd/system/zfs-import-*.service
sed -i '/zfs-import-scan.service/d' $out/etc/systemd/system/*
for i in $out/etc/systemd/system/*; do
substituteInPlace $i --replace "zfs-import-cache.service" "zfs-import.target"
done
# Remove tests because they add a runtime dependency on gcc
rm -rf $out/share/zfs/zfs-tests
# Add Bash completions.
install -v -m444 -D -t $out/share/bash-completion/completions contrib/bash_completion.d/zfs
(cd $out/share/bash-completion/completions; ln -s zfs zpool)
'';
postFixup = let
path = "PATH=${makeBinPath [ coreutils gawk gnused gnugrep util-linux smartmon sysstat ]}:$PATH";
in ''
for i in $out/libexec/zfs/zpool.d/*; do
sed -i '2i${path}' $i
done
'';
outputs = [ "out" ] ++ optionals buildUser [ "dev" ];
passthru = {
inherit enableMail latestCompatibleLinuxPackages;
tests =
if isUnstable then [
nixosTests.zfs.unstable
] else [
nixosTests.zfs.installer
nixosTests.zfs.stable
];
};
meta = {
description = "ZFS Filesystem Linux Kernel module";
longDescription = ''
ZFS is a filesystem that combines a logical volume manager with a
Copy-On-Write filesystem with data integrity detection and repair,
snapshotting, cloning, block devices, deduplication, and more.
'';
homepage = "https://github.com/openzfs/zfs";
changelog = "https://github.com/openzfs/zfs/releases/tag/zfs-${version}";
license = lib.licenses.cddl;
platforms = lib.platforms.linux;
maintainers = with lib.maintainers; [ jcumming jonringer globin raitobezarius ];
mainProgram = "zfs";
# If your Linux kernel version is not yet supported by zfs, try zfsUnstable.
# On NixOS set the option boot.zfs.enableUnstable.
broken = buildKernel && (kernelCompatible != null) && !kernelCompatible;
};
}

View File

@ -0,0 +1,26 @@
{ callPackage
, kernel ? null
, stdenv
, linuxKernel
, ...
} @ args:
let
stdenv' = if kernel == null then stdenv else kernel.stdenv;
in
callPackage ./generic.nix args {
# check the release notes for compatible kernels
kernelCompatible =
if stdenv'.isx86_64
then kernel.kernelOlder "6.3"
else kernel.kernelOlder "6.2";
latestCompatibleLinuxPackages =
if stdenv'.isx86_64
then linuxKernel.packages.linux_6_1
else linuxKernel.packages.linux_6_2;
# this package should point to the latest release.
version = "2.1.11";
sha256 = "tJLwyqUj1l5F0WKZDeMGrEFa8fc/axKqm31xtN51a5M=";
}

View File

@ -0,0 +1,34 @@
{ callPackage
, kernel ? null
, stdenv
, linuxKernel
, ...
} @ args:
let
stdenv' = if kernel == null then stdenv else kernel.stdenv;
in
callPackage ./generic.nix args {
# check the release notes for compatible kernels
# NOTE:
# zfs-2.1.9<=x<=2.1.10 is broken with aarch64-linux-6.2
# for future releases, please delete this condition.
kernelCompatible = if stdenv'.isx86_64
then kernel.kernelOlder "6.3"
else kernel.kernelOlder "6.2";
latestCompatibleLinuxPackages =
if stdenv'.isx86_64
then linuxKernel.packages.linux_6_2
else linuxKernel.packages.linux_6_1;
# this package should point to a version / git revision compatible with the latest kernel release
# IMPORTANT: Always use a tagged release candidate or commits from the
# zfs-<version>-staging branch, because this is tested by the OpenZFS
# maintainers.
version = "2.1.12-staging-2023-04-18";
rev = "e25f9131d679692704c11dc0c1df6d4585b70c35";
sha256 = "tJLwyqUj1l5F0WKZDeMGrEFa8fc/axKqm31xtN51a5M=";
isUnstable = true;
}

View File

@ -27711,10 +27711,12 @@ with pkgs;
zenmonitor = callPackage ../os-specific/linux/zenmonitor { };
inherit (callPackages ../os-specific/linux/zfs {
zfsStable = callPackage ../os-specific/linux/zfs/stable.nix {
configFile = "user";
}) zfsStable zfsUnstable;
};
zfsUnstable = callPackage ../os-specific/linux/zfs/unstable.nix {
configFile = "user";
};
zfs = zfsStable;
### DATA

View File

@ -558,10 +558,14 @@ in {
zenpower = callPackage ../os-specific/linux/zenpower { };
inherit (callPackage ../os-specific/linux/zfs {
configFile = "kernel";
inherit pkgs kernel;
}) zfsStable zfsUnstable;
zfsStable = callPackage ../os-specific/linux/zfs/stable.nix {
configFile = "kernel";
inherit pkgs kernel;
};
zfsUnstable = callPackage ../os-specific/linux/zfs/unstable.nix {
configFile = "kernel";
inherit pkgs kernel;
};
zfs = zfsStable;
can-isotp = callPackage ../os-specific/linux/can-isotp { };