diff --git a/nixos/doc/manual/from_md/release-notes/rl-2111.section.xml b/nixos/doc/manual/from_md/release-notes/rl-2111.section.xml
index 8104f293e8b0..700ad741c003 100644
--- a/nixos/doc/manual/from_md/release-notes/rl-2111.section.xml
+++ b/nixos/doc/manual/from_md/release-notes/rl-2111.section.xml
@@ -344,6 +344,13 @@
controller support.
+
+
+ multipath,
+ the device mapper multipath (DM-MP) daemon. Available as
+ services.multipath.
+
+
diff --git a/nixos/doc/manual/release-notes/rl-2111.section.md b/nixos/doc/manual/release-notes/rl-2111.section.md
index 73a43be8c741..ab1e94d59f18 100644
--- a/nixos/doc/manual/release-notes/rl-2111.section.md
+++ b/nixos/doc/manual/release-notes/rl-2111.section.md
@@ -105,6 +105,8 @@ In addition to numerous new and upgraded packages, this release has the followin
- [joycond](https://github.com/DanielOgorchock/joycond), a service that uses `hid-nintendo` to provide nintendo joycond pairing and better nintendo switch pro controller support.
+- [multipath](https://github.com/opensvc/multipath-tools), the device mapper multipath (DM-MP) daemon. Available as [services.multipath](#opt-services.multipath.enable).
+
## Backward Incompatibilities {#sec-release-21.11-incompatibilities}
- The `services.wakeonlan` option was removed, and replaced with `networking.interfaces..wakeOnLan`.
diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix
index c45309bd6b69..df35a57d047f 100644
--- a/nixos/modules/module-list.nix
+++ b/nixos/modules/module-list.nix
@@ -779,6 +779,7 @@
./services/networking/mstpd.nix
./services/networking/mtprotoproxy.nix
./services/networking/mullvad-vpn.nix
+ ./services/networking/multipath.nix
./services/networking/murmur.nix
./services/networking/mxisd.nix
./services/networking/namecoind.nix
diff --git a/nixos/modules/services/networking/iscsi/root-initiator.nix b/nixos/modules/services/networking/iscsi/root-initiator.nix
index 3274878c4fae..c12aca1bc24d 100644
--- a/nixos/modules/services/networking/iscsi/root-initiator.nix
+++ b/nixos/modules/services/networking/iscsi/root-initiator.nix
@@ -64,6 +64,12 @@ in
default = false;
};
+ extraIscsiCommands = mkOption {
+ description = "Extra iscsi commands to run in the initrd.";
+ default = "";
+ type = lines;
+ };
+
extraConfig = mkOption {
description = "Extra lines to append to /etc/iscsid.conf";
default = null;
@@ -162,6 +168,9 @@ in
'' else ''
iscsiadm --mode node --targetname ${escapeShellArg cfg.target} --login
''}
+
+ ${cfg.extraIscsiCommands}
+
pkill -9 iscsid
'';
};
diff --git a/nixos/modules/services/networking/multipath.nix b/nixos/modules/services/networking/multipath.nix
new file mode 100644
index 000000000000..1cc2ad1fc849
--- /dev/null
+++ b/nixos/modules/services/networking/multipath.nix
@@ -0,0 +1,572 @@
+{ config, lib, pkgs, ... }: with lib;
+
+# See http://christophe.varoqui.free.fr/usage.html and
+# https://github.com/opensvc/multipath-tools/blob/master/multipath/multipath.conf.5
+
+let
+ cfg = config.services.multipath;
+
+ indentLines = n: str: concatStringsSep "\n" (
+ map (line: "${fixedWidthString n " " " "}${line}") (
+ filter ( x: x != "" ) ( splitString "\n" str )
+ )
+ );
+
+ addCheckDesc = desc: elemType: check: types.addCheck elemType check
+ // { description = "${elemType.description} (with check: ${desc})"; };
+ hexChars = stringToCharacters "0123456789abcdef";
+ isHexString = s: all (c: elem c hexChars) (stringToCharacters (toLower s));
+ hexStr = addCheckDesc "hexadecimal string" types.str isHexString;
+
+in {
+
+ options.services.multipath = with types; {
+
+ enable = mkEnableOption "the device mapper multipath (DM-MP) daemon";
+
+ package = mkOption {
+ type = package;
+ description = "multipath-tools package to use";
+ default = pkgs.multipath-tools;
+ defaultText = "pkgs.multipath-tools";
+ };
+
+ devices = mkOption {
+ default = [ ];
+ example = literalExpression ''
+ [
+ {
+ vendor = "\"COMPELNT\"";
+ product = "\"Compellent Vol\"";
+ path_checker = "tur";
+ no_path_retry = "queue";
+ max_sectors_kb = 256;
+ }, ...
+ ]
+ '';
+ description = ''
+ This option allows you to define arrays for use in multipath
+ groups.
+ '';
+ type = listOf (submodule {
+ options = {
+
+ vendor = mkOption {
+ type = str;
+ example = "COMPELNT";
+ description = "Regular expression to match the vendor name";
+ };
+
+ product = mkOption {
+ type = str;
+ example = "Compellent Vol";
+ description = "Regular expression to match the product name";
+ };
+
+ revision = mkOption {
+ type = nullOr str;
+ default = null;
+ description = "Regular expression to match the product revision";
+ };
+
+ product_blacklist = mkOption {
+ type = nullOr str;
+ default = null;
+ description = "Products with the given vendor matching this string are blacklisted";
+ };
+
+ alias_prefix = mkOption {
+ type = nullOr str;
+ default = null;
+ description = "The user_friendly_names prefix to use for this device type, instead of the default mpath";
+ };
+
+ vpd_vendor = mkOption {
+ type = nullOr str;
+ default = null;
+ description = "The vendor specific vpd page information, using the vpd page abbreviation";
+ };
+
+ hardware_handler = mkOption {
+ type = nullOr (enum [ "emc" "rdac" "hp_sw" "alua" "ana" ]);
+ default = null;
+ description = "The hardware handler to use for this device type";
+ };
+
+ # Optional arguments
+ path_grouping_policy = mkOption {
+ type = nullOr (enum [ "failover" "multibus" "group_by_serial" "group_by_prio" "group_by_node_name" ]);
+ default = null; # real default: "failover"
+ description = "The default path grouping policy to apply to unspecified multipaths";
+ };
+
+ uid_attribute = mkOption {
+ type = nullOr str;
+ default = null;
+ description = "The udev attribute providing a unique path identifier (WWID)";
+ };
+
+ getuid_callout = mkOption {
+ type = nullOr str;
+ default = null;
+ description = ''
+ (Superseded by uid_attribute) The default program and args to callout
+ to obtain a unique path identifier. Should be specified with an absolute path.
+ '';
+ };
+
+ path_selector = mkOption {
+ type = nullOr (enum [
+ ''"round-robin 0"''
+ ''"queue-length 0"''
+ ''"service-time 0"''
+ ''"historical-service-time 0"''
+ ]);
+ default = null; # real default: "service-time 0"
+ description = "The default path selector algorithm to use; they are offered by the kernel multipath target";
+ };
+
+ path_checker = mkOption {
+ type = enum [ "readsector0" "tur" "emc_clariion" "hp_sw" "rdac" "directio" "cciss_tur" "none" ];
+ default = "tur";
+ description = "The default method used to determine the paths state";
+ };
+
+ prio = mkOption {
+ type = nullOr (enum [
+ "none" "const" "sysfs" "emc" "alua" "ontap" "rdac" "hp_sw" "hds"
+ "random" "weightedpath" "path_latency" "ana" "datacore" "iet"
+ ]);
+ default = null; # real default: "const"
+ description = "The name of the path priority routine";
+ };
+
+ prio_args = mkOption {
+ type = nullOr str;
+ default = null;
+ description = "Arguments to pass to to the prio function";
+ };
+
+ features = mkOption {
+ type = nullOr str;
+ default = null;
+ description = "Specify any device-mapper features to be used";
+ };
+
+ failback = mkOption {
+ type = nullOr str;
+ default = null; # real default: "manual"
+ description = "Tell multipathd how to manage path group failback. Quote integers as strings";
+ };
+
+ rr_weight = mkOption {
+ type = nullOr (enum [ "priorities" "uniform" ]);
+ default = null; # real default: "uniform"
+ description = ''
+ If set to priorities the multipath configurator will assign path weights
+ as "path prio * rr_min_io".
+ '';
+ };
+
+ no_path_retry = mkOption {
+ type = nullOr str;
+ default = null; # real default: "fail"
+ description = "Specify what to do when all paths are down. Quote integers as strings";
+ };
+
+ rr_min_io = mkOption {
+ type = nullOr int;
+ default = null; # real default: 1000
+ description = ''
+ Number of I/O requests to route to a path before switching to the next in the
+ same path group. This is only for Block I/O (BIO) based multipath and
+ only apply to round-robin path_selector.
+ '';
+ };
+
+ rr_min_io_rq = mkOption {
+ type = nullOr int;
+ default = null; # real default: 1
+ description = ''
+ Number of I/O requests to route to a path before switching to the next in the
+ same path group. This is only for Request based multipath and
+ only apply to round-robin path_selector.
+ '';
+ };
+
+ fast_io_fail_tmo = mkOption {
+ type = nullOr str;
+ default = null; # real default: 5
+ description = ''
+ Specify the number of seconds the SCSI layer will wait after a problem has been
+ detected on a FC remote port before failing I/O to devices on that remote port.
+ This should be smaller than dev_loss_tmo. Setting this to "off" will disable
+ the timeout. Quote integers as strings.
+ '';
+ };
+
+ dev_loss_tmo = mkOption {
+ type = nullOr str;
+ default = null; # real default: 600
+ description = ''
+ Specify the number of seconds the SCSI layer will wait after a problem has
+ been detected on a FC remote port before removing it from the system. This
+ can be set to "infinity" which sets it to the max value of 2147483647
+ seconds, or 68 years. It will be automatically adjusted to the overall
+ retry interval no_path_retry * polling_interval
+ if a number of retries is given with no_path_retry and the
+ overall retry interval is longer than the specified dev_loss_tmo value.
+ The Linux kernel will cap this value to 600 if fast_io_fail_tmo
+ is not set.
+ '';
+ };
+
+ flush_on_last_del = mkOption {
+ type = nullOr (enum [ "yes" "no" ]);
+ default = null; # real default: "no"
+ description = ''
+ If set to "yes" multipathd will disable queueing when the last path to a
+ device has been deleted.
+ '';
+ };
+
+ user_friendly_names = mkOption {
+ type = nullOr (enum [ "yes" "no" ]);
+ default = null; # real default: "no"
+ description = ''
+ If set to "yes", using the bindings file /etc/multipath/bindings
+ to assign a persistent and unique alias to the multipath, in the
+ form of mpath. If set to "no" use the WWID as the alias. In either
+ case this be will be overridden by any specific aliases in the
+ multipaths section.
+ '';
+ };
+
+ retain_attached_hw_handler = mkOption {
+ type = nullOr (enum [ "yes" "no" ]);
+ default = null; # real default: "yes"
+ description = ''
+ (Obsolete for kernels >= 4.3) If set to "yes" and the SCSI layer has
+ already attached a hardware_handler to the device, multipath will not
+ force the device to use the hardware_handler specified by mutipath.conf.
+ If the SCSI layer has not attached a hardware handler, multipath will
+ continue to use its configured hardware handler.
+
+ Important Note: Linux kernel 4.3 or newer always behaves as if
+ "retain_attached_hw_handler yes" was set.
+ '';
+ };
+
+ detect_prio = mkOption {
+ type = nullOr (enum [ "yes" "no" ]);
+ default = null; # real default: "yes"
+ description = ''
+ If set to "yes", multipath will try to detect if the device supports
+ SCSI-3 ALUA. If so, the device will automatically use the sysfs
+ prioritizer if the required sysf attributes access_state and
+ preferred_path are supported, or the alua prioritizer if not. If set
+ to "no", the prioritizer will be selected as usual.
+ '';
+ };
+
+ detect_checker = mkOption {
+ type = nullOr (enum [ "yes" "no" ]);
+ default = null; # real default: "yes"
+ description = ''
+ If set to "yes", multipath will try to detect if the device supports
+ SCSI-3 ALUA. If so, the device will automatically use the tur checker.
+ If set to "no", the checker will be selected as usual.
+ '';
+ };
+
+ deferred_remove = mkOption {
+ type = nullOr (enum [ "yes" "no" ]);
+ default = null; # real default: "no"
+ description = ''
+ If set to "yes", multipathd will do a deferred remove instead of a
+ regular remove when the last path device has been deleted. This means
+ that if the multipath device is still in use, it will be freed when
+ the last user closes it. If path is added to the multipath device
+ before the last user closes it, the deferred remove will be canceled.
+ '';
+ };
+
+ san_path_err_threshold = mkOption {
+ type = nullOr str;
+ default = null;
+ description = ''
+ If set to a value greater than 0, multipathd will watch paths and check
+ how many times a path has been failed due to errors.If the number of
+ failures on a particular path is greater then the san_path_err_threshold,
+ then the path will not reinstate till san_path_err_recovery_time. These
+ path failures should occur within a san_path_err_forget_rate checks, if
+ not we will consider the path is good enough to reinstantate.
+ '';
+ };
+
+ san_path_err_forget_rate = mkOption {
+ type = nullOr str;
+ default = null;
+ description = ''
+ If set to a value greater than 0, multipathd will check whether the path
+ failures has exceeded the san_path_err_threshold within this many checks
+ i.e san_path_err_forget_rate. If so we will not reinstante the path till
+ san_path_err_recovery_time.
+ '';
+ };
+
+ san_path_err_recovery_time = mkOption {
+ type = nullOr str;
+ default = null;
+ description = ''
+ If set to a value greater than 0, multipathd will make sure that when
+ path failures has exceeded the san_path_err_threshold within
+ san_path_err_forget_rate then the path will be placed in failed state
+ for san_path_err_recovery_time duration. Once san_path_err_recovery_time
+ has timeout we will reinstante the failed path. san_path_err_recovery_time
+ value should be in secs.
+ '';
+ };
+
+ marginal_path_err_sample_time = mkOption {
+ type = nullOr int;
+ default = null;
+ description = "One of the four parameters of supporting path check based on accounting IO error such as intermittent error";
+ };
+
+ marginal_path_err_rate_threshold = mkOption {
+ type = nullOr int;
+ default = null;
+ description = "The error rate threshold as a permillage (1/1000)";
+ };
+
+ marginal_path_err_recheck_gap_time = mkOption {
+ type = nullOr str;
+ default = null;
+ description = "One of the four parameters of supporting path check based on accounting IO error such as intermittent error";
+ };
+
+ marginal_path_double_failed_time = mkOption {
+ type = nullOr str;
+ default = null;
+ description = "One of the four parameters of supporting path check based on accounting IO error such as intermittent error";
+ };
+
+ delay_watch_checks = mkOption {
+ type = nullOr str;
+ default = null;
+ description = "This option is deprecated, and mapped to san_path_err_forget_rate";
+ };
+
+ delay_wait_checks = mkOption {
+ type = nullOr str;
+ default = null;
+ description = "This option is deprecated, and mapped to san_path_err_recovery_time";
+ };
+
+ skip_kpartx = mkOption {
+ type = nullOr (enum [ "yes" "no" ]);
+ default = null; # real default: "no"
+ description = "If set to yes, kpartx will not automatically create partitions on the device";
+ };
+
+ max_sectors_kb = mkOption {
+ type = nullOr int;
+ default = null;
+ description = "Sets the max_sectors_kb device parameter on all path devices and the multipath device to the specified value";
+ };
+
+ ghost_delay = mkOption {
+ type = nullOr int;
+ default = null;
+ description = "Sets the number of seconds that multipath will wait after creating a device with only ghost paths before marking it ready for use in systemd";
+ };
+
+ all_tg_pt = mkOption {
+ type = nullOr str;
+ default = null;
+ description = "Set the 'all targets ports' flag when registering keys with mpathpersist";
+ };
+
+ };
+ });
+ };
+
+ defaults = mkOption {
+ type = nullOr str;
+ default = null;
+ description = ''
+ This section defines default values for attributes which are used
+ whenever no values are given in the appropriate device or multipath
+ sections.
+ '';
+ };
+
+ blacklist = mkOption {
+ type = nullOr str;
+ default = null;
+ description = ''
+ This section defines which devices should be excluded from the
+ multipath topology discovery.
+ '';
+ };
+
+ blacklist_exceptions = mkOption {
+ type = nullOr str;
+ default = null;
+ description = ''
+ This section defines which devices should be included in the
+ multipath topology discovery, despite being listed in the
+ blacklist section.
+ '';
+ };
+
+ overrides = mkOption {
+ type = nullOr str;
+ default = null;
+ description = ''
+ This section defines values for attributes that should override the
+ device-specific settings for all devices.
+ '';
+ };
+
+ extraConfig = mkOption {
+ type = nullOr str;
+ default = null;
+ description = "Lines to append to default multipath.conf";
+ };
+
+ extraConfigFile = mkOption {
+ type = nullOr str;
+ default = null;
+ description = "Append an additional file's contents to /etc/multipath.conf";
+ };
+
+ pathGroups = mkOption {
+ example = literalExpression ''
+ [
+ {
+ wwid = "360080e500043b35c0123456789abcdef";
+ alias = 10001234;
+ array = "bigarray.example.com";
+ fsType = "zfs"; # optional
+ options = "ro"; # optional
+ }, ...
+ ]
+ '';
+ description = ''
+ This option allows you to define multipath groups as described
+ in http://christophe.varoqui.free.fr/usage.html.
+ '';
+ type = listOf (submodule {
+ options = {
+
+ alias = mkOption {
+ type = int;
+ example = 1001234;
+ description = "The name of the multipath device";
+ };
+
+ wwid = mkOption {
+ type = hexStr;
+ example = "360080e500043b35c0123456789abcdef";
+ description = "The identifier for the multipath device";
+ };
+
+ array = mkOption {
+ type = str;
+ default = null;
+ example = "bigarray.example.com";
+ description = "The DNS name of the storage array";
+ };
+
+ fsType = mkOption {
+ type = nullOr str;
+ default = null;
+ example = "zfs";
+ description = "Type of the filesystem";
+ };
+
+ options = mkOption {
+ type = nullOr str;
+ default = null;
+ example = "ro";
+ description = "Options used to mount the file system";
+ };
+
+ };
+ });
+ };
+
+ };
+
+ config = mkIf cfg.enable {
+ environment.etc."multipath.conf".text =
+ let
+ inherit (cfg) defaults blacklist blacklist_exceptions overrides;
+
+ mkDeviceBlock = cfg: let
+ nonNullCfg = lib.filterAttrs (k: v: v != null) cfg;
+ attrs = lib.mapAttrsToList (name: value: " ${name} ${toString value}") nonNullCfg;
+ in ''
+ device {
+ ${lib.concatStringsSep "\n" attrs}
+ }
+ '';
+ devices = lib.concatMapStringsSep "\n" mkDeviceBlock cfg.devices;
+
+ mkMultipathBlock = m: ''
+ multipath {
+ wwid ${m.wwid}
+ alias ${toString m.alias}
+ }
+ '';
+ multipaths = lib.concatMapStringsSep "\n" mkMultipathBlock cfg.pathGroups;
+
+ in ''
+ devices {
+ ${indentLines 2 devices}
+ }
+
+ ${optionalString (!isNull defaults) ''
+ defaults {
+ ${indentLines 2 defaults}
+ multipath_dir ${cfg.package}/lib/multipath
+ }
+ ''}
+ ${optionalString (!isNull blacklist) ''
+ blacklist {
+ ${indentLines 2 blacklist}
+ }
+ ''}
+ ${optionalString (!isNull blacklist_exceptions) ''
+ blacklist_exceptions {
+ ${indentLines 2 blacklist_exceptions}
+ }
+ ''}
+ ${optionalString (!isNull overrides) ''
+ overrides {
+ ${indentLines 2 overrides}
+ }
+ ''}
+ multipaths {
+ ${indentLines 2 multipaths}
+ }
+ '';
+
+ systemd.packages = [ cfg.package ];
+
+ environment.systemPackages = [ cfg.package ];
+ boot.kernelModules = [ "dm-multipath" "dm-service-time" ];
+
+ # We do not have systemd in stage-1 boot so must invoke `multipathd`
+ # with the `-1` argument which disables systemd calls. Invoke `multipath`
+ # to display the multipath mappings in the output of `journalctl -b`.
+ boot.initrd.kernelModules = [ "dm-multipath" "dm-service-time" ];
+ boot.initrd.postDeviceCommands = ''
+ modprobe -a dm-multipath dm-service-time
+ multipathd -s
+ (set -x && sleep 1 && multipath -ll)
+ '';
+ };
+}
diff --git a/nixos/modules/system/boot/stage-1.nix b/nixos/modules/system/boot/stage-1.nix
index bd7e955a6f42..adbed9d8d58e 100644
--- a/nixos/modules/system/boot/stage-1.nix
+++ b/nixos/modules/system/boot/stage-1.nix
@@ -137,6 +137,14 @@ let
copy_bin_and_libs ${pkgs.e2fsprogs}/sbin/resize2fs
''}
+ # Copy multipath.
+ ${optionalString config.services.multipath.enable ''
+ copy_bin_and_libs ${config.services.multipath.package}/bin/multipath
+ copy_bin_and_libs ${config.services.multipath.package}/bin/multipathd
+ # Copy lib/multipath manually.
+ cp -rpv ${config.services.multipath.package}/lib/multipath $out/lib
+ ''}
+
# Copy secrets if needed.
#
# TODO: move out to a separate script; see #85000.
@@ -199,6 +207,10 @@ let
$out/bin/dmsetup --version 2>&1 | tee -a log | grep -q "version:"
LVM_SYSTEM_DIR=$out $out/bin/lvm version 2>&1 | tee -a log | grep -q "LVM"
$out/bin/mdadm --version
+ ${optionalString config.services.multipath.enable ''
+ ($out/bin/multipath || true) 2>&1 | grep -q 'need to be root'
+ ($out/bin/multipathd || true) 2>&1 | grep -q 'need to be root'
+ ''}
${config.boot.initrd.extraUtilsCommandsTest}
fi
@@ -338,7 +350,26 @@ let
{ object = pkgs.kmod-debian-aliases;
symlink = "/etc/modprobe.d/debian.conf";
}
- ];
+ ] ++ lib.optionals config.services.multipath.enable [
+ { object = pkgs.runCommand "multipath.conf" {
+ src = config.environment.etc."multipath.conf".text;
+ preferLocalBuild = true;
+ } ''
+ target=$out
+ printf "$src" > $out
+ substituteInPlace $out \
+ --replace ${config.services.multipath.package}/lib ${extraUtils}/lib
+ '';
+ symlink = "/etc/multipath.conf";
+ }
+ ] ++ (lib.mapAttrsToList
+ (symlink: options:
+ {
+ inherit symlink;
+ object = options.source;
+ }
+ )
+ config.boot.initrd.extraFiles);
};
# Script to add secret files to the initrd at bootloader update time
@@ -419,6 +450,22 @@ in
'';
};
+ boot.initrd.extraFiles = mkOption {
+ default = { };
+ type = types.attrsOf
+ (types.submodule {
+ options = {
+ source = mkOption {
+ type = types.package;
+ description = "The object to make available inside the initrd.";
+ };
+ };
+ });
+ description = ''
+ Extra files to link and copy in to the initrd.
+ '';
+ };
+
boot.initrd.prepend = mkOption {
default = [ ];
type = types.listOf types.str;
diff --git a/nixos/tests/iscsi-multipath-root.nix b/nixos/tests/iscsi-multipath-root.nix
new file mode 100644
index 000000000000..a26fea503b62
--- /dev/null
+++ b/nixos/tests/iscsi-multipath-root.nix
@@ -0,0 +1,267 @@
+import ./make-test-python.nix (
+ { pkgs, lib, ... }:
+ let
+ initiatorName = "iqn.2020-08.org.linux-iscsi.initiatorhost:example";
+ targetName = "iqn.2003-01.org.linux-iscsi.target.x8664:sn.acf8fd9c23af";
+ in
+ {
+ name = "iscsi";
+ meta = {
+ maintainers = pkgs.lib.teams.deshaw.members;
+ };
+
+ nodes = {
+ target = { config, pkgs, lib, ... }: {
+ virtualisation.vlans = [ 1 2 ];
+ services.target = {
+ enable = true;
+ config = {
+ fabric_modules = [ ];
+ storage_objects = [
+ {
+ dev = "/dev/vdb";
+ name = "test";
+ plugin = "block";
+ write_back = true;
+ wwn = "92b17c3f-6b40-4168-b082-ceeb7b495522";
+ }
+ ];
+ targets = [
+ {
+ fabric = "iscsi";
+ tpgs = [
+ {
+ enable = true;
+ attributes = {
+ authentication = 0;
+ generate_node_acls = 1;
+ };
+ luns = [
+ {
+ alias = "94dfe06967";
+ alua_tg_pt_gp_name = "default_tg_pt_gp";
+ index = 0;
+ storage_object = "/backstores/block/test";
+ }
+ ];
+ node_acls = [
+ {
+ mapped_luns = [
+ {
+ alias = "d42f5bdf8a";
+ index = 0;
+ tpg_lun = 0;
+ write_protect = false;
+ }
+ ];
+ node_wwn = initiatorName;
+ }
+ ];
+ portals = [
+ {
+ ip_address = "0.0.0.0";
+ iser = false;
+ offload = false;
+ port = 3260;
+ }
+ ];
+ tag = 1;
+ }
+ ];
+ wwn = targetName;
+ }
+ ];
+ };
+ };
+
+ networking.firewall.allowedTCPPorts = [ 3260 ];
+ networking.firewall.allowedUDPPorts = [ 3260 ];
+
+ virtualisation.memorySize = 2048;
+ virtualisation.emptyDiskImages = [ 2048 ];
+ };
+
+ initiatorAuto = { nodes, config, pkgs, ... }: {
+ virtualisation.vlans = [ 1 2 ];
+
+ services.multipath = {
+ enable = true;
+ defaults = ''
+ find_multipaths yes
+ user_friendly_names yes
+ '';
+ pathGroups = [
+ {
+ alias = 123456;
+ wwid = "3600140592b17c3f6b404168b082ceeb7";
+ }
+ ];
+ };
+
+ services.openiscsi = {
+ enable = true;
+ enableAutoLoginOut = true;
+ discoverPortal = "target";
+ name = initiatorName;
+ };
+
+ environment.systemPackages = with pkgs; [
+ xfsprogs
+ ];
+
+ environment.etc."initiator-root-disk-closure".source = nodes.initiatorRootDisk.config.system.build.toplevel;
+
+ nix.binaryCaches = lib.mkForce [ ];
+ nix.extraOptions = ''
+ hashed-mirrors =
+ connect-timeout = 1
+ '';
+ };
+
+ initiatorRootDisk = { config, pkgs, modulesPath, lib, ... }: {
+ boot.initrd.network.enable = true;
+ boot.loader.grub.enable = false;
+
+ boot.kernelParams = lib.mkOverride 5 (
+ [
+ "boot.shell_on_fail"
+ "console=tty1"
+ "ip=192.168.1.1:::255.255.255.0::ens9:none"
+ "ip=192.168.2.1:::255.255.255.0::ens10:none"
+ ]
+ );
+
+ # defaults to true, puts some code in the initrd that tries to mount an overlayfs on /nix/store
+ virtualisation.writableStore = false;
+ virtualisation.vlans = [ 1 2 ];
+
+ services.multipath = {
+ enable = true;
+ defaults = ''
+ find_multipaths yes
+ user_friendly_names yes
+ '';
+ pathGroups = [
+ {
+ alias = 123456;
+ wwid = "3600140592b17c3f6b404168b082ceeb7";
+ }
+ ];
+ };
+
+ fileSystems = lib.mkOverride 5 {
+ "/" = {
+ fsType = "xfs";
+ device = "/dev/mapper/123456";
+ options = [ "_netdev" ];
+ };
+ };
+
+ boot.initrd.extraFiles."etc/multipath/wwids".source = pkgs.writeText "wwids" "/3600140592b17c3f6b404168b082ceeb7/";
+
+ boot.iscsi-initiator = {
+ discoverPortal = "target";
+ name = initiatorName;
+ target = targetName;
+ extraIscsiCommands = ''
+ iscsiadm -m discovery -o update -t sendtargets -p 192.168.2.3 --login
+ '';
+ };
+ };
+
+ };
+
+ testScript = { nodes, ... }: ''
+ target.start()
+ target.wait_for_unit("iscsi-target.service")
+
+ initiatorAuto.start()
+
+ initiatorAuto.wait_for_unit("iscsid.service")
+ initiatorAuto.wait_for_unit("iscsi.service")
+ initiatorAuto.get_unit_info("iscsi")
+
+ # Expecting this to fail since we should already know about 192.168.1.3
+ initiatorAuto.fail("iscsiadm -m discovery -o update -t sendtargets -p 192.168.1.3 --login")
+ # Expecting this to succeed since we don't yet know about 192.168.2.3
+ initiatorAuto.succeed("iscsiadm -m discovery -o update -t sendtargets -p 192.168.2.3 --login")
+
+ # /dev/sda is provided by iscsi on target
+ initiatorAuto.succeed("set -x; while ! test -e /dev/sda; do sleep 1; done")
+
+ initiatorAuto.succeed("mkfs.xfs /dev/sda")
+ initiatorAuto.succeed("mkdir /mnt")
+
+ # Start by verifying /dev/sda and /dev/sdb are both the same disk
+ initiatorAuto.succeed("mount /dev/sda /mnt")
+ initiatorAuto.succeed("touch /mnt/hi")
+ initiatorAuto.succeed("umount /mnt")
+
+ initiatorAuto.succeed("mount /dev/sdb /mnt")
+ initiatorAuto.succeed("test -e /mnt/hi")
+ initiatorAuto.succeed("umount /mnt")
+
+ initiatorAuto.succeed("systemctl restart multipathd")
+ initiatorAuto.succeed("multipath -ll | systemd-cat")
+
+ # Install our RootDisk machine to 123456, the alias to the device that multipath is now managing
+ initiatorAuto.succeed("mount /dev/mapper/123456 /mnt")
+ initiatorAuto.succeed("mkdir -p /mnt/etc/{multipath,iscsi}")
+ initiatorAuto.succeed("cp -r /etc/multipath/wwids /mnt/etc/multipath/wwids")
+ initiatorAuto.succeed("cp -r /etc/iscsi/{nodes,send_targets} /mnt/etc/iscsi")
+ initiatorAuto.succeed(
+ "nixos-install --no-bootloader --no-root-passwd --system /etc/initiator-root-disk-closure"
+ )
+ initiatorAuto.succeed("umount /mnt")
+ initiatorAuto.shutdown()
+
+ initiatorRootDisk.start()
+ initiatorRootDisk.wait_for_unit("multi-user.target")
+ initiatorRootDisk.wait_for_unit("iscsid")
+
+ # Log in over both nodes
+ initiatorRootDisk.fail("iscsiadm -m discovery -o update -t sendtargets -p 192.168.1.3 --login")
+ initiatorRootDisk.fail("iscsiadm -m discovery -o update -t sendtargets -p 192.168.2.3 --login")
+ initiatorRootDisk.succeed("systemctl restart multipathd")
+ initiatorRootDisk.succeed("multipath -ll | systemd-cat")
+
+ # Verify we can write and sync the root disk
+ initiatorRootDisk.succeed("mkdir /scratch")
+ initiatorRootDisk.succeed("touch /scratch/both-up")
+ initiatorRootDisk.succeed("sync /scratch")
+
+ # Verify we can write to the root with ens9 (sda, 192.168.1.3) down
+ initiatorRootDisk.succeed("ip link set ens9 down")
+ initiatorRootDisk.succeed("touch /scratch/ens9-down")
+ initiatorRootDisk.succeed("sync /scratch")
+ initiatorRootDisk.succeed("ip link set ens9 up")
+
+ # todo: better way to wait until multipath notices the link is back
+ initiatorRootDisk.succeed("sleep 5")
+ initiatorRootDisk.succeed("touch /scratch/both-down")
+ initiatorRootDisk.succeed("sync /scratch")
+
+ # Verify we can write to the root with ens10 (sdb, 192.168.2.3) down
+ initiatorRootDisk.succeed("ip link set ens10 down")
+ initiatorRootDisk.succeed("touch /scratch/ens10-down")
+ initiatorRootDisk.succeed("sync /scratch")
+ initiatorRootDisk.succeed("ip link set ens10 up")
+ initiatorRootDisk.succeed("touch /scratch/ens10-down")
+ initiatorRootDisk.succeed("sync /scratch")
+
+ initiatorRootDisk.succeed("ip link set ens9 up")
+ initiatorRootDisk.succeed("ip link set ens10 up")
+ initiatorRootDisk.shutdown()
+
+ # Verify we can boot with the target's eth1 down, forcing
+ # it to multipath via the second link
+ target.succeed("ip link set eth1 down")
+ initiatorRootDisk.start()
+ initiatorRootDisk.wait_for_unit("multi-user.target")
+ initiatorRootDisk.wait_for_unit("iscsid")
+ initiatorRootDisk.succeed("test -e /scratch/both-up")
+ '';
+ }
+)
+
+
diff --git a/pkgs/os-specific/linux/multipath-tools/default.nix b/pkgs/os-specific/linux/multipath-tools/default.nix
index 241498057232..13e989243646 100644
--- a/pkgs/os-specific/linux/multipath-tools/default.nix
+++ b/pkgs/os-specific/linux/multipath-tools/default.nix
@@ -19,9 +19,11 @@ stdenv.mkDerivation rec {
substituteInPlace libmultipath/Makefile \
--replace /usr/include/libdevmapper.h ${lib.getDev lvm2}/include/libdevmapper.h
+ # systemd-udev-settle.service is deprecated.
substituteInPlace multipathd/multipathd.service \
--replace /sbin/modprobe ${lib.getBin kmod}/sbin/modprobe \
- --replace /sbin/multipathd "$out/bin/multipathd"
+ --replace /sbin/multipathd "$out/bin/multipathd" \
+ --replace " systemd-udev-settle.service" ""
sed -i -re '
s,^( *#define +DEFAULT_MULTIPATHDIR\>).*,\1 "'"$out/lib/multipath"'",