nixpkgs/nixos/modules/services/backup/znapzend.nix
2021-10-04 12:47:20 +02:00

470 lines
16 KiB
Nix

{ config, lib, pkgs, ... }:
with lib;
with types;
let
planDescription = ''
The znapzend backup plan to use for the source.
The plan specifies how often to backup and for how long to keep the
backups. It consists of a series of retention periodes to interval
associations:
<literal>
retA=>intA,retB=>intB,...
</literal>
Both intervals and retention periods are expressed in standard units
of time or multiples of them. You can use both the full name or a
shortcut according to the following listing:
<literal>
second|sec|s, minute|min, hour|h, day|d, week|w, month|mon|m, year|y
</literal>
See <citerefentry><refentrytitle>znapzendzetup</refentrytitle><manvolnum>1</manvolnum></citerefentry> for more info.
'';
planExample = "1h=>10min,1d=>1h,1w=>1d,1m=>1w,1y=>1m";
# A type for a string of the form number{b|k|M|G}
mbufferSizeType = str // {
check = x: str.check x && builtins.isList (builtins.match "^[0-9]+[bkMG]$" x);
description = "string of the form number{b|k|M|G}";
};
enabledFeatures = concatLists (mapAttrsToList (name: enabled: optional enabled name) cfg.features);
# Type for a string that must contain certain other strings (the list parameter).
# Note that these would need regex escaping.
stringContainingStrings = list: let
matching = s: map (str: builtins.match ".*${str}.*" s) list;
in str // {
check = x: str.check x && all isList (matching x);
description = "string containing all of the characters ${concatStringsSep ", " list}";
};
timestampType = stringContainingStrings [ "%Y" "%m" "%d" "%H" "%M" "%S" ];
destType = srcConfig: submodule ({ name, ... }: {
options = {
label = mkOption {
type = str;
description = "Label for this destination. Defaults to the attribute name.";
};
plan = mkOption {
type = str;
description = planDescription;
example = planExample;
};
dataset = mkOption {
type = str;
description = "Dataset name to send snapshots to.";
example = "tank/main";
};
host = mkOption {
type = nullOr str;
description = ''
Host to use for the destination dataset. Can be prefixed with
<literal>user@</literal> to specify the ssh user.
'';
default = null;
example = "john@example.com";
};
presend = mkOption {
type = nullOr str;
description = ''
Command to run before sending the snapshot to the destination.
Intended to run a remote script via <command>ssh</command> on the
destination, e.g. to bring up a backup disk or server or to put a
zpool online/offline. See also <option>postsend</option>.
'';
default = null;
example = "ssh root@bserv zpool import -Nf tank";
};
postsend = mkOption {
type = nullOr str;
description = ''
Command to run after sending the snapshot to the destination.
Intended to run a remote script via <command>ssh</command> on the
destination, e.g. to bring up a backup disk or server or to put a
zpool online/offline. See also <option>presend</option>.
'';
default = null;
example = "ssh root@bserv zpool export tank";
};
};
config = {
label = mkDefault name;
plan = mkDefault srcConfig.plan;
};
});
srcType = submodule ({ name, config, ... }: {
options = {
enable = mkOption {
type = bool;
description = "Whether to enable this source.";
default = true;
};
recursive = mkOption {
type = bool;
description = "Whether to do recursive snapshots.";
default = false;
};
mbuffer = {
enable = mkOption {
type = bool;
description = "Whether to use <command>mbuffer</command>.";
default = false;
};
port = mkOption {
type = nullOr ints.u16;
description = ''
Port to use for <command>mbuffer</command>.
If this is null, it will run <command>mbuffer</command> through
ssh.
If this is not null, it will run <command>mbuffer</command>
directly through TCP, which is not encrypted but faster. In that
case the given port needs to be open on the destination host.
'';
default = null;
};
size = mkOption {
type = mbufferSizeType;
description = ''
The size for <command>mbuffer</command>.
Supports the units b, k, M, G.
'';
default = "1G";
example = "128M";
};
};
presnap = mkOption {
type = nullOr str;
description = ''
Command to run before snapshots are taken on the source dataset,
e.g. for database locking/flushing. See also
<option>postsnap</option>.
'';
default = null;
example = literalExpression ''
'''''${pkgs.mariadb}/bin/mysql -e "set autocommit=0;flush tables with read lock;\\! ''${pkgs.coreutils}/bin/sleep 600" & ''${pkgs.coreutils}/bin/echo $! > /tmp/mariadblock.pid ; sleep 10'''
'';
};
postsnap = mkOption {
type = nullOr str;
description = ''
Command to run after snapshots are taken on the source dataset,
e.g. for database unlocking. See also <option>presnap</option>.
'';
default = null;
example = literalExpression ''
"''${pkgs.coreutils}/bin/kill `''${pkgs.coreutils}/bin/cat /tmp/mariadblock.pid`;''${pkgs.coreutils}/bin/rm /tmp/mariadblock.pid"
'';
};
timestampFormat = mkOption {
type = timestampType;
description = ''
The timestamp format to use for constructing snapshot names.
The syntax is <literal>strftime</literal>-like. The string must
consist of the mandatory <literal>%Y %m %d %H %M %S</literal>.
Optionally <literal>- _ . :</literal> characters as well as any
alphanumeric character are allowed. If suffixed by a
<literal>Z</literal>, times will be in UTC.
'';
default = "%Y-%m-%d-%H%M%S";
example = "znapzend-%m.%d.%Y-%H%M%SZ";
};
sendDelay = mkOption {
type = int;
description = ''
Specify delay (in seconds) before sending snaps to the destination.
May be useful if you want to control sending time.
'';
default = 0;
example = 60;
};
plan = mkOption {
type = str;
description = planDescription;
example = planExample;
};
dataset = mkOption {
type = str;
description = "The dataset to use for this source.";
example = "tank/home";
};
destinations = mkOption {
type = attrsOf (destType config);
description = "Additional destinations.";
default = {};
example = literalExpression ''
{
local = {
dataset = "btank/backup";
presend = "zpool import -N btank";
postsend = "zpool export btank";
};
remote = {
host = "john@example.com";
dataset = "tank/john";
};
};
'';
};
};
config = {
dataset = mkDefault name;
};
});
### Generating the configuration from here
cfg = config.services.znapzend;
onOff = b: if b then "on" else "off";
nullOff = b: if b == null then "off" else toString b;
stripSlashes = replaceStrings [ "/" ] [ "." ];
attrsToFile = config: concatStringsSep "\n" (builtins.attrValues (
mapAttrs (n: v: "${n}=${v}") config));
mkDestAttrs = dst: with dst;
mapAttrs' (n: v: nameValuePair "dst_${label}${n}" v) ({
"" = optionalString (host != null) "${host}:" + dataset;
_plan = plan;
} // optionalAttrs (presend != null) {
_precmd = presend;
} // optionalAttrs (postsend != null) {
_pstcmd = postsend;
});
mkSrcAttrs = srcCfg: with srcCfg; {
enabled = onOff enable;
# mbuffer is not referenced by its full path to accomodate non-NixOS systems or differing mbuffer versions between source and target
mbuffer = with mbuffer; if enable then "mbuffer"
+ optionalString (port != null) ":${toString port}" else "off";
mbuffer_size = mbuffer.size;
post_znap_cmd = nullOff postsnap;
pre_znap_cmd = nullOff presnap;
recursive = onOff recursive;
src = dataset;
src_plan = plan;
tsformat = timestampFormat;
zend_delay = toString sendDelay;
} // foldr (a: b: a // b) {} (
map mkDestAttrs (builtins.attrValues destinations)
);
files = mapAttrs' (n: srcCfg: let
fileText = attrsToFile (mkSrcAttrs srcCfg);
in {
name = srcCfg.dataset;
value = pkgs.writeText (stripSlashes srcCfg.dataset) fileText;
}) cfg.zetup;
in
{
options = {
services.znapzend = {
enable = mkEnableOption "ZnapZend ZFS backup daemon";
logLevel = mkOption {
default = "debug";
example = "warning";
type = enum ["debug" "info" "warning" "err" "alert"];
description = ''
The log level when logging to file. Any of debug, info, warning, err,
alert. Default in daemonized form is debug.
'';
};
logTo = mkOption {
type = str;
default = "syslog::daemon";
example = "/var/log/znapzend.log";
description = ''
Where to log to (syslog::&lt;facility&gt; or &lt;filepath&gt;).
'';
};
noDestroy = mkOption {
type = bool;
default = false;
description = "Does all changes to the filesystem except destroy.";
};
autoCreation = mkOption {
type = bool;
default = false;
description = "Automatically create the destination dataset if it does not exist.";
};
zetup = mkOption {
type = attrsOf srcType;
description = "Znapzend configuration.";
default = {};
example = literalExpression ''
{
"tank/home" = {
# Make snapshots of tank/home every hour, keep those for 1 day,
# keep every days snapshot for 1 month, etc.
plan = "1d=>1h,1m=>1d,1y=>1m";
recursive = true;
# Send all those snapshots to john@example.com:rtank/john as well
destinations.remote = {
host = "john@example.com";
dataset = "rtank/john";
};
};
};
'';
};
pure = mkOption {
type = bool;
description = ''
Do not persist any stateful znapzend setups. If this option is
enabled, your previously set znapzend setups will be cleared and only
the ones defined with this module will be applied.
'';
default = false;
};
features.oracleMode = mkEnableOption ''
Destroy snapshots one by one instead of using one long argument list.
If source and destination are out of sync for a long time, you may have
so many snapshots to destroy that the argument gets is too long and the
command fails.
'';
features.recvu = mkEnableOption ''
recvu feature which uses <literal>-u</literal> on the receiving end to keep the destination
filesystem unmounted.
'';
features.compressed = mkEnableOption ''
compressed feature which adds the options <literal>-Lce</literal> to
the <command>zfs send</command> command. When this is enabled, make
sure that both the sending and receiving pool have the same relevant
features enabled. Using <literal>-c</literal> will skip unneccessary
decompress-compress stages, <literal>-L</literal> is for large block
support and -e is for embedded data support. see
<citerefentry><refentrytitle>znapzend</refentrytitle><manvolnum>1</manvolnum></citerefentry>
and <citerefentry><refentrytitle>zfs</refentrytitle><manvolnum>8</manvolnum></citerefentry>
for more info.
'';
features.sendRaw = mkEnableOption ''
sendRaw feature which adds the options <literal>-w</literal> to the
<command>zfs send</command> command. For encrypted source datasets this
instructs zfs not to decrypt before sending which results in a remote
backup that can't be read without the encryption key/passphrase, useful
when the remote isn't fully trusted or not physically secure. This
option must be used consistently, raw incrementals cannot be based on
non-raw snapshots and vice versa.
'';
features.skipIntermediates = mkEnableOption ''
Enable the skipIntermediates feature to send a single increment
between latest common snapshot and the newly made one. It may skip
several source snaps if the destination was offline for some time, and
it should skip snapshots not managed by znapzend. Normally for online
destinations, the new snapshot is sent as soon as it is created on the
source, so there are no automatic increments to skip.
'';
features.lowmemRecurse = mkEnableOption ''
use lowmemRecurse on systems where you have too many datasets, so a
recursive listing of attributes to find backup plans exhausts the
memory available to <command>znapzend</command>: instead, go the slower
way to first list all impacted dataset names, and then query their
configs one by one.
'';
features.zfsGetType = mkEnableOption ''
use zfsGetType if your <command>zfs get</command> supports a
<literal>-t</literal> argument for filtering by dataset type at all AND
lists properties for snapshots by default when recursing, so that there
is too much data to process while searching for backup plans.
If these two conditions apply to your system, the time needed for a
<literal>--recursive</literal> search for backup plans can literally
differ by hundreds of times (depending on the amount of snapshots in
that dataset tree... and a decent backup plan will ensure you have a lot
of those), so you would benefit from requesting this feature.
'';
};
};
config = mkIf cfg.enable {
environment.systemPackages = [ pkgs.znapzend ];
systemd.services = {
znapzend = {
description = "ZnapZend - ZFS Backup System";
wantedBy = [ "zfs.target" ];
after = [ "zfs.target" ];
path = with pkgs; [ zfs mbuffer openssh ];
preStart = optionalString cfg.pure ''
echo Resetting znapzend zetups
${pkgs.znapzend}/bin/znapzendzetup list \
| grep -oP '(?<=\*\*\* backup plan: ).*(?= \*\*\*)' \
| xargs -I{} ${pkgs.znapzend}/bin/znapzendzetup delete "{}"
'' + concatStringsSep "\n" (mapAttrsToList (dataset: config: ''
echo Importing znapzend zetup ${config} for dataset ${dataset}
${pkgs.znapzend}/bin/znapzendzetup import --write ${dataset} ${config} &
'') files) + ''
wait
'';
serviceConfig = {
# znapzendzetup --import apparently tries to connect to the backup
# host 3 times with a timeout of 30 seconds, leading to a startup
# delay of >90s when the host is down, which is just above the default
# service timeout of 90 seconds. Increase the timeout so it doesn't
# make the service fail in that case.
TimeoutStartSec = 180;
# Needs to have write access to ZFS
User = "root";
ExecStart = let
args = concatStringsSep " " [
"--logto=${cfg.logTo}"
"--loglevel=${cfg.logLevel}"
(optionalString cfg.noDestroy "--nodestroy")
(optionalString cfg.autoCreation "--autoCreation")
(optionalString (enabledFeatures != [])
"--features=${concatStringsSep "," enabledFeatures}")
]; in "${pkgs.znapzend}/bin/znapzend ${args}";
ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID";
Restart = "on-failure";
};
};
};
};
meta.maintainers = with maintainers; [ infinisil SlothOfAnarchy ];
}