From 0d99293b2f000e1c5e1f3e2957e2e33290e52a91 Mon Sep 17 00:00:00 2001 From: Colin Date: Mon, 17 Jun 2024 08:34:39 +0000 Subject: [PATCH] servo: split the doof/ovpns netns config into its own module a big thing this gets me is that the attributes (like IP addresses) are now accessible via 'config' an i won't have to hardcode them so much --- hosts/by-name/servo/net.nix | 112 ++++++--------------------------- modules/default.nix | 1 + modules/netns.nix | 120 ++++++++++++++++++++++++++++++++++++ 3 files changed, 140 insertions(+), 93 deletions(-) create mode 100644 modules/netns.nix diff --git a/hosts/by-name/servo/net.nix b/hosts/by-name/servo/net.nix index 79dec00d..1a041499 100644 --- a/hosts/by-name/servo/net.nix +++ b/hosts/by-name/servo/net.nix @@ -19,81 +19,11 @@ let }; }; }; - - bridgedWireguardNamespace = { name, ip4, peers, privateKeyFile, vethSubnet, vpnDns }: let - ip = "${pkgs.iproute2}/bin/ip"; - in-ns = "${ip} netns exec ${name}"; - iptables = "${pkgs.iptables}/bin/iptables"; - veth-host-ip = "${vethSubnet}.5"; # "x.y.z.5", "x.y.z.6" is legacy: some things elsewhere assume this for ovpns - veth-local-ip = "${vethSubnet}.6"; - bridgePort = port: proto: '' - ${in-ns} ${iptables} -A PREROUTING -t nat -p ${proto} --dport ${port} -m iprange --dst-range ${ip4} \ - -j DNAT --to-destination ${veth-host-ip} - ''; - bridgeStatements = lib.foldlAttrs - (acc: port: portCfg: acc ++ (builtins.map (bridgePort port) portCfg.protocol)) - [] - (lib.filterAttrs - (port: portCfg: portCfg.visibleTo."${name}") - config.sane.ports.ports - ) - ; - in { - inherit peers privateKeyFile; #< passthrough wireguard config - interfaceNamespace = name; - ips = [ "${ip4}/32" ]; - - preSetup = '' - ${ip} netns add ${name} || (test -e /run/netns/${name} && echo "${name} already exists") - ''; - - postSetup = '' - # DOCS: - # - some of this approach is described here: - # - iptables primer: - # create veth pair - ${ip} link add ${name}-veth-a type veth peer name ${name}-veth-b - ${ip} addr add ${veth-host-ip}/24 dev ${name}-veth-a - ${ip} link set ${name}-veth-a up - - # move veth-b into the namespace - ${ip} link set ${name}-veth-b netns ${name} - ${in-ns} ip addr add ${veth-local-ip}/24 dev ${name}-veth-b - ${in-ns} ip link set ${name}-veth-b up - - # make it so traffic originating from the host side of the veth - # is sent over the veth no matter its destination. - ${ip} rule add from ${veth-host-ip} lookup ${name} pref 50 - - # for traffic originating at the host veth to the WAN, use the veth as our gateway - # not sure if the metric 1002 matters. - ${ip} route add default via ${veth-local-ip} dev ${name}-veth-a proto kernel src ${veth-host-ip} metric 1002 table ${name} - # give the default route lower priority - ${ip} rule add from all lookup local pref 100 - ${ip} rule del from all lookup local pref 0 - - # in order to access DNS in this netns, we need to route it to the VPN's nameservers - # - alternatively, we could fix DNS servers like 1.1.1.1. - ${in-ns} ${iptables} -A OUTPUT -t nat -p udp --dport 53 -m iprange --dst-range 127.0.0.53 \ - -j DNAT --to-destination ${vpnDns}:53 - '' + (lib.concatStringsSep "\n" bridgeStatements); - - postShutdown = '' - ${in-ns} ip link del ${name}-veth-b || echo "couldn't delete ${name}-veth-b" - ${ip} link del ${name}-veth-a || echo "couldn't delete ${name}-veth-a" - ${ip} netns delete ${name} || echo "couldn't delete ${name}" - # restore rules/routes - ${ip} rule del from ${veth-host-ip} lookup ${name} pref 50 || echo "couldn't delete init -> ${name} rule" - ${ip} route del default via ${veth-local-ip} dev ${name}-veth-a proto kernel src ${veth-host-ip} metric 1002 table ${name} || echo "couldn't delete init -> ${name} route" - ${ip} rule add from all lookup local pref 0 - ${ip} rule del from all lookup local pref 100 - ''; - }; in { options = with lib; { sane.ports.ports = mkOption { - # add the `visibleTo.ovpns` option + # add the `visibleTo.{doof,ovpns}` options type = types.attrsOf portOpts; }; }; @@ -121,14 +51,16 @@ in # tun-sea config sane.dns.zones."uninsane.org".inet.A."doof.tunnel" = "205.201.63.12"; # sane.dns.zones."uninsane.org".inet.AAAA."doof.tunnel" = "2602:fce8:106::51"; #< TODO: enable IPv6 - networking.wireguard.interfaces.wg-doof = bridgedWireguardNamespace { + networking.wireguard.interfaces.wg-doof = { privateKeyFile = config.sops.secrets.wg_doof_privkey.path; # wg is active only in this namespace. # run e.g. ip netns exec doof # sudo ip netns exec doof ping www.google.com - name = "doof"; - ip4 = "205.201.63.12"; - # ip6 = "2602:fce8:106::51/128" #< TODO: enable IPv6 + interfaceNamespace = "doof"; + ips = [ + "205.201.63.12" + # "2602:fce8:106::51/128" #< TODO: enable IPv6 + ]; peers = [ { publicKey = "nuESyYEJ3YU0hTZZgAd7iHBz1ytWBVM5PjEL1VEoTkU="; @@ -139,23 +71,24 @@ in persistentKeepalive = 25; #< keep the NAT alive } ]; - - vethSubnet = "10.0.2"; #< 10.0.2.x is used for forwarding traffic between the root namespace and the VPN namespace - vpnDns = "1.1.1.1"; #< DNS requests inside the namespace are forwarded here (TODO: forward to the init namespace resolver) }; + sane.netns.doof.hostVethIpv4 = "10.0.2.5"; + sane.netns.doof.netnsVethIpv4 = "10.0.2.6"; + sane.netns.doof.netnsPubIpv4 = "205.201.63.12"; + sane.netns.doof.routeTable = 12; # OVPN CONFIG (https://www.ovpn.com): # DOCS: https://nixos.wiki/wiki/WireGuard # if you `systemctl restart wireguard-wg-ovpns`, make sure to also restart any other services in `NetworkNamespacePath = .../ovpns`. # TODO: why not create the namespace as a seperate operation (nix config for that?) networking.wireguard.enable = true; - networking.wireguard.interfaces.wg-ovpns = bridgedWireguardNamespace { + networking.wireguard.interfaces.wg-ovpns = { privateKeyFile = config.sops.secrets.wg_ovpns_privkey.path; # wg is active only in this namespace. # run e.g. ip netns exec ovpns # sudo ip netns exec ovpns ping www.google.com - name = "ovpns"; - ip4 = "185.157.162.178"; + interfaceNamespace = "ovpns"; + ips = [ "185.157.162.178" ]; peers = [ { publicKey = "SkkEZDCBde22KTs/Hc7FWvDBfdOCQA4YtBEuC3n5KGs="; @@ -173,18 +106,11 @@ in # dynamicEndpointRefreshRestartSeconds = 5; } ]; - - vethSubnet = "10.0.1"; #< 10.0.1.x is used for forwarding traffic between the root namespace and the VPN namespace - vpnDns = "46.227.67.134"; #< DNS requests inside the namespace are forwarded here }; - - # create a new routing table that we can use to proxy traffic out of the root namespace - # through the wireguard namespaces, and to the WAN via VPN. - # i think the numbers here aren't particularly important. - networking.iproute2.rttablesExtraConfig = '' - 11 ovpns - 12 doof - ''; - networking.iproute2.enable = true; + sane.netns.ovpns.hostVethIpv4 = "10.0.1.5"; + sane.netns.ovpns.netnsVethIpv4 = "10.0.1.6"; + sane.netns.ovpns.netnsPubIpv4 = "185.157.162.178"; + sane.netns.ovpns.routeTable = 11; + sane.netns.ovpns.dns = "46.227.67.134"; #< DNS requests inside the namespace are forwarded here }; } diff --git a/modules/default.nix b/modules/default.nix index 47e57f68..68afa7c2 100644 --- a/modules/default.nix +++ b/modules/default.nix @@ -8,6 +8,7 @@ ./ids.nix ./programs ./image.nix + ./netns.nix ./persist ./ports.nix ./root-on-tmpfs.nix diff --git a/modules/netns.nix b/modules/netns.nix new file mode 100644 index 00000000..8145c544 --- /dev/null +++ b/modules/netns.nix @@ -0,0 +1,120 @@ +{ config, lib, pkgs, sane-lib, ... }: +let + cfg = config.sane.netns; + netnsOpts = with lib; types.submodule { + options = { + dns = mkOption { + type = types.str; + default = "1.1.1.1"; #< TODO: make the default be to forward DNS queries to the init namespace. + }; + hostVethIpv4 = mkOption { + type = types.str; + }; + netnsVethIpv4 = mkOption { + type = types.str; + }; + netnsPubIpv4 = mkOption { + type = types.str; + }; + routeTable = mkOption { + type = types.int; + description = '' + numeric ID for iproute2 (0-255?). + each netns gets its own routing table so that i can route a packet out by placing it in the table. + ''; + }; + }; + }; + mkNetNsConfig = name: opts: with opts; { + networking.localCommands = let + iptables = "${pkgs.iptables}/bin/iptables"; + in-ns = "ip netns exec ${name}"; + bridgePort = port: proto: '' + ${in-ns} ${iptables} -A PREROUTING -t nat -p ${proto} --dport ${port} -m iprange --dst-range ${netnsPubIpv4} \ + -j DNAT --to-destination ${hostVethIpv4} + ''; + bridgeStatements = lib.foldlAttrs + (acc: port: portCfg: acc ++ (builtins.map (bridgePort port) portCfg.protocol)) + [] + (lib.filterAttrs + (port: portCfg: portCfg.visibleTo."${name}") + config.sane.ports.ports + ) + ; + in '' + ip netns add ${name} || (test -e /run/netns/${name} && echo "${name} already exists") + # DOCS: + # - some of this approach is described here: + # - iptables primer: + # create veth pair + ip link add ${name}-veth-a type veth peer name ${name}-veth-b || echo "${name}-veth-{a,b} aleady exists" + ip addr add ${hostVethIpv4}/24 dev ${name}-veth-a || echo "${name}-veth-a aleady has IP address" + ip link set ${name}-veth-a up + + # move veth-b into the namespace + ip link set ${name}-veth-b netns ${name} || echo "${name}-veth-b was already moved into its netns" + ${in-ns} ip addr add ${netnsVethIpv4}/24 dev ${name}-veth-b || echo "${name}-veth-b aleady has IP address" + ${in-ns} ip link set ${name}-veth-b up + + # make it so traffic originating from the host side of the veth + # is sent over the veth no matter its destination. + ip rule add from ${hostVethIpv4} lookup ${name} pref 50 || echo "${name} already has ip rules (pref 50)" + + # for traffic originating at the host veth to the WAN, use the veth as our gateway + # not sure if the metric 1002 matters. + ip route add default via ${netnsVethIpv4} dev ${name}-veth-a proto kernel src ${hostVethIpv4} metric 1002 table ${name} || \ + echo "${name} already has default route" + # give the default route lower priority + ip rule add from all lookup local pref 100 || echo "${name}: already has ip rules (pref 100)" + ip rule del from all lookup local pref 0 || echo "${name}: already removed ip rule of default lookup (pref 0)" + + # in order to access DNS in this netns, we need to route it to the VPN's nameservers + # - alternatively, we could fix DNS servers like 1.1.1.1. + ${in-ns} ${iptables} -A OUTPUT -t nat -p udp --dport 53 -m iprange --dst-range 127.0.0.53 \ + -j DNAT --to-destination ${dns}:53 + '' + (lib.concatStringsSep "\n" bridgeStatements); + + # postShutdown = '' + # ${in-ns} ip link del ${name}-veth-b || echo "couldn't delete ${name}-veth-b" + # ip link del ${name}-veth-a || echo "couldn't delete ${name}-veth-a" + # ip netns delete ${name} || echo "couldn't delete ${name}" + # # restore rules/routes + # ip rule del from ${veth-host-ip} lookup ${name} pref 50 || echo "couldn't delete init -> ${name} rule" + # ip route del default via ${veth-local-ip} dev ${name}-veth-a proto kernel src ${veth-host-ip} metric 1002 table ${name} || echo "couldn't delete init > #{name} route" + # ip rule add from all lookup local pref 0 + # ip rule del from all lookup local pref 100 + # ''; + + # specifically, we need to set these up before wireguard-wg-*, + # whose unit files are BEFORE "network.target", and therefore + # ordered ambiguously w.r.t. network-local-commands (a dep of "network.target"). + systemd.services.network-local-commands.wantedBy = [ "network-pre.target" ]; + systemd.services.network-local-commands.before = [ "network-pre.target" ]; + + # create a new routing table that we can use to proxy traffic out of the root namespace + # through the wireguard namespaces, and to the WAN via VPN. + # i think the numbers here aren't particularly important. + networking.iproute2.rttablesExtraConfig = '' + ${builtins.toString routeTable} ${name} + ''; + networking.iproute2.enable = true; + }; +in +{ + options = with lib; { + sane.netns = mkOption { + type = types.attrsOf netnsOpts; + default = {}; + }; + }; + + config = let + configs = lib.mapAttrsToList mkNetNsConfig cfg; + take = f: { + networking.localCommands = f.networking.localCommands; + networking.iproute2.rttablesExtraConfig = f.networking.iproute2.rttablesExtraConfig; + networking.iproute2.enable = f.networking.iproute2.enable; + systemd.services.network-local-commands = f.systemd.services.network-local-commands; + }; + in take (sane-lib.mkTypedMerge take configs); +}