2023-09-19 16:03:20 +00:00
{ config , lib , pkgs , . . . }:
2022-06-10 00:41:03 +00:00
2022-12-13 03:45:49 +00:00
# to add a new OVPN VPN:
# - generate a privkey `wg genkey`
# - add this key to `sops secrets/universal.yaml`
2024-01-16 03:20:40 +00:00
# - upload pubkey to OVPN.com (`cat wg.priv | wg pubkey`)
2022-12-13 03:45:49 +00:00
# - generate config @ OVPN.com
# - copy the Address, PublicKey, Endpoint from OVPN's config
2024-01-16 03:20:40 +00:00
#
# debugging:
# - `journalctl -u systemd-networkd`
#
# docs:
# - wireguard (nixos): <https://nixos.wiki/wiki/WireGuard#Setting_up_WireGuard_with_systemd-networkd>
# - wireguard (arch): <https://wiki.archlinux.org/title/WireGuard>
2022-12-13 03:17:27 +00:00
let
2024-01-19 09:54:01 +00:00
def-wg-vpn = name : { endpoint , publicKey , addrV4 , bridgeAddrV4 , dns , privateKeyFile , id }: {
# id is a number < 2^15 which has to be unique in iproute and as a fwmark.
# my own convention is > 10000
systemd . network . netdevs . " 9 8 - ${ name } " = {
2024-01-15 04:15:17 +00:00
# see: `man 5 systemd.netdev`
2024-01-16 03:20:40 +00:00
netdevConfig = {
Kind = " w i r e g u a r d " ;
Name = name ;
} ;
2024-01-15 04:15:17 +00:00
wireguardConfig = {
PrivateKeyFile = privateKeyFile ;
2024-01-19 09:54:01 +00:00
FirewallMark = id ;
2024-01-15 04:15:17 +00:00
} ;
wireguardPeers = [ {
2024-01-16 03:20:40 +00:00
wireguardPeerConfig = {
AllowedIPs = [
" 0 . 0 . 0 . 0 / 0 "
" : : / 0 "
] ;
Endpoint = endpoint ;
PublicKey = publicKey ;
} ;
2024-01-15 04:15:17 +00:00
} ] ;
} ;
2024-01-16 03:20:40 +00:00
systemd . network . networks . " 5 0 - ${ name } " = {
2024-01-15 04:15:17 +00:00
# see: `man 5 systemd.network`
matchConfig . Name = name ;
2024-01-19 09:54:01 +00:00
networkConfig . Address = [ addrV4 ] ;
2024-01-15 04:15:17 +00:00
networkConfig . DNS = dns ;
2024-01-19 09:58:13 +00:00
# TODO: `sane-vpn up <vpn>` should configure DNS to be sent over the VPN
2024-01-16 03:20:40 +00:00
# DNSDefaultRoute: system DNS queries are sent to this link's DNS server
2024-01-19 09:54:01 +00:00
# networkConfig.DNSDefaultRoute = true;
2024-01-16 03:20:40 +00:00
# Domains = ~.: system DNS queries are sent to this link's DNS server
2024-01-19 09:54:01 +00:00
# networkConfig.Domains = "~.";
2024-01-16 03:20:40 +00:00
routingPolicyRules = [
{
routingPolicyRuleConfig = {
2024-01-19 09:54:01 +00:00
# send traffic from the the container bridge out over the VPN.
# the bridge itself does source nat (SNAT) to rewrite the packet source address to that of the VPNs
# -- but that happens in POSTROUTING: *after* the kernel decides which interface to give the packet to next.
# therefore, we have to route here based on the packet's address as it is in PREROUTING, i.e. the bridge address. weird!
Priority = id ;
From = bridgeAddrV4 ;
Table = id ;
2024-01-16 03:20:40 +00:00
} ;
}
] ;
routes = [ {
2024-01-19 09:54:01 +00:00
routeConfig . Table = id ;
2024-01-16 03:20:40 +00:00
routeConfig . Scope = " l i n k " ;
routeConfig . Destination = " 0 . 0 . 0 . 0 / 0 " ;
2024-01-19 09:54:01 +00:00
routeConfig . Source = addrV4 ;
2024-01-16 03:20:40 +00:00
} ] ;
# RequiredForOnline => should `systemd-networkd-wait-online` fail if this network can't come up?
linkConfig . RequiredForOnline = false ;
2024-01-19 09:54:01 +00:00
} ;
systemd . network . netdevs . " 9 9 - b r - ${ name } " = {
netdevConfig . Kind = " b r i d g e " ;
netdevConfig . Name = " b r - ${ name } " ;
} ;
systemd . network . networks . " 5 1 - b r - ${ name } " = {
matchConfig . Name = " b r - ${ name } " ;
networkConfig . Description = " N A T s i n b o u n d t r a f f i c t o ${ name } , i n t e n d e d f o r c o n t a i n e r i s o l a t i o n " ;
networkConfig . Address = [ bridgeAddrV4 ] ;
networkConfig . DNS = dns ;
# ConfigureWithoutCarrier: a bridge with no attached interface has no carrier (this is to be expected).
# systemd ordinarily waits for a carrier to be present before "configuring" the bridge.
# i tell it to not do that, so that i can assign an IP address to the bridge before i connect it to anything.
# alternative may be to issue the bridge a static MACAddress?
# see: <https://github.com/systemd/systemd/issues/9252#issuecomment-808710185>
networkConfig . ConfigureWithoutCarrier = true ;
# networkConfig.DHCPServer = true;
linkConfig . RequiredForOnline = false ;
} ;
networking . localCommands = with pkgs ; ''
# view rules with:
# - `sudo iptables -t nat --list-rules -v`
# rewrite the source address of every packet leaving the container so that it's routable by the wg tunnel.
# note that this alone doesn't get it routed *to* the wg device. we can't, since SNAT is POSTROUTING (not PREROUTING).
# that part's done by an `ip rule` entry.
$ { iptables } /bin/iptables - A POSTROUTING - t nat - s $ { bridgeAddrV4 } - j SNAT - - to-source $ { addrV4 }
'' ;
# linux will drop inbound packets if it thinks a reply to that packet wouldn't exit via the same interface (rpfilter).
# wg-quick has a solution via `iptables -j CONNMARK`, and that does work for system-wide VPNs,
# but i couldn't get that to work for firejail/netns with SNAT, so set rpfilter to "loose".
networking . firewall . checkReversePath = " l o o s e " ;
# networking.firewall.extraCommands = with pkgs; ''
# # wireguard packet marking. without this, rpfilter drops responses from a wireguard VPN
# # because the "reverse path check" fails (i.e. it thinks a response to the packet would go out via a different interface than what the wireguard packet arrived at).
# # debug with e.g. `iptables --list -v -n -t mangle`
# # - and `networking.firewall.logReversePathDrops = true;`, `networking.firewall.logRefusedPackets = true;`
# # - and `journalctl -k` to see dropped packets
# #
# # note that wg-quick also adds a rule to reject non-local traffic from all interfaces EXCEPT the tunnel.
# # that may protect against actors trying to probe us: actors we connect to via wireguard who send their response packets (speculatively) to our plaintext IP to see if we accept them.
# # but that's fairly low concern, and firewalling by the gateway/NAT helps protect against that already.
# ${iptables}/bin/iptables -t mangle -I PREROUTING 1 -i ${name} -m mark --mark 0 -j CONNMARK --restore-mark
# ${iptables}/bin/iptables -t mangle -A POSTROUTING -o ${name} -m mark --mark ${builtins.toString id} -j CONNMARK --save-mark
# '';
systemd . services . " v p n - ${ name } " = {
path = with pkgs ; [ iproute2 ] ;
serviceConfig = let
prioMain = id - 200 ;
prioFwMark = id - 100 ;
mkScript = verb : pkgs . writeShellScript " v p n - ${ name } - ${ verb } " ''
# first, allow all non-default routes (prefix-length != 0) a chance to route the packet.
# - this allows the wireguard tunnel itself to pass traffic via our LAN gateway.
# - incidentally, it allows traffic to LAN devices and other machine-local or virtual networks.
ip rule $ { verb } from all lookup main suppress_prefixlength 0 priority $ { builtins . toString prioMain }
# if packet hasn't gone through the wg device yet (fwmark), then move it to the table which will cause it to.
ip rule $ { verb } not from all fwmark $ { builtins . toString id } lookup $ { builtins . toString id } priority $ { builtins . toString prioFwMark }
'' ;
in {
ExecStart = '' ${ mkScript " a d d " } '' ;
ExecStop = '' ${ mkScript " d e l " } '' ;
Type = " o n e s h o t " ;
RemainAfterExit = true ;
Restart = " o n - f a i l u r e " ;
# it'd be nice if this could depend on the network device we just declared, but there seems no way to do that except via 3rd-party tooling.
} ;
2024-01-15 03:52:31 +00:00
} ;
2022-06-10 00:41:03 +00:00
} ;
2024-01-19 09:54:01 +00:00
def-ovpn = name : { endpoint , publicKey , addrV4 , id }: ( def-wg-vpn " o v p n d - ${ name } " {
inherit endpoint publicKey addrV4 ;
id = 10000 + id ;
2023-09-19 15:29:47 +00:00
privateKeyFile = config . sops . secrets . " w g / o v p n d _ ${ name } _ p r i v k e y " . path ;
2024-01-19 09:54:01 +00:00
bridgeAddrV4 = " 1 0 . 2 0 . ${ builtins . toString id } . 1 / 2 4 " ;
2023-09-19 15:29:47 +00:00
dns = [
" 4 6 . 2 2 7 . 6 7 . 1 3 4 "
" 1 9 2 . 1 6 5 . 9 . 1 5 8 "
] ;
2024-01-16 03:20:40 +00:00
} ) // {
sops . secrets . " w g / o v p n d _ ${ name } _ p r i v k e y " = {
# needs to be readable by systemd-network or else it says "Ignoring network device" and doesn't expose it to networkctl.
owner = " s y s t e m d - n e t w o r k " ;
} ;
2023-09-19 15:29:47 +00:00
} ;
2022-12-13 03:45:49 +00:00
in lib . mkMerge [
( def-ovpn " u s " {
2022-12-13 03:17:27 +00:00
endpoint = " v p n 3 1 . p r d . l o s a n g e l e s . o v p n . c o m : 9 9 2 9 " ;
publicKey = " V W 6 b E W M O l O n e t a 1 b f 6 Y F E 2 5 N / o M G h 1 E 1 U F B C f y g g d 0 k = " ;
2024-01-19 09:54:01 +00:00
id = 1 ;
addrV4 = " 1 7 2 . 2 7 . 2 3 7 . 2 1 8 " ;
# addrV6 = "fd00:0000:1337:cafe:1111:1111:ab00:4c8f";
2022-12-13 03:45:49 +00:00
} )
2024-01-19 09:54:01 +00:00
# TODO: us-atl disabled until i can give it a different link-local address and wireguard key than us-mi
# (def-ovpn "us-atl" {
# endpoint = "vpn18.prd.atlanta.ovpn.com:9929";
# publicKey = "Dpg/4v5s9u0YbrXukfrMpkA+XQqKIFpf8ZFgyw0IkE0=";
# address = [
# "172.21.182.178/32"
# "fd00:0000:1337:cafe:1111:1111:cfcb:27e3/128"
# ];
# })
2022-12-13 04:26:00 +00:00
( def-ovpn " u s - m i " {
endpoint = " v p n 3 4 . p r d . m i a m i . o v p n . c o m : 9 9 2 9 " ;
publicKey = " V t J z 2 i r b u 8 m d k I Q v z l s Y h U + k 9 d 5 5 o r 9 m x 4 A 2 a 1 4 t 0 V 0 = " ;
2024-01-19 09:54:01 +00:00
id = 2 ;
addrV4 = " 1 7 2 . 2 1 . 1 8 2 . 1 7 8 " ;
# addrV6 = "fd00:0000:1337:cafe:1111:1111:cfcb:27e3";
2022-12-13 04:26:00 +00:00
} )
2022-12-13 03:45:49 +00:00
( def-ovpn " u k r " {
2022-12-13 03:17:27 +00:00
endpoint = " v p n 9 6 . p r d . k y i v . o v p n . c o m : 9 9 2 9 " ;
publicKey = " C j Z c X D x a a K p W 8 b 5 A s 1 E c N b I 6 + 4 2 A 6 B j W a h w X D C w f V F g = " ;
2024-01-19 09:54:01 +00:00
id = 3 ;
addrV4 = " 1 7 2 . 1 8 . 1 8 0 . 1 5 9 " ;
# addrV6 = "fd00:0000:1337:cafe:1111:1111:ec5c:add3";
2022-12-13 03:45:49 +00:00
} )
]