tailscale: clean up the IP routes so that it can coexist with by home wireguard network

hosts/modules/roles/work/tailscale-iproute2/default.nix

@@ -0,0 +1,18 @@
+{
+  iproute2,
+  static-nix-shell,
+  symlinkJoin,
+}:
+let
+  ipCmd = static-nix-shell.mkYsh {
+    pname = "ip";
+    pkgs = [ "iproute2" "systemdMinimal" ];
+    srcRoot = ./.;
+  };
+in symlinkJoin {
+  name = "tailscale-iproute2";
+  paths = [
+    ipCmd
+    iproute2
+  ];
+}

hosts/modules/roles/work/tailscale-iproute2/ip

@@ -0,0 +1,68 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i ysh -p iproute2 -p oils-for-unix -p systemdMinimal
+
+# this script intercepts tailscale's calls into the `ip` tool.
+# example invocations:
+# ip rule
+# ip addr add 100.12.34.56/32 dev tailscale0
+# ip -4 rule add pref 5210 fwmark 0x80000/0xff0000 table main
+# ip -4 rule add pref 5270 table 52
+# ip link set dev tailscale0 up
+# ip route add 100.23.145.167/32 dev tailscale0 table 52
+# ip route add 10.100.0.0/16 dev tailscale0 table 52
+#
+# ip -4 rule del pref 5250 type unreachable
+# ip -4 rule del pref 5210 table main
+
+
+proc log(...args) {
+  echo "[tailscale-iproute2]" @args | systemd-cat --identifier=tailscaled
+  echo "[tailscale-iproute2]" @args >&2
+}
+
+log ip @ARGV
+
+func isPermitted(...args) {
+  for a in (args) {
+    case (a) {
+      route | rule {
+        # DON'T allow these operations:
+        # - modify `route`s
+        # - add `rule`s to perform lookups in other tables, sometimes matched by fwmark
+        #   - these should be safe, but since the tables themselves stay empty, it's just useless spam
+        #     and makes debugging trickier
+        #   - note that tailscale uses empty `ip rule` invocation to test for support;
+        #     doesn't parse the output, just the exit code.
+        return (false)
+      }
+      addr | link {
+        # DO allow these operations:
+        # - `add` addresses to the device
+        # - bring up/down tailscale0 `link` device
+        return (true)
+      }
+    }
+  }
+
+  log "UNRECOGNIZED IP COMMAND" @args
+  return (false)
+}
+
+if (not isPermitted(...ARGV)) {
+  log "command not permitted; exiting 0:" @ARGV
+  exit 0
+}
+
+var me = $(realpath "$_this_dir/ip")
+for p in (ENV.PATH => split(":")) {
+  if test -x "$p/ip" {
+    var them = $(realpath "$p/ip")
+    if (me !== them) {
+      log "forwarding request:" "$them" @ARGV
+      exec "$them" @ARGV
+    }
+  }
+}
+
+log "NO IPROUTE2 FOUND"
+exit 1

hosts/modules/roles/work/tailscale.nix

@@ -1,16 +1,151 @@
 # first run:
 # - `sudo tailscale login --hostname $myHostname`
-{ config, lib, ... }:
+#
+# N.B.: manage with:
+# - `systemctl {stop,start} tailscaled`
+#   NOT `tailscale {down,up}`
+#   the latter isn't compatible with my ip routing patches, below
+{ config, lib, pkgs, ... }:
+let
+  ip = lib.getExe' pkgs.iproute2 "ip";
+  ### TAILSCALE ROUTING
+  # - tailscale maintains the following `ip rule`s:
+  #   - 5210: from all fwmark 0x80000/0xff0000 lookup main
+  #   - 5230: from all fwmark 0x80000/0xff0000 lookup default
+  #   - 5250: from all fwmark 0x80000/0xff0000 unreachable
+  #   - 5270: from all lookup 52
+  # - tailscale _always_ adds its wireguard peers (100.64.0.0/10) to table 52
+  #   - view with: `ip route showq table 52`
+  # - tailscale _conditionally_ adds routes (to _any_ destination address) _via_ these peers, to table 52.
+  #   - THIS is what the `--accept-routes` argument does.
+  #   - these routes are critical even for DNS.
+  #   - `--accept-routes` seems to impact both tailscale-internal behavior AND iptables;
+  #     hence it's NOT possible to omit `--accept-routes` and then manually define the routes i want.
+  # - peer-advertised routes (via `--accept-routes`) often conflict with pre-existing local routes.
+  #   e.g. in the 10.0.0.0/8 range.
+  # - official way to handle conflicting routes is by manually making higher-precedence ip rules for what i care about:
+  #   - <https://tailscale.com/kb/1023/troubleshooting#lan-traffic-prioritization-with-overlapping-subnet-routes>
+  # - workarounds for conflicting routes is to provide tailscale a custom `ip` tool for it to shell out to:
+  #   - <https://github.com/tailscale/tailscale/issues/6231#issuecomment-1420912939>
+  #
+  # HOW I CONFIGURE TAILSCALE ROUTING:
+  # - provide `--accept-routes`
+  # - override the `ip` tool such that tailscale doesn't actually modify the routing table.
+  # - explicitly configure the range of routes i actually want.
+  tailscale = let
+    iproute2' = pkgs.callPackage ./tailscale-iproute2 { };
+    # tailscale package wraps binaries with `--prefix PATH ${iproute2}/bin`.
+    # tailscale takes 1m to compile, 5m to run tests => slow to iterate.
+    # instead, remove iproute2 from tailscale,
+    # then re-wrap the binaries with my custom iproute2, separately.
+    tailscaleNoIproute2 = pkgs.tailscale.override {
+      iproute2 = null;
+      makeWrapper = pkgs.makeBinaryWrapper;  #< only BinaryWrapper handles `--inherit-argv0` correctly
+    };
+  in pkgs.stdenvNoCC.mkDerivation {
+    inherit (tailscaleNoIproute2) pname version;
+    nativeBuildInputs = [
+      pkgs.makeBinaryWrapper  #< only BinaryWrapper handles `--inherit-argv0` correctly
+    ];
+    dontUnpack = true;
+    dontBuild = true;
+    installPhase = ''
+      mkdir -p $out
+
+      mkdir -p $out/lib/systemd/system
+      substitute ${tailscaleNoIproute2}/lib/systemd/system/tailscaled.service $out/lib/systemd/system/tailscaled.service \
+        --replace-fail ${tailscaleNoIproute2} $out
+      ln -s ${tailscaleNoIproute2}/share $out/share
+
+      mkdir -p $out/bin
+      ln -s ${tailscaleNoIproute2}/bin/get-authkey $out/bin/get-authkey
+      ln -s tailscaled $out/bin/tailscale
+      ln -s ${tailscaleNoIproute2}/bin/tailscaled $out/bin/tailscaled
+      ln -s ${tailscaleNoIproute2}/bin/tsidp $out/bin/tsidp
+
+      wrapProgram $out/bin/tailscaled \
+        --inherit-argv0 \
+        --prefix PATH : ${iproute2'}/bin
+    '';
+
+    passthru.iproute2 = iproute2';
+  };
+in
 {
   config = lib.mkIf config.sane.roles.work {
     sane.persist.sys.byStore.private = [
       { user = "root"; group = "root"; mode = "0700"; path = "/var/lib/tailscale"; method = "bind"; }
     ];
     services.tailscale.enable = true;
+
+    services.tailscale.package = tailscale;
+    systemd.services.tailscaled.environment.TS_DEBUG_USE_IP_COMMAND = "1";
+
+    # "statically" configure the routes to tailscale.
+    # tailscale doesn't use the kernel wireguard module,
+    # but a userspace `wireguard-go` (coupled with `/dev/net/tun`, or a pure
+    # pasta-style TCP/UDP userspace dev).
+    #
+    # it therefore appears as an "unmanaged" device to network managers like systemd-networkd.
+    # in order to configure routes, we have to script it.
+    systemd.services.tailscaled.serviceConfig.ExecStartPost = [
+      (pkgs.writeShellScript "tailscaled-add-routes" ''
+        while ! ${lib.getExe' tailscale "tailscale"} status ; do
+          echo "tailscale not ready"
+          sleep 2
+        done
+        ${ip} route add table main 10.0.0.0/8 dev tailscale0 scope global
+        ${ip} route add table main 100.64.0.0/10 dev tailscale0 scope global
+      '')
+    ];
+    systemd.services.tailscaled.preStop = ''
+      ${ip} route del table main 10.0.0.0/8 dev tailscale0 scope global || true
+      ${ip} route del table main 100.64.0.0/10 dev tailscale0 scope global || true
+    '';
+    # systemd.network.networks."50-tailscale" = {
+    #   # see: `man 5 systemd.network`
+    #   matchConfig.Name = "tailscale0";
+    #   routes = [
+    #     # {
+    #     #   Scope = "global";
+    #     #   # 0.0.0.0/8 is a reserved-for-local-network range in IPv4
+    #     #   Destination = "0.0.0.0/8";
+    #     # }
+    #     {
+    #       Scope = "global";
+    #       # Scope = "link";
+    #       # 10.0.0.0/8 is a reserved-for-private-networks range in IPv4
+    #       Destination = "10.0.0.0/8";
+    #     }
+    #     {
+    #       Scope = "global";
+    #       # Scope = "link";
+    #       # 100.64.0.0/10 is a reserved range in IPv4
+    #       Destination = "100.64.0.0/10";
+    #     }
+    #   ];
+    #   # RequiredForOnline => should `systemd-networkd-wait-online` fail if this network can't come up?
+    #   linkConfig.RequiredForOnline = false;
+    #   linkConfig.Unmanaged = lib.mkForce false;  #< tailscale nixos module declares this as unmanaged
+    # };
+
     # services.tailscale.useRoutingFeatures = "client";
     services.tailscale.extraSetFlags = [
+      # --accept-routes does _two_ things:
+      # 1. allows tailscale to discover, internally, how to route to peers-of-peers.
+      # 2. instructs tailscale to tell the kernel to route discovered routes through the tailscale0 device.
+      # even if i disable #2, i still need --accept-routes to provide #1.
       "--accept-routes"
       # "--operator=colin"  #< this *should* allow non-root control, but fails: <https://github.com/tailscale/tailscale/issues/16080>
+      # lock the preferences i care about, because even if they're default i think they _might_ be conditional on admin policy:
+      "--accept-dns=false"  #< i manage manually, with BIND
+      # "--accept-routes=false"
+      "--advertise-connector=false"
+      "--advertise-exit-node=false"
+      # "--auto-update=false"  # "automatic updates are not supported on this platform"
+      "--ssh=false"
+      "--update-check=false"
+      "--webclient=false"
     ];
     services.tailscale.extraDaemonFlags = [
       "-verbose" "7"
@@ -18,6 +153,7 @@
     services.bind.extraConfig = ''
       include "${config.sops.secrets."tailscale-work-zones-bind.conf".path}";
     '';
+
     systemd.services.tailscaled = {
       # systemd hardening (systemd-analyze security tailscaled.service)
       serviceConfig.AmbientCapabilities = "CAP_NET_ADMIN";