pipewire: switch back to bwrap sandboxing and document limitations

hosts/common/programs/pipewire.nix

@@ -1,6 +1,26 @@
 # administer with pw-cli, pw-mon, pw-top commands
 #
 # performance tuning: <https://gitlab.freedesktop.org/pipewire/pipewire/-/wikis/Performance-tuning>
+#
+# HAZARDS FOR MOBY:
+# - high-priority threads are liable to stall the lima GPU driver, and leave a half-functional OS state.
+# - symptom is messages like this (with stack traces) in dmesg or journalctl:
+#   - "[drm:lima_sched_timedout_job] *ERROR* lima job timeout"
+#   - and the UI locks up for a couple seconds, and then pipewire + wireplumber crash (but not pipewire-pulse)
+# - related, unconfirmed symptoms:
+#   - "sched: RT throttling activated"
+#   - "BUG: KFENCE: use-after-free read in vchan_complete"
+#     - this one seems to be recoverable
+# - likely to be triggered when using a small pipewire buffer (512 samples), by simple tasks like opening pavucontrol.
+#   - but a lengthier buffer is no sure way to dodge it: it will happen (less frequently) even for buffers of 2048 samples.
+# - seems ANY priority < 0 triggers this, independent of the `nice` setting.
+#   - i only tried SCHED_FIFO, not SCHED_RR (round robin) for the realtime threads.
+# - solution is some combination of:
+#   - DON'T USE RTKIT. rtkit only supports SCHED_FIFO and SCHED_RR: there's no way to use it only for adjusting `nice` values.
+#   - in pipewire.conf, remove all reference to libpipewire-module-rt.
+#     - it's loaded by default. i can either provide a custom pipewire.conf which doesn't load it, or adjust its config so that it intentionally fails.
+#     - without rtkit working, pipewire's module-rt doesn't allow niceness < -11. adjusting `nice`ness here seems to have little effect anyway.
+# - longer term, rtkit (or just rlimit based pipewire module-rt) would be cool to enable: it *does* reduce underruns.
 { config, lib, pkgs, ... }:
 let
   cfg = config.sane.programs.pipewire;
@@ -26,8 +46,8 @@ in
       "wireplumber"
     ];
 
-    sandbox.method = "landlock";  #< works, including without rtkit
+    # sandbox.method = "landlock";  #< works, including without rtkit
-    # sandbox.method = "bwrap";  #< also works, but can't claim the full scheduling priority it wants
+    sandbox.method = "bwrap";  #< also works, but can't claim the full scheduling priority it wants
     sandbox.whitelistAudio = true;
     # sandbox.whitelistDbus = [
     #   # dbus is used for rtkit integration
@@ -41,7 +61,7 @@ in
     # ];
     sandbox.wrapperType = "inplace";  #< its config files refer to its binaries by full path
     sandbox.extraConfig = [
-      "--sane-sandbox-keep-namespace" "pid"  #< required for rtkit
+      "--sane-sandbox-keep-namespace" "pid"
     ];
     sandbox.capabilities = [
       # if rtkit isn't present, and sandboxing is via landlock, these capabilities allow pipewire to claim higher scheduling priority
@@ -83,42 +103,44 @@ in
         default.clock.max-quantum = ${builtins.toString cfg.config.max-quantum}
       }
     '';
-    fs.".config/pipewire/pipewire.conf.d/20-sane-rtkit.conf".symlink.text = ''
+    # reduce realtime scheduling priority to prevent GPU instability,
-      # documented inside <repo:pipewire/pipewire:/src/modules/module-rt.c>
+    # but see the top of this file for other solutions.
-      context.modules = [{
+    # fs.".config/pipewire/pipewire.conf.d/20-sane-rtkit.conf".symlink.text = ''
-        name = libpipewire-module-rt
+    #   # documented inside <repo:pipewire/pipewire:/src/modules/module-rt.c>
-        args = {
+    #   context.modules = [{
-          nice.level   = 0
+    #     name = libpipewire-module-rt
-          rt.prio      = 0
+    #     args = {
-          #rt.time.soft = -1
+    #       nice.level   = 0
-          #rt.time.hard = -1
+    #       rt.prio      = 0
-          rlimits.enabled = false
+    #       #rt.time.soft = -1
-          rtportal.enabled = false
+    #       #rt.time.hard = -1
-          rtkit.enabled = true
+    #       rlimits.enabled = false
-          #uclamp.min = 0
+    #       rtportal.enabled = false
-          #uclamp.max = 1024
+    #       rtkit.enabled = true
-        }
+    #       #uclamp.min = 0
-        flags = [ ifexists nofail ]
+    #       #uclamp.max = 1024
-      }]
+    #     }
-    '';
+    #     flags = [ ifexists nofail ]
-    fs.".config/pipewire/pipewire-pulse.conf.d/20-sane-rtkit.conf".symlink.text = ''
+    #   }]
-      # documented: <repo:pipewire/pipewire:src/daemon/pipewire-pulse.conf.in>
+    # '';
-      context.modules = [{
+    # fs.".config/pipewire/pipewire-pulse.conf.d/20-sane-rtkit.conf".symlink.text = ''
-        name = libpipewire-module-rt
+    #   # documented: <repo:pipewire/pipewire:src/daemon/pipewire-pulse.conf.in>
-        args = {
+    #   context.modules = [{
-          nice.level   = 0
+    #     name = libpipewire-module-rt
-          rt.prio      = 0
+    #     args = {
-          #rt.time.soft = -1
+    #       nice.level   = 0
-          #rt.time.hard = -1
+    #       rt.prio      = 0
-          rlimits.enabled = false
+    #       #rt.time.soft = -1
-          rtportal.enabled = false
+    #       #rt.time.hard = -1
-          rtkit.enabled = true
+    #       rlimits.enabled = false
-          #uclamp.min = 0
+    #       rtportal.enabled = false
-          #uclamp.max = 1024
+    #       rtkit.enabled = true
-        }
+    #       #uclamp.min = 0
-        flags = [ ifexists nofail ]
+    #       #uclamp.max = 1024
-      }]
+    #     }
-    '';
+    #     flags = [ ifexists nofail ]
+    #   }]
+    # '';
 
     # see: <https://docs.pipewire.org/page_module_protocol_native.html>
     # defaults to placing the socket in /run/user/$id/{pipewire-0,pipewire-0-manager,...}
@@ -131,10 +153,11 @@ in
       # depends = [ "rtkit" ];
       # depends = [ "xdg-desktop-portal" ];  # for Realtime portal (dependency cycle)
       # env PIPEWIRE_LOG_SYSTEMD=false"
-      # env PIPEWIRE_DEBUG"*:3,mod.raop*:5,pw.rtsp-client*:5"
+      # env PIPEWIRE_DEBUG="*:3,mod.raop*:5,pw.rtsp-client*:5"
       command = pkgs.writeShellScript "pipewire-start" ''
         mkdir -p $PIPEWIRE_RUNTIME_DIR
-        exec pipewire
+        # nice -n -21 comes from pipewire defaults (niceness: -11)
+        exec nice -n -21 pipewire
       '';
       readiness.waitExists = [
         "$PIPEWIRE_RUNTIME_DIR/pipewire-0"
@@ -148,7 +171,7 @@ in
       partOf = [ "sound" ];
       command = pkgs.writeShellScript "pipewire-pulse-start" ''
         mkdir -p $XDG_RUNTIME_DIR/pulse
-        exec pipewire-pulse
+        exec nice -n -21 pipewire-pulse
       '';
       readiness.waitExists = [
         "$XDG_RUNTIME_DIR/pulse/native"