common/fs: reduce the ftp liveness checks to just one service

hosts/common/fs/remote-servo.nix

@@ -1,40 +1,37 @@
 { config, lib, utils, ... }:
 let
   fsOpts = import ./fs-opts.nix;
+  commonOptions = fsOpts.ftp ++ fsOpts.noauto;
   remoteServo = subdir: let
     mountpoint = "/mnt/servo/${subdir}";
     systemdName = utils.escapeSystemdPath mountpoint;
     device = "curlftpfs#ftp://servo-hn:/${subdir}";
     fsType = "fuse3";
-    commonOptions = fsOpts.ftp ++ fsOpts.noauto;
     options = commonOptions ++ [
       # systemd (or maybe fuse?) swallows stderr of mount units with no obvious fix.
       # instead, use this flag to log the mount output to disk
       "stderr_path=/var/log/curlftpfs/${builtins.replaceStrings [ "/" ] [ "-" ] subdir}.stderr"
     ];
   in {
-    sane.programs.curlftpfs.enableFor.system = true;
-    sane.fs."/var/log/curlftpfs".dir.acl.mode = "0777";
-    system.fsPackages = [
-      config.sane.programs.curlftpfs.package
-    ];
     fileSystems."${mountpoint}" = {
       inherit device fsType options;
       noCheck = true;
     };
+    systemd.services.servo-ftp-reachable.unitConfig.BindsTo = [ "${systemdName}.mount" ];
     systemd.mounts = [{
       where = mountpoint;
       what = device;
       type = fsType;
       options = lib.concatStringsSep "," options;
-      wantedBy = [ "default.target" ];
-      after = [ "network-online.target" "${systemdName}-reachable.service" ];
-      requires = [ "network-online.target" "${systemdName}-reachable.service" ];
+      wantedBy = [ "servo-ftp-reachable.service" ];
+      after = [ "servo-ftp-reachable.service" ];
+      requires = [ "servo-ftp-reachable.service" ];
+      bindsTo = [ "servo-ftp-reachable.service" ];
 
       #VVV patch so that when the mount fails, we start a timer to remount it.
       #    and for a disconnection after a good mount (onSuccess), restart the timer to be more aggressive
-      unitConfig.OnFailure = [ "${systemdName}.timer" ];
-      unitConfig.OnSuccess = [ "${systemdName}-restart-timer.target" ];
+      # unitConfig.OnFailure = [ "${systemdName}.timer" ];
+      # unitConfig.OnSuccess = [ "${systemdName}-restart-timer.target" ];
 
       mountConfig.TimeoutSec = "10s";
       mountConfig.ExecSearchPath = [ "/run/current-system/sw/bin" ];
@@ -71,11 +68,24 @@ let
       # mountConfig.RestrictNamespaces = true;
     }];
 
-    systemd.services."${systemdName}-reachable" = {
+  };
+in
+lib.mkMerge [
+  {
+    sane.programs.curlftpfs.enableFor.system = true;
+    system.fsPackages = [
+      config.sane.programs.curlftpfs.package
+    ];
+
+    sane.fs."/var/log/curlftpfs".dir.acl.mode = "0777";
+
+    systemd.services.servo-ftp-reachable = {
+      wantedBy = [ "default.target" ];
+      after = [ "network-online.target" ];
       serviceConfig.ExecSearchPath = [ "/run/current-system/sw/bin" ];
       serviceConfig.ExecStart = lib.escapeShellArgs [
         "curlftpfs"
-        "ftp://servo-hn:/${subdir}"
+        "ftp://servo-hn:/"
         "/dev/null"
         "-o"
         (lib.concatStringsSep "," ([
@@ -84,7 +94,12 @@ let
       ];
       serviceConfig.RemainAfterExit = true;
       serviceConfig.Type = "oneshot";
-      unitConfig.BindsTo = [ "${systemdName}.mount" ];
+
+      serviceConfig.RestartOn = "always";
+      serviceConfig.RestartSec = "10s";
+      serviceConfig.RestartMaxDelaySec = "120s";
+      serviceConfig.RestartSteps = 3;
+
       # hardening (systemd-analyze security mnt-servo-playground-reachable.service)
       serviceConfig.AmbientCapabilities = "";
       serviceConfig.CapabilityBoundingSet = "";
@@ -128,30 +143,28 @@ let
       serviceConfig.ProtectKernelTunables = false;
     };
 
-    systemd.targets."${systemdName}-restart-timer" = {
-      # hack unit which, when started, stops the timer (if running), and then starts it again.
-      after = [ "${systemdName}.timer" ];
-      conflicts = [ "${systemdName}.timer" ];
-      upholds = [ "${systemdName}.timer" ];
-      unitConfig.StopWhenUnneeded = true;
-    };
-    systemd.timers."${systemdName}" = {
-      timerConfig.Unit = "${systemdName}.mount";
-      timerConfig.AccuracySec = "2s";
-      timerConfig.OnActiveSec = [
-        # try to remount at these timestamps, backing off gradually
-        # there seems to be an implicit mount attempt at t=0.
-        "10s"
-        "30s"
-        "60s"
-        "120s"
-      ];
-      # cap the backoff to a fixed interval.
-      timerConfig.OnUnitActiveSec = [ "120s" ];
-    };
-  };
-in
-lib.mkMerge [
+    # systemd.targets."${systemdName}-restart-timer" = {
+    #   # hack unit which, when started, stops the timer (if running), and then starts it again.
+    #   after = [ "${systemdName}.timer" ];
+    #   conflicts = [ "${systemdName}.timer" ];
+    #   upholds = [ "${systemdName}.timer" ];
+    #   unitConfig.StopWhenUnneeded = true;
+    # };
+    # systemd.timers."${systemdName}" = {
+    #   timerConfig.Unit = "${systemdName}.mount";
+    #   timerConfig.AccuracySec = "2s";
+    #   timerConfig.OnActiveSec = [
+    #     # try to remount at these timestamps, backing off gradually
+    #     # there seems to be an implicit mount attempt at t=0.
+    #     "10s"
+    #     "30s"
+    #     "60s"
+    #     "120s"
+    #   ];
+    #   # cap the backoff to a fixed interval.
+    #   timerConfig.OnUnitActiveSec = [ "120s" ];
+    # };
+  }
   # this granularity of servo media mounts is necessary to support sandboxing:
   # for flaky mounts, we can only bind the mountpoint itself into the sandbox,
   # so it's either this or unconditionally bind all of media/.