nix-files/scripts/deploy

#!/usr/bin/env nix-shell
#!nix-shell -i ysh -p nettools -p oils-for-unix

var SELF = $(hostname)

proc usage {
  echo "deploy: deploy a nix config to a remote machine, possibly activating it"
  echo ""
  echo "usage: deploy [options] [host] [host2 ...]"
  echo "options:"
  echo "- --build: only build; don't copy or deploy"
  echo "- --copy: only build + copy files, nothing more"
  echo "- --switch  (default)"
  echo "- --test: switch to the new configuration, but do not make it bootable"
  echo "- --dry-run: show what would be done without actually doing it"
  echo "- --pre: alias for --action copy ..."
  echo '         if otherwise unspecified, implies `--variant all`'
  echo "         if no host is specified, copies to all known hosts"
  echo "- --reboot: reboot the target machine after deploying (if deployed with no errors)"
  echo "- --reboot-force: reboot the target machine after deploying (even on error)"
  echo "- --variant light|min|''|all  (default: '')"
  echo "- --wireguard always|never|opportunistic: deploy over wireguard"
  echo "- --ip <address>: deploy to the specific IP address"
  echo "- --deriv /nix/store/...: prebuilt store path (or .drv to realize) to deploy instead of (re-)building the default target"
  echo ""
  echo "common idioms:"
  echo "- deploy all: deploy all hosts, sequentially"
  echo "- deploy --pre: build and copy all hosts"
  echo "- deploy desko lappy: build and deploy just those hosts"
  echo "- deploy: deploy the local host"
  exit 1
}

proc info (...stmts) {
  echo "[deploy]" @stmts
}

var action = "switch"
var hosts = []
var ip = null
var defaultHost = SELF
var variants = []
var defaultVariant = ""
# by default, don't ship builds to servo. i'd guess this can be overriden by passing --builders servo
var nixArgs = ["--builders", ""]
var doReboot = false
var doRebootForce = false
var dryRun = false
var wireguard = "opportunistic"
var storePath = null
proc addHost (; host) {
  if (host === "all" ) {
    # order matters:
    call hosts->extend(["moby", "lappy", "desko", "servo"])
  } else {
    call hosts->append(host)
  }
}
proc addVariant (; v) {
  if (v === "all") {
    call variants->extend(["-min", "-light", "", "-next-min", "-next-light", "-next"])
  } elif (v === "") {
    # "full" variant
    call variants->append("")
  } else {
    call variants->append("-$v")
  }
}
proc parseArgs(;cur=null, next=null, ...rest) {
  if (cur === null) {
    # end
    :
  } else {
    case (cur) {
      --build | --copy | --switch | --test {
        setglobal action = cur
        parseArgs (next, ...rest)
      }
      --deriv {
        setglobal storePath = next
        parseArgs (...rest)
      }
      --dry-run {
        setglobal dryRun = true
        parseArgs (next, ...rest)
      }
      --help {
        usage
        parseArgs (next, ...rest)
      }
      --ip {
        setglobal ip = next
        parseArgs (...rest)
      }
      --pre {
        setglobal action = "copy"
        setglobal defaultVariant = "all"
        setglobal defaultHost = "all"
        parseArgs (next, ...rest)
      }
      --reboot {
        setglobal doReboot = true
        parseArgs (next, ...rest)
      }
      --reboot-force | --force-reboot {
        setglobal doReboot = true
        setglobal doRebootForce = true
        parseArgs (next, ...rest)
      }
      --variant {
        addVariant (next)
        parseArgs (...rest)
      }
      --wireguard {
        setglobal wireguard = next
        parseArgs (...rest)
      }
      all | crappy | desko | lappy | moby | servo {
        addHost (cur)
        parseArgs (next, ...rest)
      }
      (else) {
        call nixArgs->append(cur)
        parseArgs (next, ...rest)
      }
    }
  }

  if (hosts === [] and defaultHost) {
    addHost (defaultHost)
  }
  if (variants === []) {
    addVariant (defaultVariant)
  }
}

proc destructive (...cmd) {
  if (dryRun) {
    echo "dry-run:" @cmd
  } else {
    if (get(ENV, "ECHO_CMD")) {
      echo @cmd
    }
    @cmd
  }
}

# return "$1" or "$1-hn", based on if wireguard was requested or not
func resolveHost (host) {
  if (ip) {
    return (ip)
  } else {
    case ("$wireguard-$host") {
      opportunistic-moby {
        return ("$host-hn")
      }
      opportunistic-* {
        return (host)
      }
      never-* {
        return (host)
      }
      always-* {
        return ("$host-hn")
      }
      (else) {
        return ("$host-hn")
      }
    }
  }
}

# return the number of seconds to allot to `nix copy` when copying the given variant.
# this is done to avoid stalled copies from blocking the entire process, while hopefully not breaking the copies that are actually important
func timeoutFor (variant) {
  case (variant) {
    -min | -light | -next {
      return (3600)
    }
    -next-min | -next-light {
      return (1800)
    }
    (else) {
      # this catches the normal variant
      return (14400)
    }
  }
}

proc runOnTarget (host, ...cmd) {
  # run the command on the machine we're deploying to.
  # if that's a remote machine, then do it via ssh, else local shell.
  if (host !== "" and host !== SELF) {
    info "running on remote ($host):" @cmd
    ssh "$host" @cmd
  } else {
    info "running locally ($SELF):" @cmd
    @cmd
  }
}

proc deployOneHost (; host, variant) {
  var timeout = timeoutFor(variant)

  # storePath is allowed to be either a realized derivation,
  # or the path to a .drv file itself
  var myStorePath = storePath
  if (not myStorePath) {
    # `nix-build -A foo` evals and then realizes foo, but it never unloads the memory used to eval foo.
    # my exprs are heavyweight, we need that memory for building, so do the evals separately from the realizations:
    info "evaluating $host$variant..."
    setvar myStorePath = $(nix eval --raw -f . "hosts.$host$variant.toplevel.drvPath")
  }

  if ("$myStorePath" => endsWith(".drv")) {
    info "building $host$variant ($myStorePath)"
    setvar myStorePath = $(destructive nix-store --realize "$myStorePath" @nixArgs)
    if (myStorePath === "") {
      false
    }
    info "built $host$variant -> $myStorePath"
  }

  # mimic `nixos-rebuild --target-host`, in effect:
  # - nix-copy-closure ...
  # - nix-env --set ...
  # - switch-to-configuration <boot|dry-activate|switch|test|>
  # avoid the actual `nixos-rebuild` for a few reasons:
  # - fewer nix evals
  # - more introspectability and debuggability
  # - sandbox friendliness (especially: `git` doesn't have to be run as root)

  var netHost = resolveHost(host)

  case (action) {
    copy | switch | test {
      if (host !== "" and host !== SELF) {
        if test -e /run/secrets/nix_signing_key {
          info "signing store paths ..."
          destructive sudo nix store sign -r -k /run/secrets/nix_signing_key "$myStorePath"
        } else {
          info "not signing store paths: /run/secrets/nix_signing_key does not exist"
        }
        # add more `-v` for more verbosity (up to 5).
        # builders-use-substitutes false: optimizes so that the remote machine doesn't try to get paths from its substituters.
        #   we already have all paths here, and the remote substitution is slow to check and SERIOUSLY flaky on moby in particular.
        ECHO_CMD=1 destructive timeout "$timeout" nix copy -vv --option builders-use-substitutes false --to "ssh-ng://$netHost" "$myStorePath"
      }
    }
  }
  case (action) {
    switch | test {
      info "activating profile... "
      destructive runOnTarget "$netHost" sudo nix-env -p /nix/var/nix/profiles/system --set "$myStorePath"

      try {
        destructive runOnTarget "$netHost" sudo "$myStorePath/bin/switch-to-configuration" "$action"
      }
      var fail = false
      if failed { setvar fail = true }  # XXX: `failed` special variable is only readable via `if failed`

      if (doReboot and (not fail or doRebootForce)) {
        info "rebooting $host"
        destructive runOnTarget "$netHost" sane-reboot "$host" || return 1
      }

      if (fail) {
        false
      }
    }
  }
}

var failedDeploys = []
proc deployHosts(; ...hosts) {
  for v in (variants) {
    for h in (hosts) {
      try {
        deployOneHost (h, v)
      }
      if failed {
        call failedDeploys->append("$h$v")
      }
    }
  }
}

proc main (...args) {
  parseArgs (...args)

  # i care e.g. that full moby is deployed before crappy:
  var earlyHosts = []
  var lateHosts = []
  for host in (hosts) {
    case (host) {
      crappy {
        call lateHosts->append(host)
      }
      (else) {
        call earlyHosts->append(host)
      }
    }
  }

  info "build plan -----"
  # TODO: dedupe this logic with `deployHosts`
  for hset in (earlyHosts, lateHosts) {
    for v in (variants) {
      for h in (hset) {
        info "- $h$v"
      }
    }
  }
  info "-----"


  deployHosts (...earlyHosts)
  deployHosts (...lateHosts)

  if (failedDeploys !== []) {
    echo "FAILED DEPLOYMENT:"
    for d in (failedDeploys) {
      echo "- $d"
    }
    exit 1
  }

  echo "SUCCESS"
}

if is-main {
  main @ARGV
}