testing

configurations/server.nix

@@ -64,6 +64,56 @@
 
   host.ssh.enable = true;
 
+  # Offload Nix builds to the WSL2 NixOS instance running on vetymae
+  # (Windows 11 host). Sapphira only has 2 logical cores; the WSL exposes
+  # 24 cores + 14 GiB. The matchBlock with ProxyCommand is generated by
+  # modules/server/builder.nix, the other side of the same option lives in
+  # modules/wsl/builder.nix.
+  #
+  # `proxyCommand` is what marks this builder as needing the SSH matchBlock
+  # (see modules/server/builder.nix). A builder reachable directly would
+  # omit it.
+  host.builder.clients = [
+    {
+      hostName = "vetymae-nix";
+      sshUser = "oqyude";
+      sshKey = "/root/.ssh/id_ed25519";
+      # NixOS calls this `systems` (plural), not `systemTypes`. The
+      # default is empty — every derivation is rejected. The WSL NixOS
+      # runs on x86_64-linux, matching sapphira.
+      systems = [ "x86_64-linux" ];
+      # vetymae-nix drops kvm + nixos-test from its advertised
+      # system-features (see modules/wsl/builder.nix). Listing them here
+      # would not break anything (Nix intersects), but listing the
+      # features the WSL actually has is the documented contract.
+      supportedFeatures = [
+        "benchmark"
+        "big-parallel"
+      ];
+      mandatoryFeatures = [ ];
+      maxJobs = 24;
+      speedFactor = 0.5;
+      # Keep the SSH session alive across many small builds in one daemon
+      # session — compile-heavy workloads spam the daemon with hundreds of
+      # derivations and ControlMaster collapses those into one Windows hop.
+      # NB: `nix.buildMachines` has no `sshOptions` attribute, so the
+      # ControlMaster directive lives in the SSH matchBlock instead (see
+      # modules/server/builder.nix).
+      #
+      # The OpenSSH alias for this host (matches the user's
+      # ~/.ssh/config so known_hosts entries do not collide with the
+      # Windows OpenSSH entry on 127.0.0.1/vetymae) is consumed only by
+      # the SSH matchBlock below — not by `nix.buildMachines`, which has
+      # no such attribute.
+      hostKeyAlias = "wsl-nixos-on-vetymae";
+      # Use the Windows host's IP directly so the nix-daemon (running as
+      # root, without the user's ~/.ssh/config) does not need a separate
+      # `vetymae` host alias. With StrictHostKeyChecking=accept-new the
+      # first connection adds the Windows host key to /root/.ssh/known_hosts.
+      proxyCommand = "ssh oqyude@192.168.1.100 'wsl -d NixOS -- nc 127.0.0.1 22'";
+    }
+  ];
+
   networking = {
     networkmanager.enable = true;
     firewall.enable = false;

configurations/wsl.nix

@@ -41,5 +41,11 @@
   # passwordless key auth — nothing to repeat here.
   host.ssh.enable = true;
 
+  # Advertise this WSL instance as a remote Nix builder for sapphira (2
+  # cores, the bottleneck host). All builder wiring — fixing the
+  # `system-features` to drop the unsupported `kvm`, and adding the SSH
+  # user `oqyude` to trusted-users — lives in modules/wsl/builder.nix.
+  host.builder.enable = true;
+
   system.stateVersion = "24.11";
 }

modules/options.nix

@@ -9,6 +9,41 @@
 # the option exists. `modules/essentials/ssh.nix` does not belong here: it
 # declares and reads `host.ssh.enable` itself, within one module.
 {
+  # Remote-builder wiring. A coordinator (e.g. sapphira) sets
+  # `host.builder.clients` to register remote build machines;
+  # a builder host (e.g. the WSL on vetymae) sets `host.builder.enable`
+  # to advertise itself. The two halves are intentionally split so a single
+  # declaration in configurations/* is enough to flip each side.
+  options.host.builder = {
+    enable = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      description = ''
+        Advertise this host as a remote Nix builder and accept builds
+        from other machines in the flake over SSH.
+      '';
+    };
+    clients = lib.mkOption {
+      type = lib.types.listOf lib.types.attrs;
+      default = [ ];
+      description = ''
+        List of remote Nix build machines this coordinator should
+        register via `nix.buildMachines`. Each entry matches the NixOS
+        option schema (hostName, sshUser, sshKey, systems,
+        supportedFeatures, ...). Two extra attributes are consumed by
+        modules/server/builder.nix and stripped before reaching
+        `nix.buildMachines`:
+          - `proxyCommand` — generates a per-builder Host block in the
+            system-wide OpenSSH config (the nix-daemon runs as root and
+            cannot see the user's ~/.ssh/config).
+          - `hostKeyAlias` — alias used inside that SSH matchBlock.
+        A builder reachable on its own (no ProxyCommand needed) omits
+        both and gets no SSH matchBlock. Empty by default — opt in by
+        setting this list.
+      '';
+    };
+  };
+
   options.host."3x-ui" = {
     # Domain whose Let's Encrypt cert (at /var/lib/acme/<domain>/)
     # gets mounted read-only into the 3x-ui container so the panel

modules/server/builder.nix

@@ -0,0 +1,130 @@
+# sapphira (and any other server-class coordinator) — register remote
+# builders, and make sure the nix daemon (running as root) can resolve the
+# SSH host alias with its ProxyCommand chain.
+#
+# The `host.builder.clients` option itself is declared in
+# modules/options.nix (cross-module). The actual builder list is set by the
+# configuration (e.g. configurations/server.nix) — this module is generic
+# over every entry on the list.
+{
+  config,
+  lib,
+  ...
+}:
+let
+  # Attributes that belong to the SSH matchBlock only — NOT to
+  # `nix.buildMachines` (that schema has no hostKeyAlias/proxyCommand).
+  # Strip them before handing the list to nix.buildMachines.
+  sshOnlyAttrs = [
+    "hostKeyAlias"
+    "proxyCommand"
+  ];
+  forNix = b: removeAttrs b sshOnlyAttrs;
+  # After NixOS's nix.buildMachines submodule runs, each entry has all
+  # attributes defaulted (protocol=ssh, systems=[], etc.). Read from that
+  # processed list so the formatter never trips on a missing field.
+  processedBuilders = config.nix.buildMachines;
+
+  # Serialise one builder to the textual format Nix's daemon expects in
+  # `nix.conf`'s `builders` line. Mirrors `buildMachinesText` from
+  # nixos/modules/config/nix-remote-build.nix so the result is identical
+  # to what NixOS writes to /etc/nix/machines — we just inline it instead
+  # of relying on `@/etc/nix/machines`, which Nix 2.34 parses but does
+  # not act on (the daemon's `external-builders` list stays empty and the
+  # client reports "configure remote builders via 'builders'" forever).
+  formatBuilder = b:
+    let
+      # Nix 2.34 refuses to dispatch derivations to a builder whose protocol
+      # is `ssh` (the NixOS default): the daemon leaves `external-builders`
+      # empty even when the `builders` line is well-formed, and the client
+      # falls back to local. `ssh-ng` (the new in-band protocol) actually
+      # opens the dispatcher. Override the NixOS default here.
+      proto = "ssh-ng://";
+      user = if b.sshUser != null && b.sshUser != "" then "${b.sshUser}@" else "";
+      systems =
+        if b.system != null then b.system
+        else if b.systems != [ ] then lib.concatStringsSep "," b.systems
+        else "-";
+      sshKey = if b.sshKey != null && b.sshKey != "" then b.sshKey else "-";
+      maxJobs = toString b.maxJobs;
+      speedFactor = toString b.speedFactor;
+      allFeats = b.supportedFeatures ++ b.mandatoryFeatures;
+      supported =
+        if allFeats == [ ] then "-"
+        else lib.concatStringsSep "," allFeats;
+      mandatory =
+        if b.mandatoryFeatures == [ ] then "-"
+        else lib.concatStringsSep "," b.mandatoryFeatures;
+      publicKey = if b.publicHostKey != null then b.publicHostKey else "-";
+    in
+    lib.concatStringsSep " " [
+      "${proto}${user}${b.hostName}"
+      systems
+      sshKey
+      maxJobs
+      speedFactor
+      supported
+      mandatory
+      publicKey
+    ];
+  inlineBuilders = lib.concatMapStringsSep "\n" formatBuilder processedBuilders;
+
+  # One OpenSSH host block per builder that needs a ProxyCommand.
+  # Placed in `programs.ssh.extraConfig` so it ends up in
+  # /etc/ssh/ssh_config (the file OpenSSH consults system-wide, including
+  # for the nix-daemon running as root).
+  #
+  # Only builders with a `proxyCommand` attribute get a block: a builder
+  # reachable on its own (e.g. otreca on a public IP) needs no help from
+  # here. The attribute is the literal ProxyCommand string (passed
+  # verbatim to ssh); the configuration is responsible for matching it
+  # with the `hostName` field.
+  hostBlock = b: ''
+    Host ${b.hostName}
+      User ${b.sshUser}
+      HostKeyAlias ${b.hostKeyAlias or b.hostName}
+      ProxyCommand ${b.proxyCommand}
+      StrictHostKeyChecking accept-new
+      ServerAliveInterval 30
+      ServerAliveCountMax 3
+      ControlMaster auto
+      ControlPersist 60
+      ConnectTimeout 15
+  '';
+  blocks = map hostBlock (lib.filter (b: b ? proxyCommand) config.host.builder.clients);
+in
+{
+  config = lib.mkIf (config.host.builder.clients != [ ]) {
+    # Off-by-default in NixOS. Without this, the nix-remote-build module
+    # sets `nix.settings.builders = null` and the list is dropped from
+    # /etc/nix/nix.conf entirely, even though `nix.buildMachines` is
+    # populated. (The build-machine list still lands in /etc/nix/machines
+    # but nix-daemon reads `builders`, not /etc/nix/machines, when
+    # distributedBuilds is false.)
+    nix.distributedBuilds = true;
+    nix.buildMachines = map forNix config.host.builder.clients;
+    # Nix 2.34's daemon does not act on `@/etc/nix/machines` (the file
+    # format NixOS's nix-remote-build writes to): the `builders` config
+    # key is parsed for display but `external-builders` stays empty and
+    # the scheduler ignores it. Inlining the same builder text here — in
+    # the exact format the NixOS module itself uses — actually wires up
+    # the SSH dispatch. `mkForce` is required because the nix-remote-build
+    # module sets `builders = null` whenever distributedBuilds is *false*;
+    # our config flips it to *true*, so the module's mkIf does not fire
+    # and there is no actual conflict — but pinning it with mkForce makes
+    # the intent obvious and survives any future change in default
+    # behaviour.
+    nix.settings.builders = lib.mkForce inlineBuilders;
+
+    # Append per-builder Host blocks to the system-wide OpenSSH client
+    # config. `programs.ssh.extraConfig` is of type `lines`, merged across
+    # modules, and prepended (before `Host *`) in /etc/ssh/ssh_config —
+    # which is exactly the spot where specific Host blocks have to live.
+    programs.ssh.extraConfig = lib.concatStrings blocks;
+
+    # Parallel builds on sapphira itself stay at 2 — that matches the
+    # physical cores and keeps the coordinator responsive while the WSL
+    # absorbs the heavy lifting. The essentials/settings.nix already
+    # leaves max-jobs at the default `auto` (2 here); no override needed.
+  };
+}

modules/server/default.nix

@@ -10,6 +10,7 @@
     ../pkgs/beets.nix
     ./acme.nix
     ./bentopdf.nix
+    ./builder.nix
     ./calibre-web.nix
     ./chrony.nix
     ./coredns.nix

modules/wsl/builder.nix

@@ -0,0 +1,54 @@
+# WSL NixOS — advertise this host as a remote Nix builder.
+#
+# Why a dedicated module instead of inlining into configurations/wsl.nix:
+# every "what makes this WSL different from a desktop/server" concern
+# belongs under modules/wsl/ — that is the contract the device-type import in
+# modules/default.nix wires up. Keeping it here means flipping the feature on
+# later on another WSL host (e.g. a future vetymae-2) is one import away.
+#
+# The `host.builder.enable` option itself is declared in
+# modules/options.nix (cross-module).
+{
+  config,
+  lib,
+  ...
+}:
+{
+  config = lib.mkIf config.host.builder.enable {
+    # WSL2 does not expose /dev/kvm to the guest (no nested virt by default,
+    # and Hyper-V's /dev/kvm is not bind-mounted into the WSL namespace).
+    # The default NixOS module advertises `kvm nixos-test benchmark
+    # big-parallel` as this host's system-features, which is a lie: any
+    # derivation that requires `kvm` will be dispatched here and immediately
+    # fail with "cannot open /dev/kvm". Nix selects builders by matching the
+    # derivation's required features against what the builder advertises, so
+    # the only way to keep WSL useful is to retract the features it cannot
+    # actually deliver. `nixos-test` is dropped for the same reason — it
+    # wants kvm anyway.
+    #
+    # `mkForce` because the NixOS module base-config sets a non-empty
+    # default; without force the lists would concatenate and the WSL would
+    # *still* advertise kvm.
+    nix.settings.system-features = lib.mkForce [
+      "benchmark"
+      "big-parallel"
+    ];
+
+    # Builds arrive over SSH as the user `oqyude` (see
+    # modules/server/builder.nix). On the default trusted-users = ["root"]
+    # only root can call nix-store, so the SSH session would fail to realise
+    # any .drv. Adding the SSH user to trusted-users lets the remote nix-build
+    # driver drive nix-store on the builder side. `mkForce` for the same
+    # concatenation reason as above.
+    nix.settings.trusted-users = lib.mkForce [
+      "root"
+      "oqyude"
+    ];
+
+    # The local daemon already parallelises across all 24 logical cores
+    # (max-jobs = 24 is what we measured). When acting as a builder, we
+    # want to keep that — remote builds land through SSH and the daemon
+    # serves them on top of its normal pool. No override needed; documented
+    # here so a future reader does not "tidy up" by setting max-jobs low.
+  };
+}

modules/wsl/default.nix

@@ -9,6 +9,7 @@
     ../pkgs/beets.nix
     ./containers
     ./nix-serve.nix
+    ./builder.nix
     # ./tools
   ];
 }

result

@@ -0,0 +1 @@
+/nix/store/x6qwqsl3xprb9pwinajspkkg4r5lgxcz-nixos-system-sapphira-default