From 543fcc61d9125ec3f395c60ff33c6aa88cb90424 Mon Sep 17 00:00:00 2001 From: oqyude Date: Sat, 3 Oct 2026 23:39:21 +0300 Subject: [PATCH] testing --- configurations/server.nix | 50 ++++++++++++++ configurations/wsl.nix | 6 ++ modules/options.nix | 35 ++++++++++ modules/server/builder.nix | 130 +++++++++++++++++++++++++++++++++++++ modules/server/default.nix | 1 + modules/wsl/builder.nix | 54 +++++++++++++++ modules/wsl/default.nix | 1 + result | 1 + 8 files changed, 278 insertions(+) create mode 100644 modules/server/builder.nix create mode 100644 modules/wsl/builder.nix create mode 120000 result diff --git a/configurations/server.nix b/configurations/server.nix index a863502..6527268 100644 --- a/configurations/server.nix +++ b/configurations/server.nix @@ -64,6 +64,56 @@ host.ssh.enable = true; + # Offload Nix builds to the WSL2 NixOS instance running on vetymae + # (Windows 11 host). Sapphira only has 2 logical cores; the WSL exposes + # 24 cores + 14 GiB. The matchBlock with ProxyCommand is generated by + # modules/server/builder.nix, the other side of the same option lives in + # modules/wsl/builder.nix. + # + # `proxyCommand` is what marks this builder as needing the SSH matchBlock + # (see modules/server/builder.nix). A builder reachable directly would + # omit it. + host.builder.clients = [ + { + hostName = "vetymae-nix"; + sshUser = "oqyude"; + sshKey = "/root/.ssh/id_ed25519"; + # NixOS calls this `systems` (plural), not `systemTypes`. The + # default is empty — every derivation is rejected. The WSL NixOS + # runs on x86_64-linux, matching sapphira. + systems = [ "x86_64-linux" ]; + # vetymae-nix drops kvm + nixos-test from its advertised + # system-features (see modules/wsl/builder.nix). Listing them here + # would not break anything (Nix intersects), but listing the + # features the WSL actually has is the documented contract. + supportedFeatures = [ + "benchmark" + "big-parallel" + ]; + mandatoryFeatures = [ ]; + maxJobs = 24; + speedFactor = 0.5; + # Keep the SSH session alive across many small builds in one daemon + # session — compile-heavy workloads spam the daemon with hundreds of + # derivations and ControlMaster collapses those into one Windows hop. + # NB: `nix.buildMachines` has no `sshOptions` attribute, so the + # ControlMaster directive lives in the SSH matchBlock instead (see + # modules/server/builder.nix). + # + # The OpenSSH alias for this host (matches the user's + # ~/.ssh/config so known_hosts entries do not collide with the + # Windows OpenSSH entry on 127.0.0.1/vetymae) is consumed only by + # the SSH matchBlock below — not by `nix.buildMachines`, which has + # no such attribute. + hostKeyAlias = "wsl-nixos-on-vetymae"; + # Use the Windows host's IP directly so the nix-daemon (running as + # root, without the user's ~/.ssh/config) does not need a separate + # `vetymae` host alias. With StrictHostKeyChecking=accept-new the + # first connection adds the Windows host key to /root/.ssh/known_hosts. + proxyCommand = "ssh oqyude@192.168.1.100 'wsl -d NixOS -- nc 127.0.0.1 22'"; + } + ]; + networking = { networkmanager.enable = true; firewall.enable = false; diff --git a/configurations/wsl.nix b/configurations/wsl.nix index 319105f..f931b6d 100644 --- a/configurations/wsl.nix +++ b/configurations/wsl.nix @@ -41,5 +41,11 @@ # passwordless key auth — nothing to repeat here. host.ssh.enable = true; + # Advertise this WSL instance as a remote Nix builder for sapphira (2 + # cores, the bottleneck host). All builder wiring — fixing the + # `system-features` to drop the unsupported `kvm`, and adding the SSH + # user `oqyude` to trusted-users — lives in modules/wsl/builder.nix. + host.builder.enable = true; + system.stateVersion = "24.11"; } diff --git a/modules/options.nix b/modules/options.nix index 96f9985..3a60ee6 100644 --- a/modules/options.nix +++ b/modules/options.nix @@ -9,6 +9,41 @@ # the option exists. `modules/essentials/ssh.nix` does not belong here: it # declares and reads `host.ssh.enable` itself, within one module. { + # Remote-builder wiring. A coordinator (e.g. sapphira) sets + # `host.builder.clients` to register remote build machines; + # a builder host (e.g. the WSL on vetymae) sets `host.builder.enable` + # to advertise itself. The two halves are intentionally split so a single + # declaration in configurations/* is enough to flip each side. + options.host.builder = { + enable = lib.mkOption { + type = lib.types.bool; + default = false; + description = '' + Advertise this host as a remote Nix builder and accept builds + from other machines in the flake over SSH. + ''; + }; + clients = lib.mkOption { + type = lib.types.listOf lib.types.attrs; + default = [ ]; + description = '' + List of remote Nix build machines this coordinator should + register via `nix.buildMachines`. Each entry matches the NixOS + option schema (hostName, sshUser, sshKey, systems, + supportedFeatures, ...). Two extra attributes are consumed by + modules/server/builder.nix and stripped before reaching + `nix.buildMachines`: + - `proxyCommand` — generates a per-builder Host block in the + system-wide OpenSSH config (the nix-daemon runs as root and + cannot see the user's ~/.ssh/config). + - `hostKeyAlias` — alias used inside that SSH matchBlock. + A builder reachable on its own (no ProxyCommand needed) omits + both and gets no SSH matchBlock. Empty by default — opt in by + setting this list. + ''; + }; + }; + options.host."3x-ui" = { # Domain whose Let's Encrypt cert (at /var/lib/acme//) # gets mounted read-only into the 3x-ui container so the panel diff --git a/modules/server/builder.nix b/modules/server/builder.nix new file mode 100644 index 0000000..d59fd0f --- /dev/null +++ b/modules/server/builder.nix @@ -0,0 +1,130 @@ +# sapphira (and any other server-class coordinator) — register remote +# builders, and make sure the nix daemon (running as root) can resolve the +# SSH host alias with its ProxyCommand chain. +# +# The `host.builder.clients` option itself is declared in +# modules/options.nix (cross-module). The actual builder list is set by the +# configuration (e.g. configurations/server.nix) — this module is generic +# over every entry on the list. +{ + config, + lib, + ... +}: +let + # Attributes that belong to the SSH matchBlock only — NOT to + # `nix.buildMachines` (that schema has no hostKeyAlias/proxyCommand). + # Strip them before handing the list to nix.buildMachines. + sshOnlyAttrs = [ + "hostKeyAlias" + "proxyCommand" + ]; + forNix = b: removeAttrs b sshOnlyAttrs; + # After NixOS's nix.buildMachines submodule runs, each entry has all + # attributes defaulted (protocol=ssh, systems=[], etc.). Read from that + # processed list so the formatter never trips on a missing field. + processedBuilders = config.nix.buildMachines; + + # Serialise one builder to the textual format Nix's daemon expects in + # `nix.conf`'s `builders` line. Mirrors `buildMachinesText` from + # nixos/modules/config/nix-remote-build.nix so the result is identical + # to what NixOS writes to /etc/nix/machines — we just inline it instead + # of relying on `@/etc/nix/machines`, which Nix 2.34 parses but does + # not act on (the daemon's `external-builders` list stays empty and the + # client reports "configure remote builders via 'builders'" forever). + formatBuilder = b: + let + # Nix 2.34 refuses to dispatch derivations to a builder whose protocol + # is `ssh` (the NixOS default): the daemon leaves `external-builders` + # empty even when the `builders` line is well-formed, and the client + # falls back to local. `ssh-ng` (the new in-band protocol) actually + # opens the dispatcher. Override the NixOS default here. + proto = "ssh-ng://"; + user = if b.sshUser != null && b.sshUser != "" then "${b.sshUser}@" else ""; + systems = + if b.system != null then b.system + else if b.systems != [ ] then lib.concatStringsSep "," b.systems + else "-"; + sshKey = if b.sshKey != null && b.sshKey != "" then b.sshKey else "-"; + maxJobs = toString b.maxJobs; + speedFactor = toString b.speedFactor; + allFeats = b.supportedFeatures ++ b.mandatoryFeatures; + supported = + if allFeats == [ ] then "-" + else lib.concatStringsSep "," allFeats; + mandatory = + if b.mandatoryFeatures == [ ] then "-" + else lib.concatStringsSep "," b.mandatoryFeatures; + publicKey = if b.publicHostKey != null then b.publicHostKey else "-"; + in + lib.concatStringsSep " " [ + "${proto}${user}${b.hostName}" + systems + sshKey + maxJobs + speedFactor + supported + mandatory + publicKey + ]; + inlineBuilders = lib.concatMapStringsSep "\n" formatBuilder processedBuilders; + + # One OpenSSH host block per builder that needs a ProxyCommand. + # Placed in `programs.ssh.extraConfig` so it ends up in + # /etc/ssh/ssh_config (the file OpenSSH consults system-wide, including + # for the nix-daemon running as root). + # + # Only builders with a `proxyCommand` attribute get a block: a builder + # reachable on its own (e.g. otreca on a public IP) needs no help from + # here. The attribute is the literal ProxyCommand string (passed + # verbatim to ssh); the configuration is responsible for matching it + # with the `hostName` field. + hostBlock = b: '' + Host ${b.hostName} + User ${b.sshUser} + HostKeyAlias ${b.hostKeyAlias or b.hostName} + ProxyCommand ${b.proxyCommand} + StrictHostKeyChecking accept-new + ServerAliveInterval 30 + ServerAliveCountMax 3 + ControlMaster auto + ControlPersist 60 + ConnectTimeout 15 + ''; + blocks = map hostBlock (lib.filter (b: b ? proxyCommand) config.host.builder.clients); +in +{ + config = lib.mkIf (config.host.builder.clients != [ ]) { + # Off-by-default in NixOS. Without this, the nix-remote-build module + # sets `nix.settings.builders = null` and the list is dropped from + # /etc/nix/nix.conf entirely, even though `nix.buildMachines` is + # populated. (The build-machine list still lands in /etc/nix/machines + # but nix-daemon reads `builders`, not /etc/nix/machines, when + # distributedBuilds is false.) + nix.distributedBuilds = true; + nix.buildMachines = map forNix config.host.builder.clients; + # Nix 2.34's daemon does not act on `@/etc/nix/machines` (the file + # format NixOS's nix-remote-build writes to): the `builders` config + # key is parsed for display but `external-builders` stays empty and + # the scheduler ignores it. Inlining the same builder text here — in + # the exact format the NixOS module itself uses — actually wires up + # the SSH dispatch. `mkForce` is required because the nix-remote-build + # module sets `builders = null` whenever distributedBuilds is *false*; + # our config flips it to *true*, so the module's mkIf does not fire + # and there is no actual conflict — but pinning it with mkForce makes + # the intent obvious and survives any future change in default + # behaviour. + nix.settings.builders = lib.mkForce inlineBuilders; + + # Append per-builder Host blocks to the system-wide OpenSSH client + # config. `programs.ssh.extraConfig` is of type `lines`, merged across + # modules, and prepended (before `Host *`) in /etc/ssh/ssh_config — + # which is exactly the spot where specific Host blocks have to live. + programs.ssh.extraConfig = lib.concatStrings blocks; + + # Parallel builds on sapphira itself stay at 2 — that matches the + # physical cores and keeps the coordinator responsive while the WSL + # absorbs the heavy lifting. The essentials/settings.nix already + # leaves max-jobs at the default `auto` (2 here); no override needed. + }; +} \ No newline at end of file diff --git a/modules/server/default.nix b/modules/server/default.nix index 53e0d37..6ab5b0b 100644 --- a/modules/server/default.nix +++ b/modules/server/default.nix @@ -10,6 +10,7 @@ ../pkgs/beets.nix ./acme.nix ./bentopdf.nix + ./builder.nix ./calibre-web.nix ./chrony.nix ./coredns.nix diff --git a/modules/wsl/builder.nix b/modules/wsl/builder.nix new file mode 100644 index 0000000..220a9d0 --- /dev/null +++ b/modules/wsl/builder.nix @@ -0,0 +1,54 @@ +# WSL NixOS — advertise this host as a remote Nix builder. +# +# Why a dedicated module instead of inlining into configurations/wsl.nix: +# every "what makes this WSL different from a desktop/server" concern +# belongs under modules/wsl/ — that is the contract the device-type import in +# modules/default.nix wires up. Keeping it here means flipping the feature on +# later on another WSL host (e.g. a future vetymae-2) is one import away. +# +# The `host.builder.enable` option itself is declared in +# modules/options.nix (cross-module). +{ + config, + lib, + ... +}: +{ + config = lib.mkIf config.host.builder.enable { + # WSL2 does not expose /dev/kvm to the guest (no nested virt by default, + # and Hyper-V's /dev/kvm is not bind-mounted into the WSL namespace). + # The default NixOS module advertises `kvm nixos-test benchmark + # big-parallel` as this host's system-features, which is a lie: any + # derivation that requires `kvm` will be dispatched here and immediately + # fail with "cannot open /dev/kvm". Nix selects builders by matching the + # derivation's required features against what the builder advertises, so + # the only way to keep WSL useful is to retract the features it cannot + # actually deliver. `nixos-test` is dropped for the same reason — it + # wants kvm anyway. + # + # `mkForce` because the NixOS module base-config sets a non-empty + # default; without force the lists would concatenate and the WSL would + # *still* advertise kvm. + nix.settings.system-features = lib.mkForce [ + "benchmark" + "big-parallel" + ]; + + # Builds arrive over SSH as the user `oqyude` (see + # modules/server/builder.nix). On the default trusted-users = ["root"] + # only root can call nix-store, so the SSH session would fail to realise + # any .drv. Adding the SSH user to trusted-users lets the remote nix-build + # driver drive nix-store on the builder side. `mkForce` for the same + # concatenation reason as above. + nix.settings.trusted-users = lib.mkForce [ + "root" + "oqyude" + ]; + + # The local daemon already parallelises across all 24 logical cores + # (max-jobs = 24 is what we measured). When acting as a builder, we + # want to keep that — remote builds land through SSH and the daemon + # serves them on top of its normal pool. No override needed; documented + # here so a future reader does not "tidy up" by setting max-jobs low. + }; +} diff --git a/modules/wsl/default.nix b/modules/wsl/default.nix index c4fbcee..029723b 100644 --- a/modules/wsl/default.nix +++ b/modules/wsl/default.nix @@ -9,6 +9,7 @@ ../pkgs/beets.nix ./containers ./nix-serve.nix + ./builder.nix # ./tools ]; } diff --git a/result b/result new file mode 120000 index 0000000..2f9b6a4 --- /dev/null +++ b/result @@ -0,0 +1 @@ +/nix/store/x6qwqsl3xprb9pwinajspkkg4r5lgxcz-nixos-system-sapphira-default \ No newline at end of file