add remote hydra eval

config/services/hydra.nix

@@ -5,8 +5,10 @@
   config,
   pkgs,
   hydra,
+  nix-eval-jobs,
   ...
-}: let
+}:
+let
   machines = pkgs.writeText "machines" ''
     localhost armv7l-linux,powerpc-linux,powerpc64-linux,powerpc64le-linux,wasm32-wasi,x86_64-linux,i686-linux,riscv32-linux,riscv64-linux - 12 1 kvm,nixos-test,big-parallel,benchmark,gccarch-znver1,gccarch-skylake,ca-derivations  -
     build-aarch64 aarch64-linux,riscv32-linux,riscv64-linux - 4 1 nixos-test,benchmark,ca-derivations,gccarch-armv8-a,gccarch-armv8.1-a,gccarch-armv8.2-a,big-parallel  -
@@ -49,7 +51,22 @@
       ControlPath ~/.ssh/master-%r@%n:%p
       ControlPersist 10m
   '';
-in {
+  nix-eval-jobs-script = pkgs.stdenvNoCC.mkDerivation {
+    name = "remote-eval-jobs.py";
+    src = ./hydra/remote-eval-jobs.py;
+    dontUnpack = true;
+    dontBuild = true;
+    installPhase = ''
+      substitute $src $out \
+        --subst-var-by python3 ${pkgs.python3}/bin/python3
+        --subst-var-by ping ${pkgs.iputils}/bin/ping
+        --subst-var-by nix-eval-jobs ${pkgs.nix-eval-jobs}/bin/nix-eval-jobs
+        --subst-var-by nix ${pkgs.nix}/bin/nix
+        --subst-var-by ssh ${pkgs.openssh}/bin/ssh
+    '';
+  };
+in
+{
   imports = [
     ./postgres.nix
     ../../modules/hydra.nix
@@ -63,18 +80,20 @@ in {
     package = hydra.packages.${system}.hydra.overrideAttrs (super: {
       doCheck = false;
       doInstallCheck = false;
-      patches =
-        super.patches
-        or []
-        ++ [
-          ./hydra/0001-add-gitea-pulls.patch
-          ./hydra/0002-unlimit-output.patch
-          ./hydra/0003-remove-pr-number-from-github-job-name.patch
-          ./hydra/0004-use-pulls-instead-of-issues.patch
-          ./hydra/0005-only-list-open-prs.patch
-          ./hydra/0006-status-state.patch
-          ./hydra/0007-hydra-server-findLog-fix-issue-with-ca-derivations-e.patch
-        ];
+      patches = super.patches or [ ] ++ [
+        ./hydra/0001-add-gitea-pulls.patch
+        ./hydra/0002-unlimit-output.patch
+        ./hydra/0003-remove-pr-number-from-github-job-name.patch
+        ./hydra/0004-use-pulls-instead-of-issues.patch
+        ./hydra/0005-only-list-open-prs.patch
+        ./hydra/0006-status-state.patch
+        ./hydra/0007-hydra-server-findLog-fix-issue-with-ca-derivations-e.patch
+      ];
+      postPatch =
+        super.postPatch or ""
+        + ''
+          substituteInPlace src/script/hydra-eval-jobset --replace-fail nix-eval-jobs ${nix-eval-jobs-script}
+        '';
     });
     hydraURL = "https://hydra.chir.rs/";
     notificationSender = "hydra@chir.rs";
@@ -114,9 +133,13 @@ in {
       "/run/hydra-machines"
     ];
   };
-  nix.settings.allowed-uris = ["github:" "https://" "http://"];
-  sops.secrets."services/hydra/gitea_token" = {};
-  sops.secrets."services/hydra/github_token" = {};
+  nix.settings.allowed-uris = [
+    "github:"
+    "https://"
+    "http://"
+  ];
+  sops.secrets."services/hydra/gitea_token" = { };
+  sops.secrets."services/hydra/github_token" = { };
   sops.secrets."services/hydra/cache-key" = {
     owner = "hydra-www";
     mode = "0440";
@@ -135,7 +158,7 @@ in {
   sops.secrets."services/hydra/aws_credentials" = {
     owner = "hydra-queue-runner";
     path = "/var/lib/hydra/queue-runner/.aws/credentials";
-    restartUnits = ["hydra-notify.service"];
+    restartUnits = [ "hydra-notify.service" ];
   };
   systemd.services.update-hydra-hosts = {
     description = "Update hydra hosts";
@@ -153,21 +176,21 @@ in {
   systemd.timers.update-hydra-hosts = {
     enable = true;
     description = "Update hydra hosts";
-    requires = ["update-hydra-hosts.service"];
-    wantedBy = ["multi-user.target"];
+    requires = [ "update-hydra-hosts.service" ];
+    wantedBy = [ "multi-user.target" ];
     timerConfig = {
       OnBootSec = 300;
       OnUnitActiveSec = 300;
     };
   };
-  nix.settings.trusted-users = ["@hydra"];
+  nix.settings.trusted-users = [ "@hydra" ];
   sops.secrets."hydra/ssh/builder_id_ed25519" = {
     sopsFile = ../../secrets/shared.yaml;
     owner = "hydra-queue-runner";
     key = "ssh/builder_id_ed25519";
     path = "/var/lib/hydra/queue-runner/.ssh/builder_id_ed25519";
   };
-  system.activationScripts.setupHydraSshConfig = lib.stringAfter ["var"] ''
+  system.activationScripts.setupHydraSshConfig = lib.stringAfter [ "var" ] ''
     mkdir -p /var/lib/hydra/queue-runner/.ssh/
     chown -Rv hydra-queue-runner /var/lib/hydra/queue-runner
     ln -svf ${sshConfig} /var/lib/hydra/queue-runner/.ssh/config
@@ -189,7 +212,7 @@ in {
 
   systemd.services."attic-queue" = {
     description = "Upload build results";
-    wantedBy = ["multi-user.target"];
+    wantedBy = [ "multi-user.target" ];
     serviceConfig = {
       User = "hydra-queue-runner";
       Group = "hydra";

config/services/hydra/remote-eval-jobs.py

@@ -0,0 +1,61 @@
+#!@python3@
+import sys
+import subprocess
+import os
+import json
+
+# First check if the server is up
+
+if subprocess.call(["@ping@", "-c", "1", "rainbow-resort.int.chir.rs"], stdout=subprocess.DEVNULL).returncode != 0:
+    os.execv("@nix-eval-jobs@", ["@nix-eval-jobs@"] + sys.argv[1:])
+
+inputs_to_copy = set()
+
+remote_args = []
+skip_next = 0
+next_to_copy = False
+next_to_gcroots = False
+gcroots = None
+
+# parse arguments and add them to a list
+
+for arg in sys.argv[1:]:
+    if arg == "--gc-roots-dir" or arg == "--max-jobs" or arg == "--workers":
+        skip_next = 2
+    if arg == "--gc-roots-dir":
+        next_to_gcroots = True
+    if next_to_gcroots:
+        next_to_gcroots = false    
+        gcroots = arg
+    if skip_next > 0:
+        skip_next -= 1
+        continue
+    if next_to_copy:
+        inputs_to_copy.add('='.join(arg.split('=')[1:]))
+        next_to_copy = False
+    if arg == "-I":
+        next_to_copy = True
+    remote_args.append(arg)
+
+remote_args += ["--workers" "4"]
+
+# copy over what files we need to ensure are present on the target
+
+subprocess.call(["@nix@", "copy"] + list(inputs_to_copy) + ["--to", "ssh://build-rainbow-resort", "--no-check-sigs"], check=True, stdout=subprocess.DEVNULL)
+
+# Evaluate on target
+result = subprocess.call(["@ssh@", "build-rainbow-resort", "nix-eval-jobs"] + remote_args, check=True, stdout=subprocess.PIPE, text=True)
+
+for line in result.stdout:
+    try:
+        data = json.loads(line)
+        # copy .drv file home
+        subprocess.call(["@nix@", "copy", data["drvPath"], "--from", "ssh://build-rainbow-resort", "--no-check-sigs"], check=True, stdout=subprocess.DEVNULL)
+        # if we have a gcroot, add it to it
+        if gcroots is not None:
+            drvBasename = os.path.basename(data["drvPath"])
+            os.symlink(data["drvPath"], os.path.join(gcroots, drvBasename))
+        # Now we are done with this job, we can tell hydra about it
+        print(line)
+    except e:
+        print(e, file=sys.stderr)