add remote hydra eval

config/services/hydra.nix

@@ -5,8 +5,10 @@
   config,
   pkgs,
   hydra,
+  nix-eval-jobs,
   ...
-}: let
+}:
+let
   machines = pkgs.writeText "machines" ''
     localhost armv7l-linux,powerpc-linux,powerpc64-linux,powerpc64le-linux,wasm32-wasi,x86_64-linux,i686-linux,riscv32-linux,riscv64-linux - 12 1 kvm,nixos-test,big-parallel,benchmark,gccarch-znver1,gccarch-skylake,ca-derivations  -
     build-aarch64 aarch64-linux,riscv32-linux,riscv64-linux - 4 1 nixos-test,benchmark,ca-derivations,gccarch-armv8-a,gccarch-armv8.1-a,gccarch-armv8.2-a,big-parallel  -
@@ -49,7 +51,22 @@
       ControlPath ~/.ssh/master-%r@%n:%p
       ControlPersist 10m
   '';
-in {
+  nix-eval-jobs-script = pkgs.stdenvNoCC.mkDerivation {
+    name = "remote-eval-jobs.py";
+    src = ./hydra/remote-eval-jobs.py;
+    dontUnpack = true;
+    dontBuild = true;
+    installPhase = ''
+      substitute $src $out \
+        --subst-var-by python3 ${pkgs.python3}/bin/python3
+        --subst-var-by ping ${pkgs.iputils}/bin/ping
+        --subst-var-by nix-eval-jobs ${pkgs.nix-eval-jobs}/bin/nix-eval-jobs
+        --subst-var-by nix ${pkgs.nix}/bin/nix
+        --subst-var-by ssh ${pkgs.openssh}/bin/ssh
+    '';
+  };
+in
+{
   imports = [
     ./postgres.nix
     ../../modules/hydra.nix
@@ -63,10 +80,7 @@ in {
     package = hydra.packages.${system}.hydra.overrideAttrs (super: {
       doCheck = false;
       doInstallCheck = false;
-      patches =
-        super.patches
-        or []
-        ++ [
+      patches = super.patches or [ ] ++ [
         ./hydra/0001-add-gitea-pulls.patch
         ./hydra/0002-unlimit-output.patch
         ./hydra/0003-remove-pr-number-from-github-job-name.patch
@@ -75,6 +89,11 @@ in {
         ./hydra/0006-status-state.patch
         ./hydra/0007-hydra-server-findLog-fix-issue-with-ca-derivations-e.patch
       ];
+      postPatch =
+        super.postPatch or ""
+        + ''
+          substituteInPlace src/script/hydra-eval-jobset --replace-fail nix-eval-jobs ${nix-eval-jobs-script}
+        '';
     });
     hydraURL = "https://hydra.chir.rs/";
     notificationSender = "hydra@chir.rs";
@@ -114,7 +133,11 @@ in {
       "/run/hydra-machines"
     ];
   };
-  nix.settings.allowed-uris = ["github:" "https://" "http://"];
+  nix.settings.allowed-uris = [
+    "github:"
+    "https://"
+    "http://"
+  ];
   sops.secrets."services/hydra/gitea_token" = { };
   sops.secrets."services/hydra/github_token" = { };
   sops.secrets."services/hydra/cache-key" = {

config/services/hydra/remote-eval-jobs.py

@@ -0,0 +1,61 @@
+#!@python3@
+import sys
+import subprocess
+import os
+import json
+
+# First check if the server is up
+
+if subprocess.call(["@ping@", "-c", "1", "rainbow-resort.int.chir.rs"], stdout=subprocess.DEVNULL).returncode != 0:
+    os.execv("@nix-eval-jobs@", ["@nix-eval-jobs@"] + sys.argv[1:])
+
+inputs_to_copy = set()
+
+remote_args = []
+skip_next = 0
+next_to_copy = False
+next_to_gcroots = False
+gcroots = None
+
+# parse arguments and add them to a list
+
+for arg in sys.argv[1:]:
+    if arg == "--gc-roots-dir" or arg == "--max-jobs" or arg == "--workers":
+        skip_next = 2
+    if arg == "--gc-roots-dir":
+        next_to_gcroots = True
+    if next_to_gcroots:
+        next_to_gcroots = false    
+        gcroots = arg
+    if skip_next > 0:
+        skip_next -= 1
+        continue
+    if next_to_copy:
+        inputs_to_copy.add('='.join(arg.split('=')[1:]))
+        next_to_copy = False
+    if arg == "-I":
+        next_to_copy = True
+    remote_args.append(arg)
+
+remote_args += ["--workers" "4"]
+
+# copy over what files we need to ensure are present on the target
+
+subprocess.call(["@nix@", "copy"] + list(inputs_to_copy) + ["--to", "ssh://build-rainbow-resort", "--no-check-sigs"], check=True, stdout=subprocess.DEVNULL)
+
+# Evaluate on target
+result = subprocess.call(["@ssh@", "build-rainbow-resort", "nix-eval-jobs"] + remote_args, check=True, stdout=subprocess.PIPE, text=True)
+
+for line in result.stdout:
+    try:
+        data = json.loads(line)
+        # copy .drv file home
+        subprocess.call(["@nix@", "copy", data["drvPath"], "--from", "ssh://build-rainbow-resort", "--no-check-sigs"], check=True, stdout=subprocess.DEVNULL)
+        # if we have a gcroot, add it to it
+        if gcroots is not None:
+            drvBasename = os.path.basename(data["drvPath"])
+            os.symlink(data["drvPath"], os.path.join(gcroots, drvBasename))
+        # Now we are done with this job, we can tell hydra about it
+        print(line)
+    except e:
+        print(e, file=sys.stderr)