Merge pull request 'fix-hydra-evaluator' (#635) from fix-hydra-evaluator into main

Reviewed-on: #635

config/services/hydra.nix

@@ -7,37 +7,40 @@
   hydra,
   nix-eval-jobs,
   ...
-}: let
+}:
+let
   machines = pkgs.writeText "machines" ''
     localhost armv7l-linux,powerpc-linux,powerpc64-linux,powerpc64le-linux,wasm32-wasi,x86_64-linux,i686-linux,riscv32-linux,riscv64-linux - 12 1 kvm,nixos-test,big-parallel,benchmark,gccarch-znver1,gccarch-skylake,ca-derivations  -
     build-aarch64 aarch64-linux,riscv32-linux,riscv64-linux - 4 1 nixos-test,benchmark,ca-derivations,gccarch-armv8-a,gccarch-armv8.1-a,gccarch-armv8.2-a,big-parallel  -
     build-riscv riscv64-linux,riscv32-linux - 4 2 nixos-test,benchmark,ca-derivations,gccarch-rv64gc_zba_zbb,gccarch-rv64gc_zba,gccarch-rv64gc_zbb,ccarch-rv64gc,gccarch-rv32gc_zba_zbb,gccarch-rv32gc_zba,gccarch-rv32gc_zbb,gccarch-rv32gc,big-parallel,native-riscv  -
   '';
-  sshConfig = pkgs.writeText "ssh-config" ''
+  sshConfig =
+    home:
+    pkgs.writeText "ssh-config" ''
       Host build-aarch64
         Port 22
         IdentitiesOnly yes
         User remote-build
         HostName instance-20221213-1915.int.chir.rs
-      IdentityFile /var/lib/hydra/queue-runner/.ssh/builder_id_ed25519
+        IdentityFile ${home}/.ssh/builder_id_ed25519
       Host build-nas
         Port 22
         IdentitiesOnly yes
         User remote-build
         HostName nas.int.chir.rs
-      IdentityFile /var/lib/hydra/queue-runner/.ssh/builder_id_ed25519
+        IdentityFile ${home}/.ssh/builder_id_ed25519
       Host build-rainbow-resort
         Port 22
         IdentitiesOnly yes
         User remote-build
         HostName rainbow-resort.int.chir.rs
-      IdentityFile /var/lib/hydra/queue-runner/.ssh/builder_id_ed25519
+        IdentityFile ${home}/.ssh/builder_id_ed25519
       Host build-riscv
         Port 22
         IdentitiesOnly yes
         User remote-build
         HostName not522.tailbab65.ts.net
-      IdentityFile /var/lib/hydra/queue-runner/.ssh/builder_id_ed25519
+        IdentityFile ${home}/.ssh/builder_id_ed25519
 
       Host *
         ForwardAgent no
@@ -62,9 +65,11 @@
         --subst-var-by nix-eval-jobs ${nix-eval-jobs.packages.x86_64-linux.nix-eval-jobs}/bin/nix-eval-jobs \
         --subst-var-by nix ${pkgs.nix}/bin/nix \
         --subst-var-by ssh ${pkgs.openssh}/bin/ssh
+      chmod +x $out
     '';
   };
-in {
+in
+{
   imports = [
     ./postgres.nix
     ../../modules/hydra.nix
@@ -78,10 +83,7 @@ in {
     package = hydra.packages.${system}.hydra.overrideAttrs (super: {
       doCheck = false;
       doInstallCheck = false;
-      patches =
-        super.patches
-        or []
-        ++ [
+      patches = super.patches or [ ] ++ [
         ./hydra/0001-add-gitea-pulls.patch
         ./hydra/0002-unlimit-output.patch
         ./hydra/0003-remove-pr-number-from-github-job-name.patch
@@ -91,8 +93,7 @@ in {
         ./hydra/0007-hydra-server-findLog-fix-issue-with-ca-derivations-e.patch
       ];
       postPatch =
-        super.postPatch
-        or ""
+        super.postPatch or ""
         + ''
           substituteInPlace src/script/hydra-eval-jobset --replace-fail nix-eval-jobs ${nix-eval-jobs-script}
         '';
@@ -140,8 +141,8 @@ in {
     "https://"
     "http://"
   ];
-  sops.secrets."services/hydra/gitea_token" = {};
-  sops.secrets."services/hydra/github_token" = {};
+  sops.secrets."services/hydra/gitea_token" = { };
+  sops.secrets."services/hydra/github_token" = { };
   sops.secrets."services/hydra/cache-key" = {
     owner = "hydra-www";
     mode = "0440";
@@ -160,7 +161,7 @@ in {
   sops.secrets."services/hydra/aws_credentials" = {
     owner = "hydra-queue-runner";
     path = "/var/lib/hydra/queue-runner/.aws/credentials";
-    restartUnits = ["hydra-notify.service"];
+    restartUnits = [ "hydra-notify.service" ];
   };
   systemd.services.update-hydra-hosts = {
     description = "Update hydra hosts";
@@ -178,24 +179,33 @@ in {
   systemd.timers.update-hydra-hosts = {
     enable = true;
     description = "Update hydra hosts";
-    requires = ["update-hydra-hosts.service"];
-    wantedBy = ["multi-user.target"];
+    requires = [ "update-hydra-hosts.service" ];
+    wantedBy = [ "multi-user.target" ];
     timerConfig = {
       OnBootSec = 300;
       OnUnitActiveSec = 300;
     };
   };
-  nix.settings.trusted-users = ["@hydra"];
+  nix.settings.trusted-users = [ "@hydra" ];
+  sops.secrets."hydra-evaluator/ssh/builder_id_ed25519" = {
+    sopsFile = ../../secrets/shared.yaml;
+    owner = "hydra";
+    key = "ssh/builder_id_ed25519";
+    path = "/var/lib/hydra/.ssh/builder_id_ed25519";
+  };
   sops.secrets."hydra/ssh/builder_id_ed25519" = {
     sopsFile = ../../secrets/shared.yaml;
     owner = "hydra-queue-runner";
     key = "ssh/builder_id_ed25519";
     path = "/var/lib/hydra/queue-runner/.ssh/builder_id_ed25519";
   };
-  system.activationScripts.setupHydraSshConfig = lib.stringAfter ["var"] ''
+  system.activationScripts.setupHydraSshConfig = lib.stringAfter [ "var" ] ''
     mkdir -p /var/lib/hydra/queue-runner/.ssh/
     chown -Rv hydra-queue-runner /var/lib/hydra/queue-runner
-    ln -svf ${sshConfig} /var/lib/hydra/queue-runner/.ssh/config
+    ln -svf ${sshConfig "/var/lib/hydra/queue-runner"} /var/lib/hydra/queue-runner/.ssh/config
+    mkdir -p /var/lib/hydra/.ssh/
+    chown -Rv hydra /var/lib/hydra/.ssh
+    ln -svf ${sshConfig "/var/lib/hydra"} /var/lib/hydra/.ssh/config
   '';
   sops.secrets."attic/config.toml" = {
     owner = "hydra-queue-runner";
@@ -214,7 +224,7 @@ in {
 
   systemd.services."attic-queue" = {
     description = "Upload build results";
-    wantedBy = ["multi-user.target"];
+    wantedBy = [ "multi-user.target" ];
     serviceConfig = {
       User = "hydra-queue-runner";
       Group = "hydra";

config/services/hydra/remote-eval-jobs.py

@@ -3,10 +3,11 @@ import sys
 import subprocess
 import os
 import json
+import shlex
 
 # First check if the server is up
 
-if subprocess.call(["@ping@", "-c", "1", "rainbow-resort.int.chir.rs"], stdout=subprocess.DEVNULL).returncode != 0:
+if subprocess.run(["@ping@", "-c", "1", "rainbow-resort.int.chir.rs"], stdout=subprocess.DEVNULL).returncode != 0:
     os.execv("@nix-eval-jobs@", ["@nix-eval-jobs@"] + sys.argv[1:])
 
 inputs_to_copy = set()
@@ -22,11 +23,11 @@ gcroots = None
 for arg in sys.argv[1:]:
     if arg == "--gc-roots-dir" or arg == "--max-jobs" or arg == "--workers":
         skip_next = 2
+    if next_to_gcroots:
+        next_to_gcroots = False    
+        gcroots = arg
     if arg == "--gc-roots-dir":
         next_to_gcroots = True
-    if next_to_gcroots:
-        next_to_gcroots = false    
-        gcroots = arg
     if skip_next > 0:
         skip_next -= 1
         continue
@@ -37,25 +38,31 @@ for arg in sys.argv[1:]:
         next_to_copy = True
     remote_args.append(arg)
 
-remote_args += ["--workers" "4"]
+remote_args += ["--workers", "4", "--gc-roots-dir", "/tmp"]
 
-# copy over what files we need to ensure are present on the target
-
-subprocess.call(["@nix@", "copy"] + list(inputs_to_copy) + ["--to", "ssh://build-rainbow-resort", "--no-check-sigs"], check=True, stdout=subprocess.DEVNULL)
+if len(inputs_to_copy) != 0:
+    # copy over what files we need to ensure are present on the target
+    subprocess.run(["@nix@", "copy"] + list(inputs_to_copy) + ["--to", "ssh://build-rainbow-resort", "--no-check-sigs"], check=True, stdout=subprocess.DEVNULL)
 
 # Evaluate on target
-result = subprocess.call(["@ssh@", "build-rainbow-resort", "nix-eval-jobs"] + remote_args, check=True, stdout=subprocess.PIPE, text=True)
+result = subprocess.Popen(["@ssh@", "build-rainbow-resort", "nix-eval-jobs"] + list(map(shlex.quote, remote_args)), bufsize=1, stdout=subprocess.PIPE, text=True)
 
-for line in result.stdout:
+for line in iter(result.stdout.readline, ""):
     try:
+        line = line.strip()
         data = json.loads(line)
         # copy .drv file home
-        subprocess.call(["@nix@", "copy", data["drvPath"], "--from", "ssh://build-rainbow-resort", "--no-check-sigs"], check=True, stdout=subprocess.DEVNULL)
+        subprocess.run(["@nix@", "copy", data["drvPath"], "--from", "ssh://build-rainbow-resort", "--no-check-sigs"], check=True, stdout=subprocess.DEVNULL)
         # if we have a gcroot, add it to it
         if gcroots is not None:
             drvBasename = os.path.basename(data["drvPath"])
+            try:
                 os.symlink(data["drvPath"], os.path.join(gcroots, drvBasename))
+            except:
+                pass
         # Now we are done with this job, we can tell hydra about it
         print(line)
-    except e:
+    except Exception as e:
         print(e, file=sys.stderr)
+
+sys.exit(result.wait())