Merge pull request 'add remote hydra eval' (#634) from add-remote-hydra-eval into main

Reviewed-on: #634

config/services/hydra.nix

@@ -5,8 +5,10 @@
   config,
   pkgs,
   hydra,
+  nix-eval-jobs,
   ...
-}: let
+}:
+let
   machines = pkgs.writeText "machines" ''
     localhost armv7l-linux,powerpc-linux,powerpc64-linux,powerpc64le-linux,wasm32-wasi,x86_64-linux,i686-linux,riscv32-linux,riscv64-linux - 12 1 kvm,nixos-test,big-parallel,benchmark,gccarch-znver1,gccarch-skylake,ca-derivations  -
     build-aarch64 aarch64-linux,riscv32-linux,riscv64-linux - 4 1 nixos-test,benchmark,ca-derivations,gccarch-armv8-a,gccarch-armv8.1-a,gccarch-armv8.2-a,big-parallel  -
@@ -49,7 +51,22 @@
       ControlPath ~/.ssh/master-%r@%n:%p
       ControlPersist 10m
   '';
-in {
+  nix-eval-jobs-script = pkgs.stdenvNoCC.mkDerivation {
+    name = "remote-eval-jobs.py";
+    src = ./hydra/remote-eval-jobs.py;
+    dontUnpack = true;
+    dontBuild = true;
+    installPhase = ''
+      substitute $src $out \
+        --subst-var-by python3 ${pkgs.python3}/bin/python3 \
+        --subst-var-by ping ${pkgs.iputils}/bin/ping \
+        --subst-var-by nix-eval-jobs ${nix-eval-jobs.packages.x86_64-linux.nix-eval-jobs}/bin/nix-eval-jobs \
+        --subst-var-by nix ${pkgs.nix}/bin/nix \
+        --subst-var-by ssh ${pkgs.openssh}/bin/ssh
+    '';
+  };
+in
+{
   imports = [
     ./postgres.nix
     ../../modules/hydra.nix
@@ -63,10 +80,7 @@ in {
     package = hydra.packages.${system}.hydra.overrideAttrs (super: {
       doCheck = false;
       doInstallCheck = false;
-      patches =
+      patches = super.patches or [ ] ++ [
-        super.patches
-        or []
-        ++ [
         ./hydra/0001-add-gitea-pulls.patch
         ./hydra/0002-unlimit-output.patch
         ./hydra/0003-remove-pr-number-from-github-job-name.patch
@@ -75,6 +89,11 @@ in {
         ./hydra/0006-status-state.patch
         ./hydra/0007-hydra-server-findLog-fix-issue-with-ca-derivations-e.patch
       ];
+      postPatch =
+        super.postPatch or ""
+        + ''
+          substituteInPlace src/script/hydra-eval-jobset --replace-fail nix-eval-jobs ${nix-eval-jobs-script}
+        '';
     });
     hydraURL = "https://hydra.chir.rs/";
     notificationSender = "hydra@chir.rs";
@@ -114,7 +133,11 @@ in {
       "/run/hydra-machines"
     ];
   };
-  nix.settings.allowed-uris = ["github:" "https://" "http://"];
+  nix.settings.allowed-uris = [
+    "github:"
+    "https://"
+    "http://"
+  ];
   sops.secrets."services/hydra/gitea_token" = { };
   sops.secrets."services/hydra/github_token" = { };
   sops.secrets."services/hydra/cache-key" = {

config/services/hydra/remote-eval-jobs.py

@@ -0,0 +1,61 @@
+#!@python3@
+import sys
+import subprocess
+import os
+import json
+
+# First check if the server is up
+
+if subprocess.call(["@ping@", "-c", "1", "rainbow-resort.int.chir.rs"], stdout=subprocess.DEVNULL).returncode != 0:
+    os.execv("@nix-eval-jobs@", ["@nix-eval-jobs@"] + sys.argv[1:])
+
+inputs_to_copy = set()
+
+remote_args = []
+skip_next = 0
+next_to_copy = False
+next_to_gcroots = False
+gcroots = None
+
+# parse arguments and add them to a list
+
+for arg in sys.argv[1:]:
+    if arg == "--gc-roots-dir" or arg == "--max-jobs" or arg == "--workers":
+        skip_next = 2
+    if arg == "--gc-roots-dir":
+        next_to_gcroots = True
+    if next_to_gcroots:
+        next_to_gcroots = false    
+        gcroots = arg
+    if skip_next > 0:
+        skip_next -= 1
+        continue
+    if next_to_copy:
+        inputs_to_copy.add('='.join(arg.split('=')[1:]))
+        next_to_copy = False
+    if arg == "-I":
+        next_to_copy = True
+    remote_args.append(arg)
+
+remote_args += ["--workers" "4"]
+
+# copy over what files we need to ensure are present on the target
+
+subprocess.call(["@nix@", "copy"] + list(inputs_to_copy) + ["--to", "ssh://build-rainbow-resort", "--no-check-sigs"], check=True, stdout=subprocess.DEVNULL)
+
+# Evaluate on target
+result = subprocess.call(["@ssh@", "build-rainbow-resort", "nix-eval-jobs"] + remote_args, check=True, stdout=subprocess.PIPE, text=True)
+
+for line in result.stdout:
+    try:
+        data = json.loads(line)
+        # copy .drv file home
+        subprocess.call(["@nix@", "copy", data["drvPath"], "--from", "ssh://build-rainbow-resort", "--no-check-sigs"], check=True, stdout=subprocess.DEVNULL)
+        # if we have a gcroot, add it to it
+        if gcroots is not None:
+            drvBasename = os.path.basename(data["drvPath"])
+            os.symlink(data["drvPath"], os.path.join(gcroots, drvBasename))
+        # Now we are done with this job, we can tell hydra about it
+        print(line)
+    except e:
+        print(e, file=sys.stderr)

flake.lock

@@ -301,6 +301,22 @@
         "type": "github"
       }
     },
+    "flake-compat_3": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1696426674,
+        "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=",
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33",
+        "type": "github"
+      },
+      "original": {
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
     "flake-parts": {
       "inputs": {
         "nixpkgs-lib": [
@@ -364,6 +380,27 @@
         "type": "github"
       }
     },
+    "flake-parts_4": {
+      "inputs": {
+        "nixpkgs-lib": [
+          "nix-eval-jobs",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1730504689,
+        "narHash": "sha256-hgmguH29K2fvs9szpq2r3pz2/8cJd2LPS+b4tfNFCwE=",
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "rev": "506278e768c2a08bec68eb62932193e341f55c90",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "type": "github"
+      }
+    },
     "flake-utils": {
       "inputs": {
         "systems": [
@@ -619,6 +656,29 @@
         "url": "https://git.lix.systems/lix-project/lix"
       }
     },
+    "lix_3": {
+      "inputs": {
+        "flake-compat": "flake-compat_3",
+        "nix2container": "nix2container_2",
+        "nixpkgs": [
+          "nix-eval-jobs",
+          "nixpkgs"
+        ],
+        "nixpkgs-regression": "nixpkgs-regression_3",
+        "pre-commit-hooks": "pre-commit-hooks_2"
+      },
+      "locked": {
+        "lastModified": 1732112222,
+        "narHash": "sha256-H7GN4++a4vE49SUNojZx+FSk4mmpb2ifJUtJMJHProI=",
+        "rev": "66f6dbda32959dd5cf3a9aaba15af72d037ab7ff",
+        "type": "tarball",
+        "url": "https://git.lix.systems/api/v1/repos/lix-project/lix/archive/66f6dbda32959dd5cf3a9aaba15af72d037ab7ff.tar.gz?rev=66f6dbda32959dd5cf3a9aaba15af72d037ab7ff"
+      },
+      "original": {
+        "type": "tarball",
+        "url": "https://git.lix.systems/lix-project/lix/archive/main.tar.gz"
+      }
+    },
     "microformats2-parser": {
       "flake": false,
       "locked": {
@@ -684,6 +744,30 @@
         "url": "https://git.lix.systems/lix-project/nix-eval-jobs"
       }
     },
+    "nix-eval-jobs_2": {
+      "inputs": {
+        "flake-parts": "flake-parts_4",
+        "lix": "lix_3",
+        "nix-github-actions": "nix-github-actions_2",
+        "nixpkgs": [
+          "nixpkgs"
+        ],
+        "treefmt-nix": "treefmt-nix_3"
+      },
+      "locked": {
+        "lastModified": 1732351635,
+        "narHash": "sha256-H94CcQ3yamG5+RMxtxXllR02YIlxQ5WD/8PcolO9yEA=",
+        "ref": "refs/heads/main",
+        "rev": "dfc286ca3dc49118c30d8d6205d6d6af76c62b7a",
+        "revCount": 617,
+        "type": "git",
+        "url": "https://git.lix.systems/lix-project/nix-eval-jobs"
+      },
+      "original": {
+        "type": "git",
+        "url": "https://git.lix.systems/lix-project/nix-eval-jobs"
+      }
+    },
     "nix-gaming": {
       "inputs": {
         "flake-parts": [
@@ -730,6 +814,27 @@
         "type": "github"
       }
     },
+    "nix-github-actions_2": {
+      "inputs": {
+        "nixpkgs": [
+          "nix-eval-jobs",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1731952509,
+        "narHash": "sha256-p4gB3Rhw8R6Ak4eMl8pqjCPOLCZRqaehZxdZ/mbFClM=",
+        "owner": "nix-community",
+        "repo": "nix-github-actions",
+        "rev": "7b5f051df789b6b20d259924d349a9ba3319b226",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "nix-github-actions",
+        "type": "github"
+      }
+    },
     "nix2container": {
       "flake": false,
       "locked": {
@@ -747,6 +852,22 @@
       }
     },
     "nix2container_2": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1724996935,
+        "narHash": "sha256-njRK9vvZ1JJsP8oV2OgkBrpJhgQezI03S7gzskCcHos=",
+        "owner": "nlewo",
+        "repo": "nix2container",
+        "rev": "fa6bb0a1159f55d071ba99331355955ae30b3401",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nlewo",
+        "repo": "nix2container",
+        "type": "github"
+      }
+    },
+    "nix2container_3": {
       "inputs": {
         "flake-utils": [
           "flake-utils"
@@ -854,6 +975,22 @@
         "type": "github"
       }
     },
+    "nixpkgs-regression_3": {
+      "locked": {
+        "lastModified": 1643052045,
+        "narHash": "sha256-uGJ0VXIhWKGXxkeNnq4TvV3CIOkUJ3PAoLZ3HMzNVMw=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2",
+        "type": "github"
+      }
+    },
     "nixpkgs-stable": {
       "locked": {
         "lastModified": 1725826545,
@@ -933,6 +1070,22 @@
       }
     },
     "pre-commit-hooks_2": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1726745158,
+        "narHash": "sha256-D5AegvGoEjt4rkKedmxlSEmC+nNLMBPWFxvmYnVLhjk=",
+        "owner": "cachix",
+        "repo": "git-hooks.nix",
+        "rev": "4e743a6920eab45e8ba0fbe49dc459f1423a4b74",
+        "type": "github"
+      },
+      "original": {
+        "owner": "cachix",
+        "repo": "git-hooks.nix",
+        "type": "github"
+      }
+    },
+    "pre-commit-hooks_3": {
       "inputs": {
         "flake-compat": [
           "flake-compat"
@@ -986,12 +1139,13 @@
         "lix": "lix_2",
         "lix-module": "lix-module",
         "naersk": "naersk",
+        "nix-eval-jobs": "nix-eval-jobs_2",
         "nix-gaming": "nix-gaming",
-        "nix2container": "nix2container_2",
+        "nix2container": "nix2container_3",
         "nixos-hardware": "nixos-hardware",
         "nixos-vscode-server": "nixos-vscode-server",
         "nixpkgs": "nixpkgs_4",
-        "pre-commit-hooks": "pre-commit-hooks_2",
+        "pre-commit-hooks": "pre-commit-hooks_3",
         "rust-overlay": "rust-overlay",
         "sops-nix": "sops-nix",
         "systems": "systems"
@@ -1095,6 +1249,27 @@
         "type": "github"
       }
     },
+    "treefmt-nix_3": {
+      "inputs": {
+        "nixpkgs": [
+          "nix-eval-jobs",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1732292307,
+        "narHash": "sha256-5WSng844vXt8uytT5djmqBCkopyle6ciFgteuA9bJpw=",
+        "owner": "numtide",
+        "repo": "treefmt-nix",
+        "rev": "705df92694af7093dfbb27109ce16d828a79155f",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "treefmt-nix",
+        "type": "github"
+      }
+    },
     "umu": {
       "inputs": {
         "nixpkgs": [

flake.nix

@@ -126,6 +126,10 @@ rec {
       url = "github:nix-community/naersk/master";
       inputs.nixpkgs.follows = "nixpkgs";
     };
+    nix-eval-jobs = {
+      url = "git+https://git.lix.systems/lix-project/nix-eval-jobs";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
     nix-gaming = {
       url = "github:fufexan/nix-gaming";
       inputs.nixpkgs.follows = "nixpkgs";
@@ -162,14 +166,16 @@ rec {
     systems.url = "github:nix-systems/default";
   };
 
-  outputs = {
+  outputs =
+    {
       self,
       nixpkgs,
       sops-nix,
       home-manager,
       lix-module,
       ...
-  } @ args: let
+    }@args:
+    let
       systems = [
         {
           name = "nixos-8gb-fsn1-1"; # Hetzner Server
@@ -190,7 +196,9 @@ rec {
           }
         */
       ];
-    mkPackages = system: let
+      mkPackages =
+        system:
+        let
           pkgs = import nixpkgs {
             inherit system;
             overlays = [
@@ -204,8 +212,7 @@ rec {
             ];
           };
           common = {
-        inherit
+            inherit (pkgs)
-          (pkgs)
               emoji-lotte
               emoji-volpeon-blobfox
               emoji-volpeon-blobfox-flip
@@ -270,21 +277,20 @@ rec {
           };
         in
         common // perSystem.${system} or { };
-  in rec {
+    in
-    nixosConfigurations = builtins.listToAttrs (map
+    rec {
-      ({
+      nixosConfigurations = builtins.listToAttrs (
+        map (
+          {
             name,
             system,
             configName ? name,
-      }: {
+          }:
-        inherit name;
-        value =
-          nixpkgs.lib.nixosSystem
           {
+            inherit name;
+            value = nixpkgs.lib.nixosSystem {
               inherit system;
-            specialArgs =
+              specialArgs = args // {
-              args
-              // {
                 inherit system;
               };
               modules = [
@@ -292,20 +298,27 @@ rec {
                 ./config/default.nix
                 sops-nix.nixosModules.sops
                 home-manager.nixosModules.home-manager
-              ({pkgs, ...}: {
+                (
-                home-manager.extraSpecialArgs = args // {inherit system;};
+                  { pkgs, ... }:
-              })
+                  {
+                    home-manager.extraSpecialArgs = args // {
+                      inherit system;
+                    };
+                  }
+                )
                 (import utils/link-input.nix args)
                 lix-module.nixosModules.default
               ];
             };
-      })
+          }
-      systems);
+        ) systems
+      );
       overlays = {
         x86_64-linux = import ./overlays args "x86_64-linux";
         aarch64-linux = import ./overlays args "aarch64-linux";
       };
-    devShell.x86_64-linux = let
+      devShell.x86_64-linux =
+        let
           pkgs = import nixpkgs {
             system = "x86_64-linux";
             overlays = [
@@ -334,18 +347,21 @@ rec {
       packages.x86_64-linux = mkPackages "x86_64-linux";
       packages.aarch64-linux = mkPackages "aarch64-linux";
       hydraJobs =
-      (builtins.listToAttrs (map
+        (builtins.listToAttrs (
-        ({
+          map (
+            {
               name,
               system,
               ...
-        }: {
+            }:
+            {
               inherit name;
               value = {
                 ${system} = nixosConfigurations.${name}.config.system.build.toplevel;
               };
-        })
+            }
-        systems))
+          ) systems
+        ))
         // {
           inherit devShell;
           inherit packages;