From 2518c5e3b05ebd9c8ed185489430af58b3415f59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Charlotte=20=F0=9F=A6=9D=20Delenk?= Date: Sun, 1 Dec 2024 09:48:07 +0100 Subject: [PATCH] add remote hydra eval --- config/services/hydra.nix | 69 +++++++++++++++-------- config/services/hydra/remote-eval-jobs.py | 61 ++++++++++++++++++++ 2 files changed, 107 insertions(+), 23 deletions(-) create mode 100644 config/services/hydra/remote-eval-jobs.py diff --git a/config/services/hydra.nix b/config/services/hydra.nix index dbc0e1bd..a9149863 100644 --- a/config/services/hydra.nix +++ b/config/services/hydra.nix @@ -5,8 +5,10 @@ config, pkgs, hydra, + nix-eval-jobs, ... -}: let +}: +let machines = pkgs.writeText "machines" '' localhost armv7l-linux,powerpc-linux,powerpc64-linux,powerpc64le-linux,wasm32-wasi,x86_64-linux,i686-linux,riscv32-linux,riscv64-linux - 12 1 kvm,nixos-test,big-parallel,benchmark,gccarch-znver1,gccarch-skylake,ca-derivations - build-aarch64 aarch64-linux,riscv32-linux,riscv64-linux - 4 1 nixos-test,benchmark,ca-derivations,gccarch-armv8-a,gccarch-armv8.1-a,gccarch-armv8.2-a,big-parallel - @@ -49,7 +51,22 @@ ControlPath ~/.ssh/master-%r@%n:%p ControlPersist 10m ''; -in { + nix-eval-jobs-script = pkgs.stdenvNoCC.mkDerivation { + name = "remote-eval-jobs.py"; + src = ./hydra/remote-eval-jobs.py; + dontUnpack = true; + dontBuild = true; + installPhase = '' + substitute $src $out \ + --subst-var-by python3 ${pkgs.python3}/bin/python3 + --subst-var-by ping ${pkgs.iputils}/bin/ping + --subst-var-by nix-eval-jobs ${pkgs.nix-eval-jobs}/bin/nix-eval-jobs + --subst-var-by nix ${pkgs.nix}/bin/nix + --subst-var-by ssh ${pkgs.openssh}/bin/ssh + ''; + }; +in +{ imports = [ ./postgres.nix ../../modules/hydra.nix @@ -63,18 +80,20 @@ in { package = hydra.packages.${system}.hydra.overrideAttrs (super: { doCheck = false; doInstallCheck = false; - patches = - super.patches - or [] - ++ [ - ./hydra/0001-add-gitea-pulls.patch - ./hydra/0002-unlimit-output.patch - ./hydra/0003-remove-pr-number-from-github-job-name.patch - ./hydra/0004-use-pulls-instead-of-issues.patch - ./hydra/0005-only-list-open-prs.patch - ./hydra/0006-status-state.patch - ./hydra/0007-hydra-server-findLog-fix-issue-with-ca-derivations-e.patch - ]; + patches = super.patches or [ ] ++ [ + ./hydra/0001-add-gitea-pulls.patch + ./hydra/0002-unlimit-output.patch + ./hydra/0003-remove-pr-number-from-github-job-name.patch + ./hydra/0004-use-pulls-instead-of-issues.patch + ./hydra/0005-only-list-open-prs.patch + ./hydra/0006-status-state.patch + ./hydra/0007-hydra-server-findLog-fix-issue-with-ca-derivations-e.patch + ]; + postPatch = + super.postPatch or "" + + '' + substituteInPlace src/script/hydra-eval-jobset --replace-fail nix-eval-jobs ${nix-eval-jobs-script} + ''; }); hydraURL = "https://hydra.chir.rs/"; notificationSender = "hydra@chir.rs"; @@ -114,9 +133,13 @@ in { "/run/hydra-machines" ]; }; - nix.settings.allowed-uris = ["github:" "https://" "http://"]; - sops.secrets."services/hydra/gitea_token" = {}; - sops.secrets."services/hydra/github_token" = {}; + nix.settings.allowed-uris = [ + "github:" + "https://" + "http://" + ]; + sops.secrets."services/hydra/gitea_token" = { }; + sops.secrets."services/hydra/github_token" = { }; sops.secrets."services/hydra/cache-key" = { owner = "hydra-www"; mode = "0440"; @@ -135,7 +158,7 @@ in { sops.secrets."services/hydra/aws_credentials" = { owner = "hydra-queue-runner"; path = "/var/lib/hydra/queue-runner/.aws/credentials"; - restartUnits = ["hydra-notify.service"]; + restartUnits = [ "hydra-notify.service" ]; }; systemd.services.update-hydra-hosts = { description = "Update hydra hosts"; @@ -153,21 +176,21 @@ in { systemd.timers.update-hydra-hosts = { enable = true; description = "Update hydra hosts"; - requires = ["update-hydra-hosts.service"]; - wantedBy = ["multi-user.target"]; + requires = [ "update-hydra-hosts.service" ]; + wantedBy = [ "multi-user.target" ]; timerConfig = { OnBootSec = 300; OnUnitActiveSec = 300; }; }; - nix.settings.trusted-users = ["@hydra"]; + nix.settings.trusted-users = [ "@hydra" ]; sops.secrets."hydra/ssh/builder_id_ed25519" = { sopsFile = ../../secrets/shared.yaml; owner = "hydra-queue-runner"; key = "ssh/builder_id_ed25519"; path = "/var/lib/hydra/queue-runner/.ssh/builder_id_ed25519"; }; - system.activationScripts.setupHydraSshConfig = lib.stringAfter ["var"] '' + system.activationScripts.setupHydraSshConfig = lib.stringAfter [ "var" ] '' mkdir -p /var/lib/hydra/queue-runner/.ssh/ chown -Rv hydra-queue-runner /var/lib/hydra/queue-runner ln -svf ${sshConfig} /var/lib/hydra/queue-runner/.ssh/config @@ -189,7 +212,7 @@ in { systemd.services."attic-queue" = { description = "Upload build results"; - wantedBy = ["multi-user.target"]; + wantedBy = [ "multi-user.target" ]; serviceConfig = { User = "hydra-queue-runner"; Group = "hydra"; diff --git a/config/services/hydra/remote-eval-jobs.py b/config/services/hydra/remote-eval-jobs.py new file mode 100644 index 00000000..bb33f8b9 --- /dev/null +++ b/config/services/hydra/remote-eval-jobs.py @@ -0,0 +1,61 @@ +#!@python3@ +import sys +import subprocess +import os +import json + +# First check if the server is up + +if subprocess.call(["@ping@", "-c", "1", "rainbow-resort.int.chir.rs"], stdout=subprocess.DEVNULL).returncode != 0: + os.execv("@nix-eval-jobs@", ["@nix-eval-jobs@"] + sys.argv[1:]) + +inputs_to_copy = set() + +remote_args = [] +skip_next = 0 +next_to_copy = False +next_to_gcroots = False +gcroots = None + +# parse arguments and add them to a list + +for arg in sys.argv[1:]: + if arg == "--gc-roots-dir" or arg == "--max-jobs" or arg == "--workers": + skip_next = 2 + if arg == "--gc-roots-dir": + next_to_gcroots = True + if next_to_gcroots: + next_to_gcroots = false + gcroots = arg + if skip_next > 0: + skip_next -= 1 + continue + if next_to_copy: + inputs_to_copy.add('='.join(arg.split('=')[1:])) + next_to_copy = False + if arg == "-I": + next_to_copy = True + remote_args.append(arg) + +remote_args += ["--workers" "4"] + +# copy over what files we need to ensure are present on the target + +subprocess.call(["@nix@", "copy"] + list(inputs_to_copy) + ["--to", "ssh://build-rainbow-resort", "--no-check-sigs"], check=True, stdout=subprocess.DEVNULL) + +# Evaluate on target +result = subprocess.call(["@ssh@", "build-rainbow-resort", "nix-eval-jobs"] + remote_args, check=True, stdout=subprocess.PIPE, text=True) + +for line in result.stdout: + try: + data = json.loads(line) + # copy .drv file home + subprocess.call(["@nix@", "copy", data["drvPath"], "--from", "ssh://build-rainbow-resort", "--no-check-sigs"], check=True, stdout=subprocess.DEVNULL) + # if we have a gcroot, add it to it + if gcroots is not None: + drvBasename = os.path.basename(data["drvPath"]) + os.symlink(data["drvPath"], os.path.join(gcroots, drvBasename)) + # Now we are done with this job, we can tell hydra about it + print(line) + except e: + print(e, file=sys.stderr) \ No newline at end of file