darkkirb/nixos-config
1
0
Fork 0
Code Issues 1 Pull requests 1 Projects Releases Packages Wiki Activity Actions

Merge pull request 'fix-hydra-evaluator' (#635) from fix-hydra-evaluator into main
All checks were successful
Hydra jobsets Hydra build #29331 of nixos-config:.jobsets:jobsets
Details

Browse source 
Reviewed-on: #635
This commit is contained in:
Charlotte 🦝 Delenk 2024-12-01 18:40:26 +00:00
commit f3ce29bf7c
Signed by: gitea-bot
GPG key ID: C9974EDF9932B558
2 changed files with 92 additions and 75 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

132
config/services/hydra.nix
View file

@ -7,49 +7,52 @@
hydra, hydra,
nix-eval-jobs, nix-eval-jobs,
... ...
}: let }:
let
machines = pkgs.writeText "machines" '' machines = pkgs.writeText "machines" ''
localhost armv7l-linux,powerpc-linux,powerpc64-linux,powerpc64le-linux,wasm32-wasi,x86_64-linux,i686-linux,riscv32-linux,riscv64-linux - 12 1 kvm,nixos-test,big-parallel,benchmark,gccarch-znver1,gccarch-skylake,ca-derivations - localhost armv7l-linux,powerpc-linux,powerpc64-linux,powerpc64le-linux,wasm32-wasi,x86_64-linux,i686-linux,riscv32-linux,riscv64-linux - 12 1 kvm,nixos-test,big-parallel,benchmark,gccarch-znver1,gccarch-skylake,ca-derivations -
build-aarch64 aarch64-linux,riscv32-linux,riscv64-linux - 4 1 nixos-test,benchmark,ca-derivations,gccarch-armv8-a,gccarch-armv8.1-a,gccarch-armv8.2-a,big-parallel - build-aarch64 aarch64-linux,riscv32-linux,riscv64-linux - 4 1 nixos-test,benchmark,ca-derivations,gccarch-armv8-a,gccarch-armv8.1-a,gccarch-armv8.2-a,big-parallel -
build-riscv riscv64-linux,riscv32-linux - 4 2 nixos-test,benchmark,ca-derivations,gccarch-rv64gc_zba_zbb,gccarch-rv64gc_zba,gccarch-rv64gc_zbb,ccarch-rv64gc,gccarch-rv32gc_zba_zbb,gccarch-rv32gc_zba,gccarch-rv32gc_zbb,gccarch-rv32gc,big-parallel,native-riscv - build-riscv riscv64-linux,riscv32-linux - 4 2 nixos-test,benchmark,ca-derivations,gccarch-rv64gc_zba_zbb,gccarch-rv64gc_zba,gccarch-rv64gc_zbb,ccarch-rv64gc,gccarch-rv32gc_zba_zbb,gccarch-rv32gc_zba,gccarch-rv32gc_zbb,gccarch-rv32gc,big-parallel,native-riscv -
''; '';
sshConfig = pkgs.writeText "ssh-config" '' sshConfig =
Host build-aarch64 home:
Port 22 pkgs.writeText "ssh-config" ''
IdentitiesOnly yes Host build-aarch64
User remote-build Port 22
HostName instance-20221213-1915.int.chir.rs IdentitiesOnly yes
IdentityFile /var/lib/hydra/queue-runner/.ssh/builder_id_ed25519 User remote-build
Host build-nas HostName instance-20221213-1915.int.chir.rs
Port 22 IdentityFile ${home}/.ssh/builder_id_ed25519
IdentitiesOnly yes Host build-nas
User remote-build Port 22
HostName nas.int.chir.rs IdentitiesOnly yes
IdentityFile /var/lib/hydra/queue-runner/.ssh/builder_id_ed25519 User remote-build
Host build-rainbow-resort HostName nas.int.chir.rs
Port 22 IdentityFile ${home}/.ssh/builder_id_ed25519
IdentitiesOnly yes Host build-rainbow-resort
User remote-build Port 22
HostName rainbow-resort.int.chir.rs IdentitiesOnly yes
IdentityFile /var/lib/hydra/queue-runner/.ssh/builder_id_ed25519 User remote-build
Host build-riscv HostName rainbow-resort.int.chir.rs
Port 22 IdentityFile ${home}/.ssh/builder_id_ed25519
IdentitiesOnly yes Host build-riscv
User remote-build Port 22
HostName not522.tailbab65.ts.net IdentitiesOnly yes
IdentityFile /var/lib/hydra/queue-runner/.ssh/builder_id_ed25519 User remote-build
HostName not522.tailbab65.ts.net
IdentityFile ${home}/.ssh/builder_id_ed25519
Host * Host *
ForwardAgent no ForwardAgent no
Compression no Compression no
ServerAliveInterval 0 ServerAliveInterval 0
ServerAliveCountMax 3 ServerAliveCountMax 3
HashKnownHosts no HashKnownHosts no
UserKnownHostsFile ~/.ssh/known_hosts UserKnownHostsFile ~/.ssh/known_hosts
ControlMaster auto ControlMaster auto
ControlPath ~/.ssh/master-%r@%n:%p ControlPath ~/.ssh/master-%r@%n:%p
ControlPersist 10m ControlPersist 10m
''; '';
nix-eval-jobs-script = pkgs.stdenvNoCC.mkDerivation { nix-eval-jobs-script = pkgs.stdenvNoCC.mkDerivation {
name = "remote-eval-jobs.py"; name = "remote-eval-jobs.py";
src = ./hydra/remote-eval-jobs.py; src = ./hydra/remote-eval-jobs.py;
@ -62,9 +65,11 @@
--subst-var-by nix-eval-jobs ${nix-eval-jobs.packages.x86_64-linux.nix-eval-jobs}/bin/nix-eval-jobs \ --subst-var-by nix-eval-jobs ${nix-eval-jobs.packages.x86_64-linux.nix-eval-jobs}/bin/nix-eval-jobs \
--subst-var-by nix ${pkgs.nix}/bin/nix \ --subst-var-by nix ${pkgs.nix}/bin/nix \
--subst-var-by ssh ${pkgs.openssh}/bin/ssh --subst-var-by ssh ${pkgs.openssh}/bin/ssh
chmod +x $out
''; '';
}; };
in { in
{
imports = [ imports = [
./postgres.nix ./postgres.nix
../../modules/hydra.nix ../../modules/hydra.nix
@ -78,21 +83,17 @@ in {
package = hydra.packages.${system}.hydra.overrideAttrs (super: { package = hydra.packages.${system}.hydra.overrideAttrs (super: {
doCheck = false; doCheck = false;
doInstallCheck = false; doInstallCheck = false;
patches = patches = super.patches or [ ] ++ [
super.patches ./hydra/0001-add-gitea-pulls.patch
or [] ./hydra/0002-unlimit-output.patch
++ [ ./hydra/0003-remove-pr-number-from-github-job-name.patch
./hydra/0001-add-gitea-pulls.patch ./hydra/0004-use-pulls-instead-of-issues.patch
./hydra/0002-unlimit-output.patch ./hydra/0005-only-list-open-prs.patch
./hydra/0003-remove-pr-number-from-github-job-name.patch ./hydra/0006-status-state.patch
./hydra/0004-use-pulls-instead-of-issues.patch ./hydra/0007-hydra-server-findLog-fix-issue-with-ca-derivations-e.patch
./hydra/0005-only-list-open-prs.patch ];
./hydra/0006-status-state.patch
./hydra/0007-hydra-server-findLog-fix-issue-with-ca-derivations-e.patch
];
postPatch = postPatch =
super.postPatch super.postPatch or ""
or ""
+ '' + ''
substituteInPlace src/script/hydra-eval-jobset --replace-fail nix-eval-jobs ${nix-eval-jobs-script} substituteInPlace src/script/hydra-eval-jobset --replace-fail nix-eval-jobs ${nix-eval-jobs-script}
''; '';
@ -140,8 +141,8 @@ in {
"https://" "https://"
"http://" "http://"
]; ];
sops.secrets."services/hydra/gitea_token" = {}; sops.secrets."services/hydra/gitea_token" = { };
sops.secrets."services/hydra/github_token" = {}; sops.secrets."services/hydra/github_token" = { };
sops.secrets."services/hydra/cache-key" = { sops.secrets."services/hydra/cache-key" = {
owner = "hydra-www"; owner = "hydra-www";
mode = "0440"; mode = "0440";
@ -160,7 +161,7 @@ in {
sops.secrets."services/hydra/aws_credentials" = { sops.secrets."services/hydra/aws_credentials" = {
owner = "hydra-queue-runner"; owner = "hydra-queue-runner";
path = "/var/lib/hydra/queue-runner/.aws/credentials"; path = "/var/lib/hydra/queue-runner/.aws/credentials";
restartUnits = ["hydra-notify.service"]; restartUnits = [ "hydra-notify.service" ];
}; };
systemd.services.update-hydra-hosts = { systemd.services.update-hydra-hosts = {
description = "Update hydra hosts"; description = "Update hydra hosts";
@ -178,24 +179,33 @@ in {
systemd.timers.update-hydra-hosts = { systemd.timers.update-hydra-hosts = {
enable = true; enable = true;
description = "Update hydra hosts"; description = "Update hydra hosts";
requires = ["update-hydra-hosts.service"]; requires = [ "update-hydra-hosts.service" ];
wantedBy = ["multi-user.target"]; wantedBy = [ "multi-user.target" ];
timerConfig = { timerConfig = {
OnBootSec = 300; OnBootSec = 300;
OnUnitActiveSec = 300; OnUnitActiveSec = 300;
}; };
}; };
nix.settings.trusted-users = ["@hydra"]; nix.settings.trusted-users = [ "@hydra" ];
sops.secrets."hydra-evaluator/ssh/builder_id_ed25519" = {
sopsFile = ../../secrets/shared.yaml;
owner = "hydra";
key = "ssh/builder_id_ed25519";
path = "/var/lib/hydra/.ssh/builder_id_ed25519";
};
sops.secrets."hydra/ssh/builder_id_ed25519" = { sops.secrets."hydra/ssh/builder_id_ed25519" = {
sopsFile = ../../secrets/shared.yaml; sopsFile = ../../secrets/shared.yaml;
owner = "hydra-queue-runner"; owner = "hydra-queue-runner";
key = "ssh/builder_id_ed25519"; key = "ssh/builder_id_ed25519";
path = "/var/lib/hydra/queue-runner/.ssh/builder_id_ed25519"; path = "/var/lib/hydra/queue-runner/.ssh/builder_id_ed25519";
}; };
system.activationScripts.setupHydraSshConfig = lib.stringAfter ["var"] '' system.activationScripts.setupHydraSshConfig = lib.stringAfter [ "var" ] ''
mkdir -p /var/lib/hydra/queue-runner/.ssh/ mkdir -p /var/lib/hydra/queue-runner/.ssh/
chown -Rv hydra-queue-runner /var/lib/hydra/queue-runner chown -Rv hydra-queue-runner /var/lib/hydra/queue-runner
ln -svf ${sshConfig} /var/lib/hydra/queue-runner/.ssh/config ln -svf ${sshConfig "/var/lib/hydra/queue-runner"} /var/lib/hydra/queue-runner/.ssh/config
mkdir -p /var/lib/hydra/.ssh/
chown -Rv hydra /var/lib/hydra/.ssh
ln -svf ${sshConfig "/var/lib/hydra"} /var/lib/hydra/.ssh/config
''; '';
sops.secrets."attic/config.toml" = { sops.secrets."attic/config.toml" = {
owner = "hydra-queue-runner"; owner = "hydra-queue-runner";
@ -214,7 +224,7 @@ in {
systemd.services."attic-queue" = { systemd.services."attic-queue" = {
description = "Upload build results"; description = "Upload build results";
wantedBy = ["multi-user.target"]; wantedBy = [ "multi-user.target" ];
serviceConfig = { serviceConfig = {
User = "hydra-queue-runner"; User = "hydra-queue-runner";
Group = "hydra"; Group = "hydra";

35
config/services/hydra/remote-eval-jobs.py
View file

@ -3,10 +3,11 @@ import sys
import subprocess import subprocess
import os import os
import json import json
import shlex
# First check if the server is up # First check if the server is up
if subprocess.call(["@ping@", "-c", "1", "rainbow-resort.int.chir.rs"], stdout=subprocess.DEVNULL).returncode != 0: if subprocess.run(["@ping@", "-c", "1", "rainbow-resort.int.chir.rs"], stdout=subprocess.DEVNULL).returncode != 0:
os.execv("@nix-eval-jobs@", ["@nix-eval-jobs@"] + sys.argv[1:]) os.execv("@nix-eval-jobs@", ["@nix-eval-jobs@"] + sys.argv[1:])
inputs_to_copy = set() inputs_to_copy = set()
@ -22,11 +23,11 @@ gcroots = None
for arg in sys.argv[1:]: for arg in sys.argv[1:]:
if arg == "--gc-roots-dir" or arg == "--max-jobs" or arg == "--workers": if arg == "--gc-roots-dir" or arg == "--max-jobs" or arg == "--workers":
skip_next = 2 skip_next = 2
if next_to_gcroots:
next_to_gcroots = False
gcroots = arg
if arg == "--gc-roots-dir": if arg == "--gc-roots-dir":
next_to_gcroots = True next_to_gcroots = True
if next_to_gcroots:
next_to_gcroots = false
gcroots = arg
if skip_next > 0: if skip_next > 0:
skip_next -= 1 skip_next -= 1
continue continue
@ -37,25 +38,31 @@ for arg in sys.argv[1:]:
next_to_copy = True next_to_copy = True
remote_args.append(arg) remote_args.append(arg)
remote_args += ["--workers" "4"] remote_args += ["--workers", "4", "--gc-roots-dir", "/tmp"]
# copy over what files we need to ensure are present on the target if len(inputs_to_copy) != 0:
# copy over what files we need to ensure are present on the target
subprocess.call(["@nix@", "copy"] + list(inputs_to_copy) + ["--to", "ssh://build-rainbow-resort", "--no-check-sigs"], check=True, stdout=subprocess.DEVNULL) subprocess.run(["@nix@", "copy"] + list(inputs_to_copy) + ["--to", "ssh://build-rainbow-resort", "--no-check-sigs"], check=True, stdout=subprocess.DEVNULL)
# Evaluate on target # Evaluate on target
result = subprocess.call(["@ssh@", "build-rainbow-resort", "nix-eval-jobs"] + remote_args, check=True, stdout=subprocess.PIPE, text=True) result = subprocess.Popen(["@ssh@", "build-rainbow-resort", "nix-eval-jobs"] + list(map(shlex.quote, remote_args)), bufsize=1, stdout=subprocess.PIPE, text=True)
for line in result.stdout: for line in iter(result.stdout.readline, ""):
try: try:
line = line.strip()
data = json.loads(line) data = json.loads(line)
# copy .drv file home # copy .drv file home
subprocess.call(["@nix@", "copy", data["drvPath"], "--from", "ssh://build-rainbow-resort", "--no-check-sigs"], check=True, stdout=subprocess.DEVNULL) subprocess.run(["@nix@", "copy", data["drvPath"], "--from", "ssh://build-rainbow-resort", "--no-check-sigs"], check=True, stdout=subprocess.DEVNULL)
# if we have a gcroot, add it to it # if we have a gcroot, add it to it
if gcroots is not None: if gcroots is not None:
drvBasename = os.path.basename(data["drvPath"]) drvBasename = os.path.basename(data["drvPath"])
os.symlink(data["drvPath"], os.path.join(gcroots, drvBasename)) try:
os.symlink(data["drvPath"], os.path.join(gcroots, drvBasename))
except:
pass
# Now we are done with this job, we can tell hydra about it # Now we are done with this job, we can tell hydra about it
print(line) print(line)
except e: except Exception as e:
print(e, file=sys.stderr) print(e, file=sys.stderr)
sys.exit(result.wait())