darkkirb/nixos-config
1
0
Fork 0
Code Issues 1 Pull requests 1 Projects Releases Packages Wiki Activity Actions

add remote hydra eval #634

Merged
darkkirb merged 3 commits from add-remote-hydra-eval into main 2024-12-01 12:15:50 +00:00
Conversation 0 Commits 3 Files changed 4 +486 -211
4 changed files with 486 additions and 211 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

53
config/services/hydra.nix
View file

@ -5,8 +5,10 @@
config,
pkgs,
hydra,
nix-eval-jobs,
...
}: let
}:
let
machines = pkgs.writeText "machines" ''
localhost armv7l-linux,powerpc-linux,powerpc64-linux,powerpc64le-linux,wasm32-wasi,x86_64-linux,i686-linux,riscv32-linux,riscv64-linux - 12 1 kvm,nixos-test,big-parallel,benchmark,gccarch-znver1,gccarch-skylake,ca-derivations -
build-aarch64 aarch64-linux,riscv32-linux,riscv64-linux - 4 1 nixos-test,benchmark,ca-derivations,gccarch-armv8-a,gccarch-armv8.1-a,gccarch-armv8.2-a,big-parallel -
@ -49,7 +51,22 @@
ControlPath ~/.ssh/master-%r@%n:%p
ControlPersist 10m
'';
in {
nix-eval-jobs-script = pkgs.stdenvNoCC.mkDerivation {
name = "remote-eval-jobs.py";
src = ./hydra/remote-eval-jobs.py;
dontUnpack = true;
dontBuild = true;
installPhase = ''
substitute $src $out \
--subst-var-by python3 ${pkgs.python3}/bin/python3 \
--subst-var-by ping ${pkgs.iputils}/bin/ping \
--subst-var-by nix-eval-jobs ${nix-eval-jobs.packages.x86_64-linux.nix-eval-jobs}/bin/nix-eval-jobs \
--subst-var-by nix ${pkgs.nix}/bin/nix \
--subst-var-by ssh ${pkgs.openssh}/bin/ssh
'';
};
in
{
imports = [
./postgres.nix
../../modules/hydra.nix
@ -63,10 +80,7 @@ in {
package = hydra.packages.${system}.hydra.overrideAttrs (super: {
doCheck = false;
doInstallCheck = false;
patches =
super.patches
or []
++ [
patches = super.patches or [ ] ++ [
./hydra/0001-add-gitea-pulls.patch
./hydra/0002-unlimit-output.patch
./hydra/0003-remove-pr-number-from-github-job-name.patch
@ -75,6 +89,11 @@ in {
./hydra/0006-status-state.patch
./hydra/0007-hydra-server-findLog-fix-issue-with-ca-derivations-e.patch
];
postPatch =
super.postPatch or ""
+ ''
substituteInPlace src/script/hydra-eval-jobset --replace-fail nix-eval-jobs ${nix-eval-jobs-script}
'';
});
hydraURL = "https://hydra.chir.rs/";
notificationSender = "hydra@chir.rs";
@ -114,9 +133,13 @@ in {
"/run/hydra-machines"
];
};
nix.settings.allowed-uris = ["github:" "https://" "http://"];
sops.secrets."services/hydra/gitea_token" = {};
sops.secrets."services/hydra/github_token" = {};
nix.settings.allowed-uris = [
"github:"
"https://"
"http://"
];
sops.secrets."services/hydra/gitea_token" = { };
sops.secrets."services/hydra/github_token" = { };
sops.secrets."services/hydra/cache-key" = {
owner = "hydra-www";
mode = "0440";
@ -135,7 +158,7 @@ in {
sops.secrets."services/hydra/aws_credentials" = {
owner = "hydra-queue-runner";
path = "/var/lib/hydra/queue-runner/.aws/credentials";
restartUnits = ["hydra-notify.service"];
restartUnits = [ "hydra-notify.service" ];
};
systemd.services.update-hydra-hosts = {
description = "Update hydra hosts";
@ -153,21 +176,21 @@ in {
systemd.timers.update-hydra-hosts = {
enable = true;
description = "Update hydra hosts";
requires = ["update-hydra-hosts.service"];
wantedBy = ["multi-user.target"];
requires = [ "update-hydra-hosts.service" ];
wantedBy = [ "multi-user.target" ];
timerConfig = {
OnBootSec = 300;
OnUnitActiveSec = 300;
};
};
nix.settings.trusted-users = ["@hydra"];
nix.settings.trusted-users = [ "@hydra" ];
sops.secrets."hydra/ssh/builder_id_ed25519" = {
sopsFile = ../../secrets/shared.yaml;
owner = "hydra-queue-runner";
key = "ssh/builder_id_ed25519";
path = "/var/lib/hydra/queue-runner/.ssh/builder_id_ed25519";
};
system.activationScripts.setupHydraSshConfig = lib.stringAfter ["var"] ''
system.activationScripts.setupHydraSshConfig = lib.stringAfter [ "var" ] ''
mkdir -p /var/lib/hydra/queue-runner/.ssh/
chown -Rv hydra-queue-runner /var/lib/hydra/queue-runner
ln -svf ${sshConfig} /var/lib/hydra/queue-runner/.ssh/config
@ -189,7 +212,7 @@ in {
systemd.services."attic-queue" = {
description = "Upload build results";
wantedBy = ["multi-user.target"];
wantedBy = [ "multi-user.target" ];
serviceConfig = {
User = "hydra-queue-runner";
Group = "hydra";

61
config/services/hydra/remote-eval-jobs.py Normal file
View file

@ -0,0 +1,61 @@
#!@python3@
import sys
import subprocess
import os
import json
# First check if the server is up
if subprocess.call(["@ping@", "-c", "1", "rainbow-resort.int.chir.rs"], stdout=subprocess.DEVNULL).returncode != 0:
os.execv("@nix-eval-jobs@", ["@nix-eval-jobs@"] + sys.argv[1:])
inputs_to_copy = set()
remote_args = []
skip_next = 0
next_to_copy = False
next_to_gcroots = False
gcroots = None
# parse arguments and add them to a list
for arg in sys.argv[1:]:
if arg == "--gc-roots-dir" or arg == "--max-jobs" or arg == "--workers":
skip_next = 2
if arg == "--gc-roots-dir":
next_to_gcroots = True
if next_to_gcroots:
next_to_gcroots = false
gcroots = arg
if skip_next > 0:
skip_next -= 1
continue
if next_to_copy:
inputs_to_copy.add('='.join(arg.split('=')[1:]))
next_to_copy = False
if arg == "-I":
next_to_copy = True
remote_args.append(arg)
remote_args += ["--workers" "4"]
# copy over what files we need to ensure are present on the target
subprocess.call(["@nix@", "copy"] + list(inputs_to_copy) + ["--to", "ssh://build-rainbow-resort", "--no-check-sigs"], check=True, stdout=subprocess.DEVNULL)
# Evaluate on target
result = subprocess.call(["@ssh@", "build-rainbow-resort", "nix-eval-jobs"] + remote_args, check=True, stdout=subprocess.PIPE, text=True)
for line in result.stdout:
try:
data = json.loads(line)
# copy .drv file home
subprocess.call(["@nix@", "copy", data["drvPath"], "--from", "ssh://build-rainbow-resort", "--no-check-sigs"], check=True, stdout=subprocess.DEVNULL)
# if we have a gcroot, add it to it
if gcroots is not None:
drvBasename = os.path.basename(data["drvPath"])
os.symlink(data["drvPath"], os.path.join(gcroots, drvBasename))
# Now we are done with this job, we can tell hydra about it
print(line)
except e:
print(e, file=sys.stderr)

179
flake.lock
View file

@ -301,6 +301,22 @@
"type": "github"
}
},
"flake-compat_3": {
"flake": false,
"locked": {
"lastModified": 1696426674,
"narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=",
"owner": "edolstra",
"repo": "flake-compat",
"rev": "0f9255e01c2351cc7d116c072cb317785dd33b33",
"type": "github"
},
"original": {
"owner": "edolstra",
"repo": "flake-compat",
"type": "github"
}
},
"flake-parts": {
"inputs": {
"nixpkgs-lib": [
@ -364,6 +380,27 @@
"type": "github"
}
},
"flake-parts_4": {
"inputs": {
"nixpkgs-lib": [
"nix-eval-jobs",
"nixpkgs"
]
},
"locked": {
"lastModified": 1730504689,
"narHash": "sha256-hgmguH29K2fvs9szpq2r3pz2/8cJd2LPS+b4tfNFCwE=",
"owner": "hercules-ci",
"repo": "flake-parts",
"rev": "506278e768c2a08bec68eb62932193e341f55c90",
"type": "github"
},
"original": {
"owner": "hercules-ci",
"repo": "flake-parts",
"type": "github"
}
},
"flake-utils": {
"inputs": {
"systems": [
@ -619,6 +656,29 @@
"url": "https://git.lix.systems/lix-project/lix"
}
},
"lix_3": {
"inputs": {
"flake-compat": "flake-compat_3",
"nix2container": "nix2container_2",
"nixpkgs": [
"nix-eval-jobs",
"nixpkgs"
],
"nixpkgs-regression": "nixpkgs-regression_3",
"pre-commit-hooks": "pre-commit-hooks_2"
},
"locked": {
"lastModified": 1732112222,
"narHash": "sha256-H7GN4++a4vE49SUNojZx+FSk4mmpb2ifJUtJMJHProI=",
"rev": "66f6dbda32959dd5cf3a9aaba15af72d037ab7ff",
"type": "tarball",
"url": "https://git.lix.systems/api/v1/repos/lix-project/lix/archive/66f6dbda32959dd5cf3a9aaba15af72d037ab7ff.tar.gz?rev=66f6dbda32959dd5cf3a9aaba15af72d037ab7ff"
},
"original": {
"type": "tarball",
"url": "https://git.lix.systems/lix-project/lix/archive/main.tar.gz"
}
},
"microformats2-parser": {
"flake": false,
"locked": {
@ -684,6 +744,30 @@
"url": "https://git.lix.systems/lix-project/nix-eval-jobs"
}
},
"nix-eval-jobs_2": {
"inputs": {
"flake-parts": "flake-parts_4",
"lix": "lix_3",
"nix-github-actions": "nix-github-actions_2",
"nixpkgs": [
"nixpkgs"
],
"treefmt-nix": "treefmt-nix_3"
},
"locked": {
"lastModified": 1732351635,
"narHash": "sha256-H94CcQ3yamG5+RMxtxXllR02YIlxQ5WD/8PcolO9yEA=",
"ref": "refs/heads/main",
"rev": "dfc286ca3dc49118c30d8d6205d6d6af76c62b7a",
"revCount": 617,
"type": "git",
"url": "https://git.lix.systems/lix-project/nix-eval-jobs"
},
"original": {
"type": "git",
"url": "https://git.lix.systems/lix-project/nix-eval-jobs"
}
},
"nix-gaming": {
"inputs": {
"flake-parts": [
@ -730,6 +814,27 @@
"type": "github"
}
},
"nix-github-actions_2": {
"inputs": {
"nixpkgs": [
"nix-eval-jobs",
"nixpkgs"
]
},
"locked": {
"lastModified": 1731952509,
"narHash": "sha256-p4gB3Rhw8R6Ak4eMl8pqjCPOLCZRqaehZxdZ/mbFClM=",
"owner": "nix-community",
"repo": "nix-github-actions",
"rev": "7b5f051df789b6b20d259924d349a9ba3319b226",
"type": "github"
},
"original": {
"owner": "nix-community",
"repo": "nix-github-actions",
"type": "github"
}
},
"nix2container": {
"flake": false,
"locked": {
@ -747,6 +852,22 @@
}
},
"nix2container_2": {
"flake": false,
"locked": {
"lastModified": 1724996935,
"narHash": "sha256-njRK9vvZ1JJsP8oV2OgkBrpJhgQezI03S7gzskCcHos=",
"owner": "nlewo",
"repo": "nix2container",
"rev": "fa6bb0a1159f55d071ba99331355955ae30b3401",
"type": "github"
},
"original": {
"owner": "nlewo",
"repo": "nix2container",
"type": "github"
}
},
"nix2container_3": {
"inputs": {
"flake-utils": [
"flake-utils"
@ -854,6 +975,22 @@
"type": "github"
}
},
"nixpkgs-regression_3": {
"locked": {
"lastModified": 1643052045,
"narHash": "sha256-uGJ0VXIhWKGXxkeNnq4TvV3CIOkUJ3PAoLZ3HMzNVMw=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2",
"type": "github"
},
"original": {
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2",
"type": "github"
}
},
"nixpkgs-stable": {
"locked": {
"lastModified": 1725826545,
@ -933,6 +1070,22 @@
}
},
"pre-commit-hooks_2": {
"flake": false,
"locked": {
"lastModified": 1726745158,
"narHash": "sha256-D5AegvGoEjt4rkKedmxlSEmC+nNLMBPWFxvmYnVLhjk=",
"owner": "cachix",
"repo": "git-hooks.nix",
"rev": "4e743a6920eab45e8ba0fbe49dc459f1423a4b74",
"type": "github"
},
"original": {
"owner": "cachix",
"repo": "git-hooks.nix",
"type": "github"
}
},
"pre-commit-hooks_3": {
"inputs": {
"flake-compat": [
"flake-compat"
@ -986,12 +1139,13 @@
"lix": "lix_2",
"lix-module": "lix-module",
"naersk": "naersk",
"nix-eval-jobs": "nix-eval-jobs_2",
"nix-gaming": "nix-gaming",
"nix2container": "nix2container_2",
"nix2container": "nix2container_3",
"nixos-hardware": "nixos-hardware",
"nixos-vscode-server": "nixos-vscode-server",
"nixpkgs": "nixpkgs_4",
"pre-commit-hooks": "pre-commit-hooks_2",
"pre-commit-hooks": "pre-commit-hooks_3",
"rust-overlay": "rust-overlay",
"sops-nix": "sops-nix",
"systems": "systems"
@ -1095,6 +1249,27 @@
"type": "github"
}
},
"treefmt-nix_3": {
"inputs": {
"nixpkgs": [
"nix-eval-jobs",
"nixpkgs"
]
},
"locked": {
"lastModified": 1732292307,
"narHash": "sha256-5WSng844vXt8uytT5djmqBCkopyle6ciFgteuA9bJpw=",
"owner": "numtide",
"repo": "treefmt-nix",
"rev": "705df92694af7093dfbb27109ce16d828a79155f",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "treefmt-nix",
"type": "github"
}
},
"umu": {
"inputs": {
"nixpkgs": [

70
flake.nix
View file

@ -126,6 +126,10 @@ rec {
url = "github:nix-community/naersk/master";
inputs.nixpkgs.follows = "nixpkgs";
};
nix-eval-jobs = {
url = "git+https://git.lix.systems/lix-project/nix-eval-jobs";
inputs.nixpkgs.follows = "nixpkgs";
};
nix-gaming = {
url = "github:fufexan/nix-gaming";
inputs.nixpkgs.follows = "nixpkgs";
@ -162,14 +166,16 @@ rec {
systems.url = "github:nix-systems/default";
};
outputs = {
outputs =
{
self,
nixpkgs,
sops-nix,
home-manager,
lix-module,
...
} @ args: let
}@args:
let
systems = [
{
name = "nixos-8gb-fsn1-1"; # Hetzner Server
@ -190,7 +196,9 @@ rec {
}
*/
];
mkPackages = system: let
mkPackages =
system:
let
pkgs = import nixpkgs {
inherit system;
overlays = [
@ -204,8 +212,7 @@ rec {
];
};
common = {
inherit
(pkgs)
inherit (pkgs)
emoji-lotte
emoji-volpeon-blobfox
emoji-volpeon-blobfox-flip
@ -269,22 +276,21 @@ rec {
};
};
in
common // perSystem.${system} or {};
in rec {
nixosConfigurations = builtins.listToAttrs (map
({
common // perSystem.${system} or { };
in
rec {
nixosConfigurations = builtins.listToAttrs (
map (
{
name,
system,
configName ? name,
}: {
inherit name;
value =
nixpkgs.lib.nixosSystem
}:
{
inherit name;
value = nixpkgs.lib.nixosSystem {
inherit system;
specialArgs =
args
// {
specialArgs = args // {
inherit system;
};
modules = [
@ -292,20 +298,27 @@ rec {
./config/default.nix
sops-nix.nixosModules.sops
home-manager.nixosModules.home-manager
({pkgs, ...}: {
home-manager.extraSpecialArgs = args // {inherit system;};
})
(
{ pkgs, ... }:
{
home-manager.extraSpecialArgs = args // {
inherit system;
};
}
)
(import utils/link-input.nix args)
lix-module.nixosModules.default
];
};
})
systems);
}
) systems
);
overlays = {
x86_64-linux = import ./overlays args "x86_64-linux";
aarch64-linux = import ./overlays args "aarch64-linux";
};
devShell.x86_64-linux = let
devShell.x86_64-linux =
let
pkgs = import nixpkgs {
system = "x86_64-linux";
overlays = [
@ -334,18 +347,21 @@ rec {
packages.x86_64-linux = mkPackages "x86_64-linux";
packages.aarch64-linux = mkPackages "aarch64-linux";
hydraJobs =
(builtins.listToAttrs (map
({
(builtins.listToAttrs (
map (
{
name,
system,
...
}: {
}:
{
inherit name;
value = {
${system} = nixosConfigurations.${name}.config.system.build.toplevel;
};
})
systems))
}
) systems
))
// {
inherit devShell;
inherit packages;