From e776a3be17dd45fc694f790e3d819765bca1a9de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Charlotte=20=F0=9F=A6=9D=20Delenk?= Date: Sat, 14 Jan 2023 14:57:00 +0100 Subject: [PATCH] Improve remote building --- config/default.nix | 5 +++++ config/home-manager/base.nix | 1 - config/home-manager/darkkirb.nix | 1 + config/home-manager/root.nix | 1 + config/instance-20221213-1915.nix | 1 + config/nas.nix | 8 +------ config/nix.nix | 9 ++++++++ config/nutty-noon.nix | 35 +------------------------------ config/programs/builders.nix | 28 +++++++++++++++++++++++++ config/programs/ssh.nix | 16 -------------- config/services/hydra.nix | 2 ++ config/users/remote-build.nix | 13 ++++++++++++ secrets/shared.yaml | 6 ++++-- 13 files changed, 66 insertions(+), 60 deletions(-) create mode 100644 config/programs/builders.nix create mode 100644 config/users/remote-build.nix diff --git a/config/default.nix b/config/default.nix index a5303b57..80238ecb 100644 --- a/config/default.nix +++ b/config/default.nix @@ -74,6 +74,11 @@ key = "aws/credentials"; path = "/root/.aws/credentials"; }; + sops.secrets."ssh/builder_id_ed25519" = { + sopsFile = ../secrets/shared.yaml; + owner = "root"; + key = "ssh/builder_id_ed25519"; + }; networking.nameservers = ["fd0d:a262:1fa6:e621:b4e1:08ff:e658:6f49" "fd0d:a262:1fa6:e621:746d:4523:5c04:1453"]; # Archetype configuration diff --git a/config/home-manager/base.nix b/config/home-manager/base.nix index b14c9251..3ba04409 100644 --- a/config/home-manager/base.nix +++ b/config/home-manager/base.nix @@ -3,7 +3,6 @@ desktop: {pkgs, ...}: { (import ../programs/zsh.nix desktop) (import ../programs/helix desktop) ../programs/tmux.nix - ../programs/ssh.nix ../programs/taskwarrior.nix ../programs/mail.nix ]; diff --git a/config/home-manager/darkkirb.nix b/config/home-manager/darkkirb.nix index 37b8b663..88885126 100644 --- a/config/home-manager/darkkirb.nix +++ b/config/home-manager/darkkirb.nix @@ -5,6 +5,7 @@ imports = [ (import ./base.nix desktop) + ../programs/ssh.nix ../programs/gpg.nix ../programs/git.nix ../programs/direnv.nix diff --git a/config/home-manager/root.nix b/config/home-manager/root.nix index 49b6773f..1957fd33 100644 --- a/config/home-manager/root.nix +++ b/config/home-manager/root.nix @@ -1,5 +1,6 @@ {pkgs, ...}: { imports = [ (import ./base.nix false) + ../programs/builders.nix ]; } diff --git a/config/instance-20221213-1915.nix b/config/instance-20221213-1915.nix index 828aa5a3..8fc8252a 100644 --- a/config/instance-20221213-1915.nix +++ b/config/instance-20221213-1915.nix @@ -16,6 +16,7 @@ ./services/named-submissive.nix ./services/shitalloverme.nix ./services/chir.rs + ./users/remote-build.nix ]; boot.initrd.availableKernelModules = ["xhci_pci" "virtio_pci" "usbhid"]; diff --git a/config/nas.nix b/config/nas.nix index 96aac9b1..67484bb1 100644 --- a/config/nas.nix +++ b/config/nas.nix @@ -33,6 +33,7 @@ ./services/drone.nix ./services/drone-runner-docker.nix ./services/docker.nix + ./users/remote-build.nix ]; hardware.cpu.amd.updateMicrocode = true; @@ -301,11 +302,4 @@ max_parallel_workers = 12; max_parallel_maintenance_workers = 4; }; - - users.users.darkkirb.openssh.authorizedKeys.keys = [ - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDpO0Lh7eOE/EBttb/XWZ6ISiJ0RkmBYfruq3U6linEz root@nixos-8gb-fsn1-1" - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKB8oH1XbuGrKn/SeguXz96sw4AjJQQvZyAdpptotzOr root@thinkrac" - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAN/rVZJuwiO44LwOqimpH4zyGehYUMF2ZhYFXUCkupP hydra-queue-runner@nas" - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFLEmOYG4xipOh2YsWGbQtvoJXQzToQDotyCRFnHpVP5 root@instance-20221213-1915" - ]; } diff --git a/config/nix.nix b/config/nix.nix index 4dabd41b..92a4d3d6 100644 --- a/config/nix.nix +++ b/config/nix.nix @@ -81,6 +81,15 @@ in { speedFactor = 2; supportedFeatures = ["kvm" "nixos-test" "big-parallel" "benchmark" "gccarch-znver2" "gccarch-znver1" "gccarch-skylake" "ca-derivations"]; } + { + hostName = "build-aarch64"; + systems = [ + "aarch64-linux" + ]; + maxJobs = 2; + speedFactor = 10; + supportedFeatures = ["nixos-test" "benchmark" "ca-derivations"]; + } ]; distributedBuilds = true; }; diff --git a/config/nutty-noon.nix b/config/nutty-noon.nix index 0d10b9c6..762c9d48 100644 --- a/config/nutty-noon.nix +++ b/config/nutty-noon.nix @@ -20,6 +20,7 @@ nixos-hardware.nixosModules.common-pc-ssd ./services/postgres.nix ./services/drone-runner-docker.nix + ./users/remote-build.nix ]; hardware.cpu.amd.updateMicrocode = true; boot.initrd.availableKernelModules = ["nvme" "xhci_pci" "ahci" "usb_storage" "usbhid" "sd_mod" "sr_mod" "k10temp"]; @@ -156,33 +157,6 @@ "riscv64-linux" "wasm32-wasi" ]; - nix.buildMachines = lib.mkForce [ - #{ - # hostName = "build-nas"; - # systems = [ "x86_64-linux" ]; - # maxJobs = 12; - # speedFactor = 1; - # supportedFeatures = [ "gccarch-znver1" "ca-derivations" ]; - #} - { - maxJobs = 16; - speedFactor = 2; - hostName = "localhost"; - systems = [ - "armv7l-linux" - "aarch64-linux" - "powerpc-linux" - "powerpc64-linux" - "powerpc64le-linux" - "riscv32-linux" - "riscv64-linux" - "wasm32-wasi" - "x86_64-linux" - "i686-linux" - ]; - supportedFeatures = ["kvm" "nixos-test" "big-parallel" "benchmark" "gccarch-znver2" "gccarch-znver1" "gccarch-skylake" "ca-derivations"]; - } - ]; hardware.enableRedistributableFirmware = true; nix.daemonCPUSchedPolicy = "idle"; nix.daemonIOSchedClass = "idle"; @@ -196,14 +170,7 @@ endpoint = "192.168.2.1:51820"; } ]; - # Build server stuff - users.users.darkkirb.openssh.authorizedKeys.keys = [ - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDpO0Lh7eOE/EBttb/XWZ6ISiJ0RkmBYfruq3U6linEz root@nixos-8gb-fsn1-1" - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKB8oH1XbuGrKn/SeguXz96sw4AjJQQvZyAdpptotzOr root@thinkrac" - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAN/rVZJuwiO44LwOqimpH4zyGehYUMF2ZhYFXUCkupP hydra-queue-runner@nas" - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFLEmOYG4xipOh2YsWGbQtvoJXQzToQDotyCRFnHpVP5 root@instance-20221213-1915" - ]; nix.settings.system-features = [ "kvm" "nixos-test" diff --git a/config/programs/builders.nix b/config/programs/builders.nix new file mode 100644 index 00000000..5db6a219 --- /dev/null +++ b/config/programs/builders.nix @@ -0,0 +1,28 @@ +_: { + programs.ssh = { + enable = true; + matchBlocks = { + "build-nas" = { + hostname = "nas.int.chir.rs"; + identitiesOnly = true; + identityFile = "/run/secrets/builder_id_ed25519"; + port = 22; + user = "remote-build"; + }; + "build-pc" = { + hostname = "nutty-noon.int.chir.rs"; + identitiesOnly = true; + identityFile = "/run/secrets/builder_id_ed25519"; + port = 22; + user = "remote-build"; + }; + "build-aarch64" = { + hostname = "instance-20221213-1915.int.chir.rs"; + identitiesOnly = true; + identityFile = "/run/secrets/builder_id_ed25519"; + port = 22; + user = "remote-build"; + }; + }; + }; +} diff --git a/config/programs/ssh.nix b/config/programs/ssh.nix index c7f33b15..9604bc0d 100644 --- a/config/programs/ssh.nix +++ b/config/programs/ssh.nix @@ -3,21 +3,5 @@ _: { controlMaster = "auto"; controlPersist = "10m"; enable = true; - matchBlocks = { - "build-nas" = { - hostname = "backup.int.chir.rs"; - identitiesOnly = true; - identityFile = "~/.ssh/id_ed25519"; - port = 22; - user = "darkkirb"; - }; - "build-pc" = { - hostname = "nutty-noon.int.chir.rs"; - identitiesOnly = true; - identityFile = "~/.ssh/id_ed25519"; - port = 22; - user = "darkkirb"; - }; - }; }; } diff --git a/config/services/hydra.nix b/config/services/hydra.nix index 4177ab8f..542cec4f 100644 --- a/config/services/hydra.nix +++ b/config/services/hydra.nix @@ -15,6 +15,7 @@ clean-cache = nix-packages.packages.${system}.clean-s3-cache; machines = pkgs.writeText "machines" '' localhost armv7l-linux,aarch64-linux,powerpc-linux,powerpc64-linux,powerpc64le-linux,riscv32-linux,riscv64-linux,wasm32-wasi,x86_64-linux,i686-linux - 12 1 kvm,nixos-test,big-parallel,benchmark,gccarch-znver1,gccarch-skylake,ca-derivations - + ssh://build-aarch64 aarch64-linux - 2 10 nixos-test,benchmark,ca-derivations - ''; in { imports = [ @@ -121,4 +122,5 @@ in { }; }; nix.settings.trusted-users = ["@hydra"]; + sops.secrets."ssh/builder_id_ed25519".owner = lib.mkForce "hydra"; } diff --git a/config/users/remote-build.nix b/config/users/remote-build.nix new file mode 100644 index 00000000..dfafdc68 --- /dev/null +++ b/config/users/remote-build.nix @@ -0,0 +1,13 @@ +{config, ...}: { + users.users.remote-build = { + createHome = true; + description = "Remote builder"; + group = "users"; + home = "/home/remote-build"; + isNormalUser = true; + uid = 1002; + openssh.authorizedKeys.keys = [ + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINN5Q/L2FyB3DIgdJRYnTGHW3naw5VQ9coOdwHYmv0aZ darkkirb@thinkrac" + ]; + }; +} diff --git a/secrets/shared.yaml b/secrets/shared.yaml index e7ee506e..fe9e85e4 100644 --- a/secrets/shared.yaml +++ b/secrets/shared.yaml @@ -1,5 +1,7 @@ aws: credentials: ENC[AES256_GCM,data:Y+SbRqf7VotqN9CDMa8SmwBx+nlODeQScr7nlH90qyoa+5GtdDlgvqtbHJ4yE7QoiOBsG/CZ4sBnpXadyUhM8Erq8M2q0CAvlrb3oqgtNAgrkiLueqPAC6kTVBhsDolgWdrkMA8FVifwwqBtns9M7Q==,iv:4XheIFj90luZWxErPppOdphbw7ZXlGbeieSUl97eHAA=,tag:CrCDqHOXTRezf47vsdVAdw==,type:str] +ssh: + builder_id_ed25519: ENC[AES256_GCM,data:SjNV5HtKVjQd+cwCPGGgT9bSFKhdwJxqGclCBfWAm4UzTco/ho3TZV9OX/BxI/W0ztzSlctFUecOi98qAdtWX737dlyVmQpkGcvbDIQWt6JqpWRGsLsJ1lhlmSOIS1jkASyGksCLaSou3FZ1+dQ2+BWyh9XjWpC7nCGvEHsGOn8cCSj3tlV8cloBZKzxpwEXkEUmpZpuJb0PZp8LL0okxLF553NxClS1zty0cmucRcR71ObevwiWvJmJNI9Un1D4FhSDYFHffoOtmyHixZnNRMI8gW7Cx1suHIsslrsI4YgmD+QxJrIkx96ajkPzIBPsMu5mO+h8y2epEJpRenbTMhgdQvnspsJ1PrWOzsHWMTI9EsjmiZZEsk6lRiVgnOQwf0Bc+Sf4uro6TcbgsIWJdwOthXWi7s72JHfGHqcP03eQyA8wZf21w4oTGh4L5aDtwb29fOLdSv1xf20PP5Qiy4iUdvCV7tdsLcEA0i3aUeJCkklk9cUoWELT+Vu6CqMxnLbSeKHnP452EkK52EZnpn1MNVtFczPWW+oj,iv:7KO7yFoHCttTpw6gDcZRA43qW6F1a8xqpa5VRYUerz4=,tag:OlCbnoS0vQO1Wyn6iWlYDw==,type:str] sops: kms: [] gcp_kms: [] @@ -51,8 +53,8 @@ sops: ZFNubXhZdG1KVDB1d0FWOVVDS1ovOHcKO5m7BFeZzt+nBfaZJoH8Pkw6aeDExQrQ Gfp6KQ0oJOuquhZtMW0GpLuKnuQjjGEBaIbcZcR4OosKKlLYfOKabA== -----END AGE ENCRYPTED FILE----- - lastmodified: "2022-12-14T15:34:33Z" - mac: ENC[AES256_GCM,data:8Fnws66NKfdVl+0+63HZvgt+PTxTZ6L6y9sh2hFtWR8RNeDTmtfZXOrJhJjU8XRUogQovnv+qSn8hmMRQ4f0tw1+jEIjEMqGV9ex2YWhsryvuhKbXH11un7UYObCxiwhTbxq6aOQLJ+74V3a2YLWTnKBd3X383EV1QrH/quXOIg=,iv:8yIW+mCveO59Kb4dXxIUzMT2/PGfknDDBoxGBRQNaE8=,tag:4BJpIdEeMeNCvOEXmvsSPQ==,type:str] + lastmodified: "2023-01-14T13:36:39Z" + mac: ENC[AES256_GCM,data:tES+i3vMTywPJ6DnYw9sKdVvixmgLBJ27tjMSJvOdeMth4p3soQA6RicRzQCgBlMXC+dHXpCMmvvtQrzDpFudiG6+xUPW/cz77ARdgzOWLOSD8cgzOp5XBI3koFOq5pxK7v1GMv7IG01pFKEMVes8mGUQXvu+eXMx1ZPcQrQuxI=,iv:1DYSbBNR8l2VLTnst+Aw+nSLPOKc+jCpZ9ViPERbWq4=,tag:9+f2s9FSUajVqQXb0eziKg==,type:str] pgp: - created_at: "2022-12-14T15:34:33Z" enc: |