Merge pull request 'add remote hydra eval' (#634) from add-remote-hydra-eval into main

Reviewed-on: #634

config/services/hydra.nix

@@ -5,8 +5,10 @@
   config,
   pkgs,
   hydra,
+  nix-eval-jobs,
   ...
-}: let
+}:
+let
   machines = pkgs.writeText "machines" ''
     localhost armv7l-linux,powerpc-linux,powerpc64-linux,powerpc64le-linux,wasm32-wasi,x86_64-linux,i686-linux,riscv32-linux,riscv64-linux - 12 1 kvm,nixos-test,big-parallel,benchmark,gccarch-znver1,gccarch-skylake,ca-derivations  -
     build-aarch64 aarch64-linux,riscv32-linux,riscv64-linux - 4 1 nixos-test,benchmark,ca-derivations,gccarch-armv8-a,gccarch-armv8.1-a,gccarch-armv8.2-a,big-parallel  -
@@ -49,7 +51,22 @@
       ControlPath ~/.ssh/master-%r@%n:%p
       ControlPersist 10m
   '';
-in {
+  nix-eval-jobs-script = pkgs.stdenvNoCC.mkDerivation {
+    name = "remote-eval-jobs.py";
+    src = ./hydra/remote-eval-jobs.py;
+    dontUnpack = true;
+    dontBuild = true;
+    installPhase = ''
+      substitute $src $out \
+        --subst-var-by python3 ${pkgs.python3}/bin/python3 \
+        --subst-var-by ping ${pkgs.iputils}/bin/ping \
+        --subst-var-by nix-eval-jobs ${nix-eval-jobs.packages.x86_64-linux.nix-eval-jobs}/bin/nix-eval-jobs \
+        --subst-var-by nix ${pkgs.nix}/bin/nix \
+        --subst-var-by ssh ${pkgs.openssh}/bin/ssh
+    '';
+  };
+in
+{
   imports = [
     ./postgres.nix
     ../../modules/hydra.nix
@@ -63,18 +80,20 @@ in {
     package = hydra.packages.${system}.hydra.overrideAttrs (super: {
       doCheck = false;
       doInstallCheck = false;
-      patches =
+      patches = super.patches or [ ] ++ [
-        super.patches
+        ./hydra/0001-add-gitea-pulls.patch
-        or []
+        ./hydra/0002-unlimit-output.patch
-        ++ [
+        ./hydra/0003-remove-pr-number-from-github-job-name.patch
-          ./hydra/0001-add-gitea-pulls.patch
+        ./hydra/0004-use-pulls-instead-of-issues.patch
-          ./hydra/0002-unlimit-output.patch
+        ./hydra/0005-only-list-open-prs.patch
-          ./hydra/0003-remove-pr-number-from-github-job-name.patch
+        ./hydra/0006-status-state.patch
-          ./hydra/0004-use-pulls-instead-of-issues.patch
+        ./hydra/0007-hydra-server-findLog-fix-issue-with-ca-derivations-e.patch
-          ./hydra/0005-only-list-open-prs.patch
+      ];
-          ./hydra/0006-status-state.patch
+      postPatch =
-          ./hydra/0007-hydra-server-findLog-fix-issue-with-ca-derivations-e.patch
+        super.postPatch or ""
-        ];
+        + ''
+          substituteInPlace src/script/hydra-eval-jobset --replace-fail nix-eval-jobs ${nix-eval-jobs-script}
+        '';
     });
     hydraURL = "https://hydra.chir.rs/";
     notificationSender = "hydra@chir.rs";
@@ -114,9 +133,13 @@ in {
       "/run/hydra-machines"
     ];
   };
-  nix.settings.allowed-uris = ["github:" "https://" "http://"];
+  nix.settings.allowed-uris = [
-  sops.secrets."services/hydra/gitea_token" = {};
+    "github:"
-  sops.secrets."services/hydra/github_token" = {};
+    "https://"
+    "http://"
+  ];
+  sops.secrets."services/hydra/gitea_token" = { };
+  sops.secrets."services/hydra/github_token" = { };
   sops.secrets."services/hydra/cache-key" = {
     owner = "hydra-www";
     mode = "0440";
@@ -135,7 +158,7 @@ in {
   sops.secrets."services/hydra/aws_credentials" = {
     owner = "hydra-queue-runner";
     path = "/var/lib/hydra/queue-runner/.aws/credentials";
-    restartUnits = ["hydra-notify.service"];
+    restartUnits = [ "hydra-notify.service" ];
   };
   systemd.services.update-hydra-hosts = {
     description = "Update hydra hosts";
@@ -153,21 +176,21 @@ in {
   systemd.timers.update-hydra-hosts = {
     enable = true;
     description = "Update hydra hosts";
-    requires = ["update-hydra-hosts.service"];
+    requires = [ "update-hydra-hosts.service" ];
-    wantedBy = ["multi-user.target"];
+    wantedBy = [ "multi-user.target" ];
     timerConfig = {
       OnBootSec = 300;
       OnUnitActiveSec = 300;
     };
   };
-  nix.settings.trusted-users = ["@hydra"];
+  nix.settings.trusted-users = [ "@hydra" ];
   sops.secrets."hydra/ssh/builder_id_ed25519" = {
     sopsFile = ../../secrets/shared.yaml;
     owner = "hydra-queue-runner";
     key = "ssh/builder_id_ed25519";
     path = "/var/lib/hydra/queue-runner/.ssh/builder_id_ed25519";
   };
-  system.activationScripts.setupHydraSshConfig = lib.stringAfter ["var"] ''
+  system.activationScripts.setupHydraSshConfig = lib.stringAfter [ "var" ] ''
     mkdir -p /var/lib/hydra/queue-runner/.ssh/
     chown -Rv hydra-queue-runner /var/lib/hydra/queue-runner
     ln -svf ${sshConfig} /var/lib/hydra/queue-runner/.ssh/config
@@ -189,7 +212,7 @@ in {
 
   systemd.services."attic-queue" = {
     description = "Upload build results";
-    wantedBy = ["multi-user.target"];
+    wantedBy = [ "multi-user.target" ];
     serviceConfig = {
       User = "hydra-queue-runner";
       Group = "hydra";

config/services/hydra/remote-eval-jobs.py

@@ -0,0 +1,61 @@
+#!@python3@
+import sys
+import subprocess
+import os
+import json
+
+# First check if the server is up
+
+if subprocess.call(["@ping@", "-c", "1", "rainbow-resort.int.chir.rs"], stdout=subprocess.DEVNULL).returncode != 0:
+    os.execv("@nix-eval-jobs@", ["@nix-eval-jobs@"] + sys.argv[1:])
+
+inputs_to_copy = set()
+
+remote_args = []
+skip_next = 0
+next_to_copy = False
+next_to_gcroots = False
+gcroots = None
+
+# parse arguments and add them to a list
+
+for arg in sys.argv[1:]:
+    if arg == "--gc-roots-dir" or arg == "--max-jobs" or arg == "--workers":
+        skip_next = 2
+    if arg == "--gc-roots-dir":
+        next_to_gcroots = True
+    if next_to_gcroots:
+        next_to_gcroots = false    
+        gcroots = arg
+    if skip_next > 0:
+        skip_next -= 1
+        continue
+    if next_to_copy:
+        inputs_to_copy.add('='.join(arg.split('=')[1:]))
+        next_to_copy = False
+    if arg == "-I":
+        next_to_copy = True
+    remote_args.append(arg)
+
+remote_args += ["--workers" "4"]
+
+# copy over what files we need to ensure are present on the target
+
+subprocess.call(["@nix@", "copy"] + list(inputs_to_copy) + ["--to", "ssh://build-rainbow-resort", "--no-check-sigs"], check=True, stdout=subprocess.DEVNULL)
+
+# Evaluate on target
+result = subprocess.call(["@ssh@", "build-rainbow-resort", "nix-eval-jobs"] + remote_args, check=True, stdout=subprocess.PIPE, text=True)
+
+for line in result.stdout:
+    try:
+        data = json.loads(line)
+        # copy .drv file home
+        subprocess.call(["@nix@", "copy", data["drvPath"], "--from", "ssh://build-rainbow-resort", "--no-check-sigs"], check=True, stdout=subprocess.DEVNULL)
+        # if we have a gcroot, add it to it
+        if gcroots is not None:
+            drvBasename = os.path.basename(data["drvPath"])
+            os.symlink(data["drvPath"], os.path.join(gcroots, drvBasename))
+        # Now we are done with this job, we can tell hydra about it
+        print(line)
+    except e:
+        print(e, file=sys.stderr)

flake.lock

@@ -301,6 +301,22 @@
         "type": "github"
       }
     },
+    "flake-compat_3": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1696426674,
+        "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=",
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33",
+        "type": "github"
+      },
+      "original": {
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
     "flake-parts": {
       "inputs": {
         "nixpkgs-lib": [
@@ -364,6 +380,27 @@
         "type": "github"
       }
     },
+    "flake-parts_4": {
+      "inputs": {
+        "nixpkgs-lib": [
+          "nix-eval-jobs",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1730504689,
+        "narHash": "sha256-hgmguH29K2fvs9szpq2r3pz2/8cJd2LPS+b4tfNFCwE=",
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "rev": "506278e768c2a08bec68eb62932193e341f55c90",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "type": "github"
+      }
+    },
     "flake-utils": {
       "inputs": {
         "systems": [
@@ -619,6 +656,29 @@
         "url": "https://git.lix.systems/lix-project/lix"
       }
     },
+    "lix_3": {
+      "inputs": {
+        "flake-compat": "flake-compat_3",
+        "nix2container": "nix2container_2",
+        "nixpkgs": [
+          "nix-eval-jobs",
+          "nixpkgs"
+        ],
+        "nixpkgs-regression": "nixpkgs-regression_3",
+        "pre-commit-hooks": "pre-commit-hooks_2"
+      },
+      "locked": {
+        "lastModified": 1732112222,
+        "narHash": "sha256-H7GN4++a4vE49SUNojZx+FSk4mmpb2ifJUtJMJHProI=",
+        "rev": "66f6dbda32959dd5cf3a9aaba15af72d037ab7ff",
+        "type": "tarball",
+        "url": "https://git.lix.systems/api/v1/repos/lix-project/lix/archive/66f6dbda32959dd5cf3a9aaba15af72d037ab7ff.tar.gz?rev=66f6dbda32959dd5cf3a9aaba15af72d037ab7ff"
+      },
+      "original": {
+        "type": "tarball",
+        "url": "https://git.lix.systems/lix-project/lix/archive/main.tar.gz"
+      }
+    },
     "microformats2-parser": {
       "flake": false,
       "locked": {
@@ -684,6 +744,30 @@
         "url": "https://git.lix.systems/lix-project/nix-eval-jobs"
       }
     },
+    "nix-eval-jobs_2": {
+      "inputs": {
+        "flake-parts": "flake-parts_4",
+        "lix": "lix_3",
+        "nix-github-actions": "nix-github-actions_2",
+        "nixpkgs": [
+          "nixpkgs"
+        ],
+        "treefmt-nix": "treefmt-nix_3"
+      },
+      "locked": {
+        "lastModified": 1732351635,
+        "narHash": "sha256-H94CcQ3yamG5+RMxtxXllR02YIlxQ5WD/8PcolO9yEA=",
+        "ref": "refs/heads/main",
+        "rev": "dfc286ca3dc49118c30d8d6205d6d6af76c62b7a",
+        "revCount": 617,
+        "type": "git",
+        "url": "https://git.lix.systems/lix-project/nix-eval-jobs"
+      },
+      "original": {
+        "type": "git",
+        "url": "https://git.lix.systems/lix-project/nix-eval-jobs"
+      }
+    },
     "nix-gaming": {
       "inputs": {
         "flake-parts": [
@@ -730,6 +814,27 @@
         "type": "github"
       }
     },
+    "nix-github-actions_2": {
+      "inputs": {
+        "nixpkgs": [
+          "nix-eval-jobs",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1731952509,
+        "narHash": "sha256-p4gB3Rhw8R6Ak4eMl8pqjCPOLCZRqaehZxdZ/mbFClM=",
+        "owner": "nix-community",
+        "repo": "nix-github-actions",
+        "rev": "7b5f051df789b6b20d259924d349a9ba3319b226",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "nix-github-actions",
+        "type": "github"
+      }
+    },
     "nix2container": {
       "flake": false,
       "locked": {
@@ -747,6 +852,22 @@
       }
     },
     "nix2container_2": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1724996935,
+        "narHash": "sha256-njRK9vvZ1JJsP8oV2OgkBrpJhgQezI03S7gzskCcHos=",
+        "owner": "nlewo",
+        "repo": "nix2container",
+        "rev": "fa6bb0a1159f55d071ba99331355955ae30b3401",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nlewo",
+        "repo": "nix2container",
+        "type": "github"
+      }
+    },
+    "nix2container_3": {
       "inputs": {
         "flake-utils": [
           "flake-utils"
@@ -854,6 +975,22 @@
         "type": "github"
       }
     },
+    "nixpkgs-regression_3": {
+      "locked": {
+        "lastModified": 1643052045,
+        "narHash": "sha256-uGJ0VXIhWKGXxkeNnq4TvV3CIOkUJ3PAoLZ3HMzNVMw=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2",
+        "type": "github"
+      }
+    },
     "nixpkgs-stable": {
       "locked": {
         "lastModified": 1725826545,
@@ -933,6 +1070,22 @@
       }
     },
     "pre-commit-hooks_2": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1726745158,
+        "narHash": "sha256-D5AegvGoEjt4rkKedmxlSEmC+nNLMBPWFxvmYnVLhjk=",
+        "owner": "cachix",
+        "repo": "git-hooks.nix",
+        "rev": "4e743a6920eab45e8ba0fbe49dc459f1423a4b74",
+        "type": "github"
+      },
+      "original": {
+        "owner": "cachix",
+        "repo": "git-hooks.nix",
+        "type": "github"
+      }
+    },
+    "pre-commit-hooks_3": {
       "inputs": {
         "flake-compat": [
           "flake-compat"
@@ -986,12 +1139,13 @@
         "lix": "lix_2",
         "lix-module": "lix-module",
         "naersk": "naersk",
+        "nix-eval-jobs": "nix-eval-jobs_2",
         "nix-gaming": "nix-gaming",
-        "nix2container": "nix2container_2",
+        "nix2container": "nix2container_3",
         "nixos-hardware": "nixos-hardware",
         "nixos-vscode-server": "nixos-vscode-server",
         "nixpkgs": "nixpkgs_4",
-        "pre-commit-hooks": "pre-commit-hooks_2",
+        "pre-commit-hooks": "pre-commit-hooks_3",
         "rust-overlay": "rust-overlay",
         "sops-nix": "sops-nix",
         "systems": "systems"
@@ -1095,6 +1249,27 @@
         "type": "github"
       }
     },
+    "treefmt-nix_3": {
+      "inputs": {
+        "nixpkgs": [
+          "nix-eval-jobs",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1732292307,
+        "narHash": "sha256-5WSng844vXt8uytT5djmqBCkopyle6ciFgteuA9bJpw=",
+        "owner": "numtide",
+        "repo": "treefmt-nix",
+        "rev": "705df92694af7093dfbb27109ce16d828a79155f",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "treefmt-nix",
+        "type": "github"
+      }
+    },
     "umu": {
       "inputs": {
         "nixpkgs": [

flake.nix

@@ -126,6 +126,10 @@ rec {
       url = "github:nix-community/naersk/master";
       inputs.nixpkgs.follows = "nixpkgs";
     };
+    nix-eval-jobs = {
+      url = "git+https://git.lix.systems/lix-project/nix-eval-jobs";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
     nix-gaming = {
       url = "github:fufexan/nix-gaming";
       inputs.nixpkgs.follows = "nixpkgs";
@@ -162,196 +166,208 @@ rec {
     systems.url = "github:nix-systems/default";
   };
 
-  outputs = {
+  outputs =
-    self,
+    {
-    nixpkgs,
+      self,
-    sops-nix,
+      nixpkgs,
-    home-manager,
+      sops-nix,
-    lix-module,
+      home-manager,
-    ...
+      lix-module,
-  } @ args: let
+      ...
-    systems = [
+    }@args:
-      {
+    let
-        name = "nixos-8gb-fsn1-1"; # Hetzner Server
+      systems = [
-        system = "x86_64-linux";
-      }
-      {
-        name = "nas"; # My nas
-        system = "x86_64-linux";
-      }
-      {
-        name = "instance-20221213-1915"; # Oracle server
-        system = "aarch64-linux";
-      }
-      /*
         {
-        name = "devterm";
+          name = "nixos-8gb-fsn1-1"; # Hetzner Server
-        system = "aarch64-linux";
+          system = "x86_64-linux";
-      }
+        }
-      */
+        {
-    ];
+          name = "nas"; # My nas
-    mkPackages = system: let
+          system = "x86_64-linux";
-      pkgs = import nixpkgs {
+        }
-        inherit system;
+        {
-        overlays = [
+          name = "instance-20221213-1915"; # Oracle server
-          args.gomod2nix.overlays.default
+          system = "aarch64-linux";
-          self.overlays.${system}
+        }
-          args.hydra.overlays.default
+        /*
-        ];
+            {
-        config.allowUnfree = true;
+            name = "devterm";
-        config.permittedInsecurePackages = [
+            system = "aarch64-linux";
-          "olm-3.2.16"
+          }
-        ];
+        */
-      };
+      ];
-      common = {
+      mkPackages =
-        inherit
+        system:
-          (pkgs)
+        let
-          emoji-lotte
+          pkgs = import nixpkgs {
-          emoji-volpeon-blobfox
-          emoji-volpeon-blobfox-flip
-          emoji-volpeon-bunhd
-          emoji-volpeon-bunhd-flip
-          emoji-volpeon-drgn
-          emoji-volpeon-fox
-          emoji-volpeon-gphn
-          emoji-volpeon-raccoon
-          emoji-volpeon-vlpn
-          emoji-volpeon-neofox
-          emoji-volpeon-neocat
-          emoji-volpeon-floof
-          emoji-rosaflags
-          emoji-raccoon
-          emoji-caro
-          lotte-art
-          alco-sans
-          constructium
-          fairfax
-          fairfax-hd
-          kreative-square
-          nasin-nanpa
-          matrix-media-repo
-          mautrix-discord
-          mautrix-whatsapp
-          mautrix-telegram
-          mautrix-slack
-          python-mautrix
-          python-tulir-telethon
-          papermc
-          python-plover-stroke
-          python-rtf-tokenize
-          plover
-          plover-plugins-manager
-          python-simplefuzzyset
-          plover-plugin-emoji
-          plover-plugin-tapey-tape
-          plover-plugin-yaml-dictionary
-          plover-plugin-machine-hid
-          plover-plugin-rkb1-hid
-          plover-plugin-dotool-output
-          plover-dict-didoesdigital
-          miifox-net
-          plover-plugin-python-dictionary
-          plover-plugin-stenotype-extended
-          asar-asm
-          bsnes-plus
-          yiffstash
-          plover-plugin-dict-commands
-          plover-plugin-last-translation
-          plover-plugin-modal-dictionary
-          plover-plugin-stitching
-          plover-plugin-lapwing-aio
-          mgba-dev
-          ;
-      };
-      perSystem = {
-        aarch64-linux = {
-          #inherit (pkgs) linux-devterm;
-        };
-      };
-    in
-      common // perSystem.${system} or {};
-  in rec {
-    nixosConfigurations = builtins.listToAttrs (map
-      ({
-        name,
-        system,
-        configName ? name,
-      }: {
-        inherit name;
-        value =
-          nixpkgs.lib.nixosSystem
-          {
             inherit system;
-            specialArgs =
+            overlays = [
-              args
+              args.gomod2nix.overlays.default
-              // {
+              self.overlays.${system}
-                inherit system;
+              args.hydra.overlays.default
-              };
+            ];
-            modules = [
+            config.allowUnfree = true;
-              (./config + "/${configName}.nix")
+            config.permittedInsecurePackages = [
-              ./config/default.nix
+              "olm-3.2.16"
-              sops-nix.nixosModules.sops
-              home-manager.nixosModules.home-manager
-              ({pkgs, ...}: {
-                home-manager.extraSpecialArgs = args // {inherit system;};
-              })
-              (import utils/link-input.nix args)
-              lix-module.nixosModules.default
             ];
           };
-      })
+          common = {
-      systems);
+            inherit (pkgs)
-    overlays = {
+              emoji-lotte
-      x86_64-linux = import ./overlays args "x86_64-linux";
+              emoji-volpeon-blobfox
-      aarch64-linux = import ./overlays args "aarch64-linux";
+              emoji-volpeon-blobfox-flip
-    };
+              emoji-volpeon-bunhd
-    devShell.x86_64-linux = let
+              emoji-volpeon-bunhd-flip
-      pkgs = import nixpkgs {
+              emoji-volpeon-drgn
-        system = "x86_64-linux";
+              emoji-volpeon-fox
-        overlays = [
+              emoji-volpeon-gphn
-          args.gomod2nix.overlays.default
+              emoji-volpeon-raccoon
-          self.overlays.x86_64-linux
+              emoji-volpeon-vlpn
-        ];
+              emoji-volpeon-neofox
-      };
+              emoji-volpeon-neocat
-    in
+              emoji-volpeon-floof
-      pkgs.mkShell {
+              emoji-rosaflags
-        nativeBuildInputs = with pkgs; [
+              emoji-raccoon
-          age
+              emoji-caro
-          sops
+              lotte-art
-          ssh-to-age
+              alco-sans
-          nix-prefetch
+              constructium
-          nix-prefetch-git
+              fairfax
-          jq
+              fairfax-hd
-          bundix
+              kreative-square
-          python3
+              nasin-nanpa
-          python3Packages.yapf
+              matrix-media-repo
-          github-cli
+              mautrix-discord
-          statix
+              mautrix-whatsapp
-          alejandra
+              mautrix-telegram
-        ];
+              mautrix-slack
-      };
+              python-mautrix
-    formatter.x86_64-linux = nixpkgs.legacyPackages.x86_64-linux.alejandra;
+              python-tulir-telethon
-    packages.x86_64-linux = mkPackages "x86_64-linux";
+              papermc
-    packages.aarch64-linux = mkPackages "aarch64-linux";
+              python-plover-stroke
-    hydraJobs =
+              python-rtf-tokenize
-      (builtins.listToAttrs (map
+              plover
-        ({
+              plover-plugins-manager
-          name,
+              python-simplefuzzyset
-          system,
+              plover-plugin-emoji
-          ...
+              plover-plugin-tapey-tape
-        }: {
+              plover-plugin-yaml-dictionary
-          inherit name;
+              plover-plugin-machine-hid
-          value = {
+              plover-plugin-rkb1-hid
-            ${system} = nixosConfigurations.${name}.config.system.build.toplevel;
+              plover-plugin-dotool-output
+              plover-dict-didoesdigital
+              miifox-net
+              plover-plugin-python-dictionary
+              plover-plugin-stenotype-extended
+              asar-asm
+              bsnes-plus
+              yiffstash
+              plover-plugin-dict-commands
+              plover-plugin-last-translation
+              plover-plugin-modal-dictionary
+              plover-plugin-stitching
+              plover-plugin-lapwing-aio
+              mgba-dev
+              ;
           };
-        })
+          perSystem = {
-        systems))
+            aarch64-linux = {
-      // {
+              #inherit (pkgs) linux-devterm;
-        inherit devShell;
+            };
-        inherit packages;
+          };
-        # Uncomment the line to build an installer image
+        in
-        # This is EXTREMELY LARGE and will make builds take forever
+        common // perSystem.${system} or { };
-        # installer.x86_64-linux = nixosConfigurations.installer.config.system.build.isoImage;
+    in
+    rec {
+      nixosConfigurations = builtins.listToAttrs (
+        map (
+          {
+            name,
+            system,
+            configName ? name,
+          }:
+          {
+            inherit name;
+            value = nixpkgs.lib.nixosSystem {
+              inherit system;
+              specialArgs = args // {
+                inherit system;
+              };
+              modules = [
+                (./config + "/${configName}.nix")
+                ./config/default.nix
+                sops-nix.nixosModules.sops
+                home-manager.nixosModules.home-manager
+                (
+                  { pkgs, ... }:
+                  {
+                    home-manager.extraSpecialArgs = args // {
+                      inherit system;
+                    };
+                  }
+                )
+                (import utils/link-input.nix args)
+                lix-module.nixosModules.default
+              ];
+            };
+          }
+        ) systems
+      );
+      overlays = {
+        x86_64-linux = import ./overlays args "x86_64-linux";
+        aarch64-linux = import ./overlays args "aarch64-linux";
       };
-  };
+      devShell.x86_64-linux =
+        let
+          pkgs = import nixpkgs {
+            system = "x86_64-linux";
+            overlays = [
+              args.gomod2nix.overlays.default
+              self.overlays.x86_64-linux
+            ];
+          };
+        in
+        pkgs.mkShell {
+          nativeBuildInputs = with pkgs; [
+            age
+            sops
+            ssh-to-age
+            nix-prefetch
+            nix-prefetch-git
+            jq
+            bundix
+            python3
+            python3Packages.yapf
+            github-cli
+            statix
+            alejandra
+          ];
+        };
+      formatter.x86_64-linux = nixpkgs.legacyPackages.x86_64-linux.alejandra;
+      packages.x86_64-linux = mkPackages "x86_64-linux";
+      packages.aarch64-linux = mkPackages "aarch64-linux";
+      hydraJobs =
+        (builtins.listToAttrs (
+          map (
+            {
+              name,
+              system,
+              ...
+            }:
+            {
+              inherit name;
+              value = {
+                ${system} = nixosConfigurations.${name}.config.system.build.toplevel;
+              };
+            }
+          ) systems
+        ))
+        // {
+          inherit devShell;
+          inherit packages;
+          # Uncomment the line to build an installer image
+          # This is EXTREMELY LARGE and will make builds take forever
+          # installer.x86_64-linux = nixosConfigurations.installer.config.system.build.isoImage;
+        };
+    };
 }