Merge pull request #69 from DarkKirb/switch-to-hydra

Switch to Hydra as CI

.github/workflows/build.yml

@@ -1,103 +0,0 @@
-name: build
-on:
-  pull_request:
-    branches:
-      - main
-jobs:
-  build:
-    strategy:
-      matrix:
-        package:
-          - akkoma
-          - pleroma-fe
-          - admin-fe
-          - emoji-lotte
-          - emoji-volpeon-blobfox
-          - emoji-volpeon-blobfox-flip
-          - emoji-volpeon-bunhd
-          - emoji-volpeon-bunhd-flip
-          - emoji-volpeon-drgn
-          - emoji-volpeon-fox
-          - emoji-volpeon-gphn
-          - emoji-volpeon-raccoon
-          - emoji-volpeon-vlpn
-          - emoji-caro
-          - lotte-art
-          - drone-runner-docker
-          - alco-sans
-          - constructium
-          - fairfax
-          - fairfax-hd
-          - kreative-square
-          - nasin-nanpa
-          - matrix-media-repo
-          - mautrix-discord
-          - mautrix-whatsapp
-          - mautrix-signal
-          - mautrix-telegram
-          - python-mautrix
-          - python-tulir-telethon
-          - papermc
-          - python-plover-stroke
-          - python-rtf-tokenize
-          - plover
-          - plover-plugins-manager
-          - python-simplefuzzyset
-          - plover-plugin-emoji
-          - plover-plugin-tapey-tape
-          - plover-plugin-yaml-dictionary
-          - plover-plugin-machine-hid
-          - plover-plugin-rkb1-hid
-          - plover-dict-didoesdigital
-          - miifox-net
-          - old-homepage
-          - clean-s3-cache
-          - python-instagram
-          - moa
-          - nix-s3-dedup
-          - attic
-          - attic-client
-          - attic-server
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
-      - name: Install Nix
-        uses: cachix/install-nix-action@v18
-        with:
-          nix_path: nixpkgs=channel:nixos-unstable
-          extra_nix_config: |
-            access-tokens = github.com=${{ secrets.GITHUB_TOKEN }}
-            trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= nixcache:8KKuGz95Pk4UJ5W/Ni+pN+v+LDTkMMFV4yrGmAYgkDg= hydra.nixos.org-1:CNHJZBh9K4tP3EKF6FkkgeVYsS3ohTl+oS0Qa8bezVs= cache:6tx18bfuH66LOfrn37EmN2YxwNZI3qNk3lKHoz/XlXI=
-            experimental-features = nix-command flakes ca-derivations
-            post-build-hook = ${{ github.workspace }}/scripts/post-build-hook
-            substituters = https://cache.chir.rs/ https://cache.nixos.org/ https://attic.chir.rs/cache
-      - name: Download patched nix
-        run: nix build github:DarkKirb/nix-packages#nix-s3-dedup
-      - name: Download attic
-        run: nix build github:DarkKirb/nix-packages#attic-client
-      - name: Set up secrets
-        run: |
-          echo "$NIX_CACHE_KEY" > ~/cache.key
-          sudo mkdir /root/.aws
-          echo "$AWS_CREDENTIALS" | sudo tee /root/.aws/credentials > /dev/null
-          nix run github:DarkKirb/nix-packages#attic-client -- login attic-server https://attic.chir.rs/ "$ATTIC_TOKEN"
-        env:
-          NIX_CACHE_KEY: ${{secrets.NIX_CACHE_KEY}}
-          AWS_CREDENTIALS: ${{secrets.AWS_CREDENTIALS}}
-          ATTIC_TOKEN: ${{secrets.ATTIC_TOKEN}}
-      - name: install jq
-        run: sudo apt install jq
-      - name: Build ${{ matrix.package }}
-        run: |
-          DRV_PATH=$(nix-instantiate -E '(import ./. {}).${{ matrix.package }}')
-          OUT_PATH=$(nix show-derivation $DRV_PATH | jq -r ".\"$DRV_PATH\".outputs.out.path")
-          # check if we already built it
-          NARINFO=$(echo $OUT_PATH | sed 's|/| |g' | sed 's|-| |g' | awk '{print $3}')
-          if curl --output /dev/null --silent --fail "https://cache.chir.rs/$NARINFO.narinfo"; then
-            echo "${{ matrix.package }} already built. Skipping"
-          else
-            sudo /nix/var/nix/profiles/default/bin/nix run 'github:DarkKirb/nix-packages#attic-client' -- push cache $(nix-store -r $DRV_PATH)
-          fi
-        env:
-          NIXPKGS_ALLOW_UNFREE: 1

.github/workflows/update.yml

@@ -15,12 +15,12 @@ jobs:
           nix_path: nixpkgs=channel:nixos-unstable
           extra_nix_config: |
             access-tokens = github.com=${{ secrets.GITHUB_TOKEN }}
-            trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= nixcache:8KKuGz95Pk4UJ5W/Ni+pN+v+LDTkMMFV4yrGmAYgkDg= hydra.nixos.org-1:CNHJZBh9K4tP3EKF6FkkgeVYsS3ohTl+oS0Qa8bezVs=
+            trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= nixcache:8KKuGz95Pk4UJ5W/Ni+pN+v+LDTkMMFV4yrGmAYgkDg= hydra.nixos.org-1:CNHJZBh9K4tP3EKF6FkkgeVYsS3ohTl+oS0Qa8bezVs= chir-rs:AnwyFacopHSkprD6aXY4/R3J9JYzTbV2rosJCBPaB28=
             experimental-features = nix-command flakes ca-derivations
             post-build-hook = ${{ github.workspace }}/scripts/post-build-hook
-            substituters = https://cache.chir.rs/ https://cache.nixos.org/
-      - name: Download patched nix
-        run: nix build github:DarkKirb/nix-packages#nix-s3-dedup
+            substituters = https://attic.chir.rs/chir-rs/ https://cache.nixos.org/
+      - name: Download attic
+        run: nix build github:DarkKirb/nix-packages#attic-client
       - name: Set up secrets
         run: |
           echo "$NIX_CACHE_KEY" > ~/cache.key

default.nix

@@ -63,6 +63,5 @@ in {
   clean-s3-cache = pkgs.python3Packages.callPackage ./scripts/clean-s3-cache.nix {};
   python-instagram = pkgs.python3Packages.callPackage ./python/instagram.nix {};
   moa = pkgs.python3Packages.callPackage ./moa {};
-  nix-s3-dedup = pkgs.callPackage ./nix {};
   inherit (attic.defaultNix.packages.${pkgs.system}) attic attic-client attic-server;
 }

hydra/default.nix

@@ -0,0 +1,16 @@
+{
+  nix-packages,
+  nixpkgs,
+  system,
+}: let
+  pkgs = import nixpkgs {
+    inherit system;
+    config = {
+      allowUnfree = true;
+      allowUnsupportedSystem = true;
+    };
+  };
+  ci = import nix-packages {inherit pkgs;};
+  isReserved = n: n == "lib" || n == "overlays" || n == "modules";
+in
+  pkgs.lib.filterAttrs (name: _: !(isReserved name)) ci

hydra/jobsets.nix

@@ -0,0 +1,78 @@
+{
+  prsJSON,
+  nixpkgs,
+}: let
+  pkgs = import nixpkgs {};
+  prs = builtins.fromJSON (builtins.readFile prsJSON);
+  systems = ["x86_64-linux" "aarch64-linux" "riscv64-linux"];
+  nixpkgs_version = ["master" "nixos-22.05"];
+  mkJobsets = system: version:
+    (pkgs.lib.mapAttrs' (
+        num: info: {
+          name = "${system}-${version}-pr${num}";
+          value = {
+            enabled = 1;
+            hidden = false;
+            description = "PR ${num} (${system}-${version}): ${info.title}";
+            nixexprinput = "nix-packages";
+            nixexprpath = "hydra/default.nix";
+            checkinterval = 300;
+            schedulingshares = 100;
+            enableemail = false;
+            emailoverride = "";
+            keepnr = 1;
+            inputs = {
+              nix-packages = {
+                type = "git";
+                value = "https://github.com/${info.head.repo.owner.login}/${info.head.repo.name}.git ${info.head.ref}";
+                emailresponsible = false;
+              };
+              nixpkgs = {
+                type = "git";
+                value = "https://github.com/NixOS/nixpkgs.git ${version}";
+                emailresponsible = false;
+              };
+              system = {
+                type = "string";
+                value = system;
+              };
+            };
+          };
+        }
+      )
+      prs)
+    // {
+      "${system}-${version}" = {
+        enabled = 1;
+        hidden = false;
+        description = "nix-packages ${system}-${version}";
+        nixexprinput = "nix-packages";
+        nixexprpath = "hydra/default.nix";
+        checkinterval = 300;
+        schedulingshares = 100;
+        enableemail = false;
+        emailoverride = "";
+        keepnr = 1;
+        inputs = {
+          nix-packages = {
+            type = "git";
+            value = "https://github.com/DarkKirb/nix-packages.git main";
+            emailresponsible = false;
+          };
+          nixpkgs = {
+            type = "git";
+            value = "https://github.com/NixOS/nixpkgs.git ${version}";
+            emailresponsible = false;
+          };
+          system = {
+            type = "string";
+            value = system;
+          };
+        };
+      };
+    };
+
+  concatAttrs = pkgs.lib.foldr (a: b: a // b) {};
+
+  jobsets = concatAttrs (pkgs.lib.concatMap (system: map (version: mkJobsets system version) nixpkgs_version) systems);
+in {jobsets = pkgs.writeText "jobsets.json" (builtins.toJSON jobsets);}

hydra/spec.json

@@ -0,0 +1,29 @@
+{
+    "enabled": 1,
+    "hidden": false,
+    "description": "jobsets",
+    "nixexprinput": "nix-packages",
+    "nixexprpath": "hydra/jobsets.nix",
+    "checkinterval": 60,
+    "schedulingshares": 100,
+    "enableemail": false,
+    "emailoverride": "",
+    "keepnr": 1,
+    "inputs": {
+        "nix-packages": {
+            "type": "git",
+            "value": "https://github.com/DarkKirb/nix-packages.git switch-to-hydra",
+            "emailresponsible": false
+        },
+        "nixpkgs": {
+            "type": "git",
+            "value": "https://github.com/NixOS/nixpkgs.git",
+            "emailresponsible": false
+        },
+        "prsJSON": {
+            "type": "githubpulls",
+            "value": "DarkKirb nix-packages",
+            "emailresponsible": false
+        }
+    }
+}

nix/dedupNixCache.patch

@@ -1,54 +0,0 @@
-diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
-index 844553ad3..a7a9cd702 100644
---- a/src/libstore/s3-binary-cache-store.cc
-+++ b/src/libstore/s3-binary-cache-store.cc
-@@ -214,6 +214,14 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual
-     Stats stats;
- 
-     S3Helper s3Helper;
-+    ref<Store> mNixosCache;
-+
-+    BinaryCacheStore * nixosCache() {
-+        auto * casted = dynamic_cast<BinaryCacheStore *>(&*this->mNixosCache);
-+        if (!casted)
-+            throw UsageError("cache.nixos.org is not a BinaryCacheStore");
-+        return casted;
-+    }
- 
-     S3BinaryCacheStoreImpl(
-         const std::string & uriScheme,
-@@ -227,6 +235,7 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual
-         , S3BinaryCacheStore(params)
-         , bucketName(bucketName)
-         , s3Helper(profile, region, scheme, endpoint)
-+        , mNixosCache(openStore("https://cache.nixos.org/"))
-     {
-         diskCache = getNarInfoDiskCache();
-     }
-@@ -268,6 +277,8 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual
- 
-     bool fileExists(const std::string & path) override
-     {
-+        if(this->nixosCache()->fileExists(path))
-+            return true;
-         stats.head++;
- 
-         auto res = s3Helper.client->HeadObject(
-@@ -389,6 +400,8 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual
-         std::shared_ptr<std::basic_iostream<char>> istream,
-         const std::string & mimeType) override
-     {
-+        if(this->nixosCache()->fileExists(path))
-+            return;
-         auto compress = [&](std::string compression)
-         {
-             auto compressed = nix::compress(compression, StreamToSourceAdapter(istream).drain());
-@@ -407,6 +420,8 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual
- 
-     void getFile(const std::string & path, Sink & sink) override
-     {
-+        if(this->nixosCache()->fileExists(path))
-+            return this->nixosCache()->getFile(path, sink);
-         stats.get++;
- 
-         // FIXME: stream output to sink.

nix/default.nix

@@ -1,8 +0,0 @@
-{nix}:
-nix.overrideAttrs (super: {
-  patches =
-    (super.patches or [])
-    ++ [
-      ./dedupNixCache.patch
-    ];
-})

scripts/post-build-hook

@@ -1,6 +1,4 @@
 #!/bin/sh
 set -euf
 export IFS=' '
-/nix/var/nix/profiles/default/bin/nix store sign --key-file /home/runner/cache.key $DRV_PATH
-/nix/var/nix/profiles/default/bin/nix run 'github:DarkKirb/nix-packages#nix-s3-dedup' -- copy --to 's3://cache-chir-rs?scheme=https&endpoint=s3.us-west-000.backblazeb2.com&secret-key=/home/runner/cache.key&multipart-upload=true&compression=zstd&compression-level=15' $DRV_PATH
 /nix/var/nix/profiles/default/bin/nix run 'github:DarkKirb/nix-packages#attic-client' -- push cache $OUT_PATHS