remove s3 cache items that are already on cache.nixos.org

.github/workflows/build.yml

@@ -65,11 +65,10 @@ jobs:
             access-tokens = github.com=${{ secrets.GITHUB_TOKEN }}
             trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= nixcache:8KKuGz95Pk4UJ5W/Ni+pN+v+LDTkMMFV4yrGmAYgkDg= hydra.nixos.org-1:CNHJZBh9K4tP3EKF6FkkgeVYsS3ohTl+oS0Qa8bezVs=
             experimental-features = nix-command flakes ca-derivations
+            post-build-hook = ${{ github.workspace }}/scripts/post-build-hook
             substituters = https://cache.chir.rs/
       - name: Download patched nix
         run: nix build github:DarkKirb/nix-packages#nix-s3-dedup
-      - name: Setup post-build-hook
-        run: echo "post-build-hook = ${{ github.workspace }}/scripts/post-build-hook" | sudo tee -a /etc/nix/nix.conf
       - name: Set up secrets
         run: |
           echo "$NIX_CACHE_KEY" > ~/cache.key

.github/workflows/update.yml

@@ -17,11 +17,10 @@ jobs:
             access-tokens = github.com=${{ secrets.GITHUB_TOKEN }}
             trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= nixcache:8KKuGz95Pk4UJ5W/Ni+pN+v+LDTkMMFV4yrGmAYgkDg= hydra.nixos.org-1:CNHJZBh9K4tP3EKF6FkkgeVYsS3ohTl+oS0Qa8bezVs=
             experimental-features = nix-command flakes ca-derivations
+            post-build-hook = ${{ github.workspace }}/scripts/post-build-hook
             substituters = https://cache.chir.rs/
       - name: Download patched nix
         run: nix build github:DarkKirb/nix-packages#nix-s3-dedup
-      - name: Setup post-build-hook
-        run: echo "post-build-hook = ${{ github.workspace }}/scripts/post-build-hook" | sudo tee -a /etc/nix/nix.conf
       - name: Set up secrets
         run: |
           echo "$NIX_CACHE_KEY" > ~/cache.key

scripts/clean-s3-cache.nix

@@ -1,11 +1,12 @@
 {
   python3,
   boto3,
+  aiohttp,
   stdenvNoCC,
   lib,
 }: let
   clean-s3-cache-env = python3.buildEnv.override {
-    extraLibs = [boto3];
+    extraLibs = [boto3 aiohttp];
   };
 in
   stdenvNoCC.mkDerivation {

scripts/clean-s3-cache.py

@@ -10,6 +10,7 @@ import json
 
 import boto3
 from botocore.response import StreamingBody
+import aiohttp
 
 ENDPOINT_URL: str = "https://s3.us-west-000.backblazeb2.com"
 BUCKET_NAME: str = "cache-chir-rs"
@@ -143,33 +144,45 @@ def get_store_hashes() -> set[str]:
     return hashes
 
 
+async def is_in_nixos_cache(client: aiohttp.ClientSession, narinfo: str) -> bool:
+    async with client.get(f"https://cache.nixos.org/{narinfo}"):
+        if response.status == 200:
+            return True
+    return False
+
 async def main() -> None:
     nars_to_delete = set()
     nars_to_keep = set()
-    async for obj_key in list_cache_objects():
+    async with aiohttp.ClientSession() as client:
-        if obj_key.endswith(".narinfo"):
+        async for obj_key in list_cache_objects():
-            # check if we have the hash locally
+            if obj_key.endswith(".narinfo"):
-            narinfo = await get_object(obj_key)
+                # check if we have the hash locally
-            narinfo = NarInfo(narinfo)
+                narinfo = await get_object(obj_key)
-            if not await narinfo.exists_locally():
+                narinfo = NarInfo(narinfo)
-                print(f"Found unused NAR for {narinfo.store_path}")
+                # check if cache.nixos.org has the narinfo
-                await delete_object(obj_key)
+                if await is_in_nixos_cache(client, obj_key):
-                nars_to_delete.add(narinfo.url)
+                    print(f"Found duplicated NAR for {narinfo.store_path}")
-            else:
+                    await delete_object(obj_key)
-                nars_to_keep.add(narinfo.url)
+                    nars_to_delete.add(narinfo.url)
-        if obj_key.startswith("realisations/"):
+                elif not await narinfo.exists_locally():
-            realisation = await get_object(obj_key)
+                    print(f"Found unused NAR for {narinfo.store_path}")
-            realisation = json.loads(realisation)
+                    await delete_object(obj_key)
-            if not isinstance(realisation, dict):
+                    nars_to_delete.add(narinfo.url)
-                continue
+                else:
-            if "outPath" not in realisation:
+                    nars_to_keep.add(narinfo.url)
-                continue
+            if obj_key.startswith("realisations/"):
-            if not await exists_locally("/nix/store/" +
+                realisation = await get_object(obj_key)
-                                        realisation["outPath"]):
+                realisation = json.loads(realisation)
-                print(f"Found unused realisation for {realisation['outPath']}")
+                if not isinstance(realisation, dict):
-                await delete_object(obj_key)
+                    continue
-        if obj_key.startswith("nar/"):
+                if "outPath" not in realisation:
-            nars_to_delete.add(obj_key)
+                    continue
+                if not await exists_locally("/nix/store/" +
+                                            realisation["outPath"]):
+                    print(f"Found unused realisation for {realisation['outPath']}")
+                    await delete_object(obj_key)
+            if obj_key.startswith("nar/"):
+                nars_to_delete.add(obj_key)
     for nar in nars_to_delete:
         if nar in nars_to_keep:
             continue