Handle 16 files in parallel

packages/clean-s3-cache.py

@@ -140,11 +140,17 @@ def get_store_hashes() -> set[str]:
 
 async def main() -> None:
     store_hashes = get_store_hashes()
+    sem = asyncio.Semaphore(16)
+
     async for obj_key in list_old_cache_objects():
+        asyncio.create_task(handle_obj(obj_key))
+
+    async def handle_obj(obj_key: str) -> None:
+        async with sem:
             if obj_key.endswith(".narinfo"):
                 # check if we have the hash locally
                 if obj_key.split(".")[0] in store_hashes:
-                continue # yes, don’t delete
+                    return # yes, don’t delete
                 narinfo = await get_object(obj_key)
                 narinfo = NarInfo(narinfo)
                 if not await narinfo.exists_locally():
@@ -155,9 +161,9 @@ async def main() -> None:
                 realisation = await get_object(obj_key)
                 realisation = json.loads(realisation)
                 if not isinstance(realisation, dict):
-                continue
+                    return
                 if "outPath" not in realisation:
-                continue
+                    return
                 if not await exists_locally("/nix/store/" + realisation["outPath"]):
                     print(f"Found unused realisation for {realisation['outPath']}")
                     await delete_object(obj_key)