darkkirb/nixos-config
1
0
Fork 0
Code Issues 1 Pull requests 2 Projects Releases Packages Wiki Activity Actions

Handle 16 files in parallel

Browse source
This commit is contained in:
Charlotte 🦝 Delenk 2022-04-23 18:15:51 +01:00
parent ff475d3f12
commit dafd6af9c8
Signed by: darkkirb
GPG key ID: AB2BD8DAF2E37122
1 changed files with 26 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

46
packages/clean-s3-cache.py
View file

@ -140,27 +140,33 @@ def get_store_hashes() -> set[str]:
async def main() -> None:
store_hashes = get_store_hashes()
sem = asyncio.Semaphore(16)
async for obj_key in list_old_cache_objects():
if obj_key.endswith(".narinfo"):
# check if we have the hash locally
if obj_key.split(".")[0] in store_hashes:
continue # yes, don’t delete
narinfo = await get_object(obj_key)
narinfo = NarInfo(narinfo)
if not await narinfo.exists_locally():
print(f"Found unused NAR for {narinfo.store_path}")
await delete_object(obj_key)
await delete_object(narinfo.url)
if obj_key.startswith("realisations/"):
realisation = await get_object(obj_key)
realisation = json.loads(realisation)
if not isinstance(realisation, dict):
continue
if "outPath" not in realisation:
continue
if not await exists_locally("/nix/store/" + realisation["outPath"]):
print(f"Found unused realisation for {realisation['outPath']}")
await delete_object(obj_key)
asyncio.create_task(handle_obj(obj_key))
async def handle_obj(obj_key: str) -> None:
async with sem:
if obj_key.endswith(".narinfo"):
# check if we have the hash locally
if obj_key.split(".")[0] in store_hashes:
return # yes, don’t delete
narinfo = await get_object(obj_key)
narinfo = NarInfo(narinfo)
if not await narinfo.exists_locally():
print(f"Found unused NAR for {narinfo.store_path}")
await delete_object(obj_key)
await delete_object(narinfo.url)
if obj_key.startswith("realisations/"):
realisation = await get_object(obj_key)
realisation = json.loads(realisation)
if not isinstance(realisation, dict):
return
if "outPath" not in realisation:
return
if not await exists_locally("/nix/store/" + realisation["outPath"]):
print(f"Found unused realisation for {realisation['outPath']}")
await delete_object(obj_key)
if __name__ == "__main__":
asyncio.get_event_loop().run_until_complete(main())