From 1f8558b9c07077c3f175bfd0f52bff431cb38fd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Charlotte=20=F0=9F=A6=9D=20Delenk?= Date: Fri, 18 Nov 2022 15:21:01 +0100 Subject: [PATCH] remove s3 cache items that are already on cache.nixos.org --- .github/workflows/build.yml | 3 +- .github/workflows/update.yml | 3 +- scripts/clean-s3-cache.nix | 3 +- scripts/clean-s3-cache.py | 61 ++++++++++++++++++++++-------------- 4 files changed, 41 insertions(+), 29 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0aaf09d..229c4af 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -65,11 +65,10 @@ jobs: access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= nixcache:8KKuGz95Pk4UJ5W/Ni+pN+v+LDTkMMFV4yrGmAYgkDg= hydra.nixos.org-1:CNHJZBh9K4tP3EKF6FkkgeVYsS3ohTl+oS0Qa8bezVs= experimental-features = nix-command flakes ca-derivations + post-build-hook = ${{ github.workspace }}/scripts/post-build-hook substituters = https://cache.chir.rs/ - name: Download patched nix run: nix build github:DarkKirb/nix-packages#nix-s3-dedup - - name: Setup post-build-hook - run: echo "post-build-hook = ${{ github.workspace }}/scripts/post-build-hook" | sudo tee -a /etc/nix/nix.conf - name: Set up secrets run: | echo "$NIX_CACHE_KEY" > ~/cache.key diff --git a/.github/workflows/update.yml b/.github/workflows/update.yml index 47aeafc..494a646 100644 --- a/.github/workflows/update.yml +++ b/.github/workflows/update.yml @@ -17,11 +17,10 @@ jobs: access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= nixcache:8KKuGz95Pk4UJ5W/Ni+pN+v+LDTkMMFV4yrGmAYgkDg= hydra.nixos.org-1:CNHJZBh9K4tP3EKF6FkkgeVYsS3ohTl+oS0Qa8bezVs= experimental-features = nix-command flakes ca-derivations + post-build-hook = ${{ github.workspace }}/scripts/post-build-hook substituters = https://cache.chir.rs/ - name: Download patched nix run: nix build github:DarkKirb/nix-packages#nix-s3-dedup - - name: Setup post-build-hook - run: echo "post-build-hook = ${{ github.workspace }}/scripts/post-build-hook" | sudo tee -a /etc/nix/nix.conf - name: Set up secrets run: | echo "$NIX_CACHE_KEY" > ~/cache.key diff --git a/scripts/clean-s3-cache.nix b/scripts/clean-s3-cache.nix index 90a5682..28f6e68 100644 --- a/scripts/clean-s3-cache.nix +++ b/scripts/clean-s3-cache.nix @@ -1,11 +1,12 @@ { python3, boto3, + aiohttp, stdenvNoCC, lib, }: let clean-s3-cache-env = python3.buildEnv.override { - extraLibs = [boto3]; + extraLibs = [boto3 aiohttp]; }; in stdenvNoCC.mkDerivation { diff --git a/scripts/clean-s3-cache.py b/scripts/clean-s3-cache.py index 6a23d76..f4c4a02 100644 --- a/scripts/clean-s3-cache.py +++ b/scripts/clean-s3-cache.py @@ -10,6 +10,7 @@ import json import boto3 from botocore.response import StreamingBody +import aiohttp ENDPOINT_URL: str = "https://s3.us-west-000.backblazeb2.com" BUCKET_NAME: str = "cache-chir-rs" @@ -143,33 +144,45 @@ def get_store_hashes() -> set[str]: return hashes +async def is_in_nixos_cache(client: aiohttp.ClientSession, narinfo: str) -> bool: + async with client.get(f"https://cache.nixos.org/{narinfo}"): + if response.status == 200: + return True + return False + async def main() -> None: nars_to_delete = set() nars_to_keep = set() - async for obj_key in list_cache_objects(): - if obj_key.endswith(".narinfo"): - # check if we have the hash locally - narinfo = await get_object(obj_key) - narinfo = NarInfo(narinfo) - if not await narinfo.exists_locally(): - print(f"Found unused NAR for {narinfo.store_path}") - await delete_object(obj_key) - nars_to_delete.add(narinfo.url) - else: - nars_to_keep.add(narinfo.url) - if obj_key.startswith("realisations/"): - realisation = await get_object(obj_key) - realisation = json.loads(realisation) - if not isinstance(realisation, dict): - continue - if "outPath" not in realisation: - continue - if not await exists_locally("/nix/store/" + - realisation["outPath"]): - print(f"Found unused realisation for {realisation['outPath']}") - await delete_object(obj_key) - if obj_key.startswith("nar/"): - nars_to_delete.add(obj_key) + async with aiohttp.ClientSession() as client: + async for obj_key in list_cache_objects(): + if obj_key.endswith(".narinfo"): + # check if we have the hash locally + narinfo = await get_object(obj_key) + narinfo = NarInfo(narinfo) + # check if cache.nixos.org has the narinfo + if await is_in_nixos_cache(client, obj_key): + print(f"Found duplicated NAR for {narinfo.store_path}") + await delete_object(obj_key) + nars_to_delete.add(narinfo.url) + elif not await narinfo.exists_locally(): + print(f"Found unused NAR for {narinfo.store_path}") + await delete_object(obj_key) + nars_to_delete.add(narinfo.url) + else: + nars_to_keep.add(narinfo.url) + if obj_key.startswith("realisations/"): + realisation = await get_object(obj_key) + realisation = json.loads(realisation) + if not isinstance(realisation, dict): + continue + if "outPath" not in realisation: + continue + if not await exists_locally("/nix/store/" + + realisation["outPath"]): + print(f"Found unused realisation for {realisation['outPath']}") + await delete_object(obj_key) + if obj_key.startswith("nar/"): + nars_to_delete.add(obj_key) for nar in nars_to_delete: if nar in nars_to_keep: continue