parent
c51c34b73d
commit
26c91156c7
17 changed files with 286 additions and 243 deletions
2
.github/workflows/update.yaml
vendored
2
.github/workflows/update.yaml
vendored
|
@ -15,7 +15,7 @@ jobs:
|
|||
nix_path: nixpkgs=channel:nixos-unstable
|
||||
extra_nix_config: |
|
||||
access-tokens = github.com=${{ secrets.GITHUB_TOKEN }}
|
||||
substituters = https://cache.nixos.org/ https://cache.chir.rs/ https://hydra.chir.rs/
|
||||
substituters = https://cache.nixos.org/ https://attic.chir.rs/chir-rs/ https://hydra.chir.rs/
|
||||
trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= nixcache:8KKuGz95Pk4UJ5W/Ni+pN+v+LDTkMMFV4yrGmAYgkDg= chir-rs:/iTDNHmQw1HklELHTBAVDFVAFaJ3ACGu3eezVUtplKc=
|
||||
experimental-features = nix-command flakes
|
||||
- name: update flake lock
|
||||
|
|
|
@ -15,6 +15,7 @@
|
|||
./services/named-submissive.nix
|
||||
./services/shitalloverme.nix
|
||||
./users/remote-build.nix
|
||||
./services/atticd.nix
|
||||
./services/minecraft.nix
|
||||
./services/postgres.nix
|
||||
./services/nextcloud.nix
|
||||
|
|
|
@ -104,7 +104,7 @@
|
|||
];
|
||||
hardware.enableRedistributableFirmware = true;
|
||||
nix.settings.substituters = lib.mkForce [
|
||||
"https://cache.chir.rs/"
|
||||
"https://attic.chir.rs/chir-rs/"
|
||||
"https://cache.nixos.org/"
|
||||
"https://beam.attic.rs/riscv"
|
||||
"https://cache.ztier.in"
|
||||
|
|
|
@ -3,6 +3,7 @@
|
|||
lib,
|
||||
config,
|
||||
system,
|
||||
attic,
|
||||
...
|
||||
}: {
|
||||
imports = [
|
||||
|
@ -16,7 +17,8 @@
|
|||
require-sigs = true;
|
||||
builders-use-substitutes = true;
|
||||
substituters = [
|
||||
"https://cache.chir.rs/"
|
||||
"https://attic.chir.rs/chir-rs/"
|
||||
"https://hydra.int.chir.rs"
|
||||
];
|
||||
trusted-public-keys = [
|
||||
"nixcache:8KKuGz95Pk4UJ5W/Ni+pN+v+LDTkMMFV4yrGmAYgkDg="
|
||||
|
|
|
@ -30,6 +30,7 @@
|
|||
./services/rspamd.nix
|
||||
./wireguard/public-server.nix
|
||||
./services/shitalloverme.nix
|
||||
./services/atticd.nix
|
||||
./services/wordpress.nix
|
||||
./services/initrd-ssh.nix
|
||||
./wireguard
|
||||
|
|
62
config/services/atticd.nix
Normal file
62
config/services/atticd.nix
Normal file
|
@ -0,0 +1,62 @@
|
|||
{
|
||||
attic,
|
||||
config,
|
||||
lib,
|
||||
nix-packages,
|
||||
system,
|
||||
pkgs,
|
||||
...
|
||||
}: {
|
||||
imports = [attic.nixosModules.atticd];
|
||||
services.atticd = {
|
||||
enable = true;
|
||||
package = attic.packages.${system}.attic-server;
|
||||
credentialsFile = config.sops.secrets."services/attic".path;
|
||||
settings = {
|
||||
listen = "[::1]:57448";
|
||||
allowed-hosts = ["attic.chir.rs"];
|
||||
api-endpoint = "https://attic.chir.rs/";
|
||||
database = lib.mkForce {};
|
||||
storage = {
|
||||
type = "s3";
|
||||
region = "us-east-1";
|
||||
bucket = "attic-chir-rs";
|
||||
endpoint = "https://ams1.vultrobjects.com/";
|
||||
};
|
||||
compression = {
|
||||
type = "zstd";
|
||||
level = 12;
|
||||
};
|
||||
chunking = {
|
||||
nar-size-threshold = 131072;
|
||||
min-size = 65536;
|
||||
avg-size = 131072;
|
||||
max-size = 262144;
|
||||
};
|
||||
garbage-collection.default-retention-period = "3 months";
|
||||
};
|
||||
};
|
||||
sops.secrets."services/attic" = {};
|
||||
services.postgresql.ensureDatabases = [
|
||||
"attic"
|
||||
];
|
||||
services.postgresql.ensureUsers = [
|
||||
{
|
||||
name = "attic";
|
||||
ensurePermissions = {
|
||||
"DATABASE attic" = "ALL PRIVILEGES";
|
||||
};
|
||||
}
|
||||
];
|
||||
services.caddy.virtualHosts."attic.chir.rs" = {
|
||||
useACMEHost = "chir.rs";
|
||||
logFormat = lib.mkForce "";
|
||||
extraConfig = ''
|
||||
import baseConfig
|
||||
|
||||
reverse_proxy http://[::1]:57448 {
|
||||
trusted_proxies private_ranges
|
||||
}
|
||||
'';
|
||||
};
|
||||
}
|
|
@ -1,5 +1,6 @@
|
|||
{
|
||||
system,
|
||||
attic,
|
||||
lib,
|
||||
config,
|
||||
pkgs,
|
||||
|
@ -76,7 +77,6 @@ in {
|
|||
</prometheus>
|
||||
</hydra_notify>
|
||||
binary_cache_secret_key_file = ${config.sops.secrets."services/hydra/cache-key".path}
|
||||
store_uri = s3://cache-chir-rs?scheme=https&endpoint=ams1.vultrobjects.com&secret-key=${config.sops.secrets."services/hydra/cache-key".path}&multipart-upload=true&compression=zstd&compression-level=15
|
||||
<git-input>
|
||||
timeout = 3600
|
||||
</git-input>
|
||||
|
@ -114,7 +114,7 @@ in {
|
|||
sops.secrets."services/hydra/aws_credentials" = {
|
||||
owner = "hydra-queue-runner";
|
||||
path = "/var/lib/hydra/queue-runner/.aws/credentials";
|
||||
restartUnits = ["hydra-notify.service" "hydra-queue-runner.service"];
|
||||
restartUnits = ["hydra-notify.service"];
|
||||
};
|
||||
systemd.services.update-hydra-hosts = {
|
||||
description = "Update hydra hosts";
|
||||
|
@ -151,25 +151,38 @@ in {
|
|||
chown -Rv hydra-queue-runner /var/lib/hydra/queue-runner
|
||||
ln -svf ${sshConfig} /var/lib/hydra/queue-runner/.ssh/config
|
||||
'';
|
||||
systemd.services.clean-s3-cache = let
|
||||
clean-cache = pkgs.callPackage ../../packages/clean-s3-cache.nix {};
|
||||
in {
|
||||
enable = true;
|
||||
description = "Clean up S3 cache";
|
||||
sops.secrets."attic/config.toml" = {
|
||||
owner = "hydra-queue-runner";
|
||||
key = "attic/config.toml";
|
||||
path = "/var/lib/hydra/queue-runner/.config/attic/config.toml";
|
||||
};
|
||||
|
||||
systemd.services."upload-hydra-results" = {
|
||||
description = "Upload hydra build results";
|
||||
serviceConfig = {
|
||||
ExecStart = "${clean-cache}/bin/clean-s3-cache.py";
|
||||
Type = "oneshot";
|
||||
User = "hydra-queue-runner";
|
||||
Group = "hydra";
|
||||
};
|
||||
script = ''
|
||||
set -ex
|
||||
if [ -e /var/lib/hydra/queue-runner/uploading ]; then
|
||||
cat /var/lib/hydra/queue-runner/uploading | xargs ${attic.packages.${system}.attic-client}/bin/attic push chir-rs
|
||||
rm /var/lib/hydra/queue-runner/uploading
|
||||
fi
|
||||
mv /var/lib/hydra/queue-runner/upload-queue /var/lib/hydra/queue-runner/uploading
|
||||
cat /var/lib/hydra/queue-runner/uploading | xargs ${attic.packages.${system}.attic-client}/bin/attic push chir-rs
|
||||
rm /var/lib/hydra/queue-runner/uploading
|
||||
'';
|
||||
};
|
||||
systemd.timers.clean-s3-cache = {
|
||||
systemd.timers.upload-hydra-results = {
|
||||
enable = true;
|
||||
description = "Clean up S3 cache";
|
||||
requires = ["clean-s3-cache.service"];
|
||||
description = "Upload hydra build results";
|
||||
requires = ["upload-hydra-results.service"];
|
||||
wantedBy = ["multi-user.target"];
|
||||
timerConfig = {
|
||||
OnBootSec = 300;
|
||||
OnUnitActiveSec = 604800;
|
||||
OnUnitActiveSec = 300;
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
|
@ -65,7 +65,7 @@
|
|||
{
|
||||
name = "nextcloud";
|
||||
ensurePermissions = {
|
||||
"DATABASE nextcloud" = "ALL PRIVILEGES";
|
||||
"DATABASE attic" = "ALL PRIVILEGES";
|
||||
};
|
||||
}
|
||||
];
|
||||
|
|
169
flake.lock
169
flake.lock
|
@ -13,11 +13,11 @@
|
|||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1702718849,
|
||||
"narHash": "sha256-aE9m11l8ikQVeQNMKoKb0o1TqnTT5aSd3JrXcxW0Qw8=",
|
||||
"lastModified": 1702888877,
|
||||
"narHash": "sha256-2edEM5EujyPeZDS4eBZaL2bsIzHIprc1OEIxbJshWk0=",
|
||||
"owner": "DarkKirb",
|
||||
"repo": "admin-fe",
|
||||
"rev": "ca204ed8ff111fc6056d0391a7bacf080a79ba89",
|
||||
"rev": "bfaa8bfc76ad092856e67e417ae08a1cd9ab07f8",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
@ -65,11 +65,11 @@
|
|||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1702736102,
|
||||
"narHash": "sha256-oDq4wQ5F+gAVzu8V95qAEL1LZlCGn62aD9eM0Z5BMgg=",
|
||||
"lastModified": 1702888906,
|
||||
"narHash": "sha256-8hugxr70PzMYO0ts2CEQwtZR4atUt4VF21+VSoocWfU=",
|
||||
"owner": "DarkKirb",
|
||||
"repo": "akkoma-fe",
|
||||
"rev": "e925549b73d91abb5c6d37fe478c12e42bce00fd",
|
||||
"rev": "7250d194b64005a9d675afbebf4de936fd1c77ad",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
@ -78,6 +78,100 @@
|
|||
"type": "github"
|
||||
}
|
||||
},
|
||||
"attic": {
|
||||
"inputs": {
|
||||
"cargo2nix": [
|
||||
"cargo2nix"
|
||||
],
|
||||
"crane": [
|
||||
"crane"
|
||||
],
|
||||
"flake-compat": [
|
||||
"flake-compat"
|
||||
],
|
||||
"flake-utils": [
|
||||
"flake-utils"
|
||||
],
|
||||
"nixpkgs": [
|
||||
"nixpkgs"
|
||||
],
|
||||
"rust-overlay": [
|
||||
"rust-overlay"
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1694160842,
|
||||
"narHash": "sha256-KqzSSagAay+qBhXlDGHc05dpio9PZ/ZFVmQcuJum/qU=",
|
||||
"owner": "DarkKirb",
|
||||
"repo": "attic",
|
||||
"rev": "9460d742caf366a1f999936dacd4d6e9274d956b",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "DarkKirb",
|
||||
"repo": "attic",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"cargo2nix": {
|
||||
"inputs": {
|
||||
"flake-compat": [
|
||||
"flake-compat"
|
||||
],
|
||||
"flake-utils": [
|
||||
"flake-utils"
|
||||
],
|
||||
"nixpkgs": [
|
||||
"nixpkgs"
|
||||
],
|
||||
"rust-overlay": [
|
||||
"rust-overlay"
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1691655399,
|
||||
"narHash": "sha256-hVfFMu27OMaUPxpyovnxYNrzDYFCbQaFu+XCAIPeoAk=",
|
||||
"owner": "DarkKirb",
|
||||
"repo": "cargo2nix",
|
||||
"rev": "1a37221e07295f7d5a8842717e94229af72f1c20",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "DarkKirb",
|
||||
"ref": "release-0.11.0",
|
||||
"repo": "cargo2nix",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"crane": {
|
||||
"inputs": {
|
||||
"flake-compat": [
|
||||
"flake-compat"
|
||||
],
|
||||
"flake-utils": [
|
||||
"flake-utils"
|
||||
],
|
||||
"nixpkgs": [
|
||||
"nixpkgs"
|
||||
],
|
||||
"rust-overlay": [
|
||||
"rust-overlay"
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1674934931,
|
||||
"narHash": "sha256-TmGfRDBK7EkR0VY8Jr0WU4WdyzZxiXDGVGUzIXPFXRI=",
|
||||
"owner": "DarkKirb",
|
||||
"repo": "crane",
|
||||
"rev": "42c3f329daa267857c6bc6d21c9eec468e97e2d7",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "DarkKirb",
|
||||
"repo": "crane",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"devshell": {
|
||||
"inputs": {
|
||||
"nixpkgs": [
|
||||
|
@ -139,11 +233,11 @@
|
|||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1702736112,
|
||||
"narHash": "sha256-djXBS0EcT0YIhz7rAbguNuXTHAEDNR9fjP7YbO1RvdQ=",
|
||||
"lastModified": 1702888789,
|
||||
"narHash": "sha256-gZBrtJ552kUoWLzbtAdC10GZ+uJGZbtSiivuK+gdq6Y=",
|
||||
"owner": "DarkKirb",
|
||||
"repo": "element-web",
|
||||
"rev": "018b59b2f82f559fcb34205d87be6670deef09e9",
|
||||
"rev": "af42501884db9d0148dfe386545e63cd5b7ec350",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
@ -171,11 +265,11 @@
|
|||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1702815814,
|
||||
"narHash": "sha256-OOT32xqK6AseTUi252DDVr4DZPTZPYaeg0N4PeaXNEM=",
|
||||
"lastModified": 1702861124,
|
||||
"narHash": "sha256-dfnBanlRxozv2UpCsvLana8L8hKiiNXYmwSO6PaXYBw=",
|
||||
"owner": "nix-community",
|
||||
"repo": "flake-firefox-nightly",
|
||||
"rev": "f8b039b31ccba7dfebd1a5bdd87b7ddf8cb6eb42",
|
||||
"rev": "a37a3879276c630cb8889c73bc53e662113a2d00",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
@ -478,11 +572,11 @@
|
|||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1702776069,
|
||||
"narHash": "sha256-NCza/LTcFVu8YMzoo9XqQwzTaYjqb2Q4DMZxG6vWIX8=",
|
||||
"lastModified": 1702862139,
|
||||
"narHash": "sha256-VI9t3rJku90Y8j6riIWsVdJtaBa7S51sYWsTVNIF6+I=",
|
||||
"owner": "fufexan",
|
||||
"repo": "nix-gaming",
|
||||
"rev": "b5bcd1163fff15a3fc9d97e7d148812736212b89",
|
||||
"rev": "27a7b008400c068955d337097053fa1a7cfc7aec",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
@ -529,11 +623,11 @@
|
|||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1702762359,
|
||||
"narHash": "sha256-L/4BhsfkGH8bHsxPy7ebG5XLeBTZGtfL8vi7maIiAms=",
|
||||
"lastModified": 1702888956,
|
||||
"narHash": "sha256-ABSYVklT6QV5QxlFouhmp1SBhYy0HmdNHfVP/OgsUXI=",
|
||||
"owner": "DarkKirb",
|
||||
"repo": "nix-packages",
|
||||
"rev": "0db848e4e3d9a79ccfabb430241a65c5fd6c7b53",
|
||||
"rev": "5d78521720715c3c6a63c977778db35ea2afb625",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
@ -615,11 +709,11 @@
|
|||
},
|
||||
"nixpkgs_2": {
|
||||
"locked": {
|
||||
"lastModified": 1702836097,
|
||||
"narHash": "sha256-OUvcyojtVzpZ+tt9MR+pkqcs2N+3t9SSEdRaLOblGY4=",
|
||||
"lastModified": 1702889381,
|
||||
"narHash": "sha256-D9Pj7cm5h+7mWLdmjbxwvQtQtfwY0+6ndFcFOzE+Z2U=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "94dbd82f1932dea59ed30b5f28373f837ab53c90",
|
||||
"rev": "b4f9bed389be23eaa123b39bb7c19222db7165aa",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
@ -630,11 +724,11 @@
|
|||
},
|
||||
"nur": {
|
||||
"locked": {
|
||||
"lastModified": 1702835092,
|
||||
"narHash": "sha256-NtgOSBp69TnfMzFKa/wBNsEdR9ubxuWvgH+KuwPxcNY=",
|
||||
"lastModified": 1702889123,
|
||||
"narHash": "sha256-hgdt5ZE76rBbOXVgaBfTY5pT8VezeAeeYomyofrs9RY=",
|
||||
"owner": "nix-community",
|
||||
"repo": "NUR",
|
||||
"rev": "a64c16086ebd52548ed4d132efc8dcb14a5270ad",
|
||||
"rev": "5eb36fd2d32f43177896e8dd5a7ba134d3d5e949",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
@ -648,6 +742,9 @@
|
|||
"admin-fe": "admin-fe",
|
||||
"akkoma": "akkoma",
|
||||
"akkoma-fe": "akkoma-fe",
|
||||
"attic": "attic",
|
||||
"cargo2nix": "cargo2nix",
|
||||
"crane": "crane",
|
||||
"devshell": "devshell",
|
||||
"dns": "dns",
|
||||
"element-web": "element-web",
|
||||
|
@ -666,10 +763,34 @@
|
|||
"nixos-vscode-server": "nixos-vscode-server",
|
||||
"nixpkgs": "nixpkgs_2",
|
||||
"nur": "nur",
|
||||
"rust-overlay": "rust-overlay",
|
||||
"sops-nix": "sops-nix",
|
||||
"systems": "systems"
|
||||
}
|
||||
},
|
||||
"rust-overlay": {
|
||||
"inputs": {
|
||||
"flake-utils": [
|
||||
"flake-utils"
|
||||
],
|
||||
"nixpkgs": [
|
||||
"nixpkgs"
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1702865809,
|
||||
"narHash": "sha256-K7caQe+KqjqTBFmJawmBjmm25S6bza5CXhAqbXFLyH8=",
|
||||
"owner": "oxalica",
|
||||
"repo": "rust-overlay",
|
||||
"rev": "b2aafcee4a8842cecfc877ff7dd271f333dc0fa8",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "oxalica",
|
||||
"repo": "rust-overlay",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"sops-nix": {
|
||||
"inputs": {
|
||||
"nixpkgs": [
|
||||
|
|
28
flake.nix
28
flake.nix
|
@ -22,6 +22,29 @@ rec {
|
|||
inputs.flake-parts.follows = "flake-parts";
|
||||
inputs.nixpkgs.follows = "nixpkgs";
|
||||
};
|
||||
attic = {
|
||||
url = "github:DarkKirb/attic";
|
||||
inputs.cargo2nix.follows = "cargo2nix";
|
||||
inputs.crane.follows = "crane";
|
||||
inputs.flake-compat.follows = "flake-compat";
|
||||
inputs.flake-utils.follows = "flake-utils";
|
||||
inputs.nixpkgs.follows = "nixpkgs";
|
||||
inputs.rust-overlay.follows = "rust-overlay";
|
||||
};
|
||||
cargo2nix = {
|
||||
url = "github:DarkKirb/cargo2nix/release-0.11.0";
|
||||
inputs.flake-compat.follows = "flake-compat";
|
||||
inputs.flake-utils.follows = "flake-utils";
|
||||
inputs.nixpkgs.follows = "nixpkgs";
|
||||
inputs.rust-overlay.follows = "rust-overlay";
|
||||
};
|
||||
crane = {
|
||||
url = "github:DarkKirb/crane";
|
||||
inputs.flake-compat.follows = "flake-compat";
|
||||
inputs.flake-utils.follows = "flake-utils";
|
||||
inputs.nixpkgs.follows = "nixpkgs";
|
||||
inputs.rust-overlay.follows = "rust-overlay";
|
||||
};
|
||||
devshell = {
|
||||
url = "github:numtide/devshell";
|
||||
inputs.nixpkgs.follows = "nixpkgs";
|
||||
|
@ -96,6 +119,11 @@ rec {
|
|||
};
|
||||
nixpkgs.url = "github:NixOS/nixpkgs";
|
||||
nur.url = "github:nix-community/NUR";
|
||||
rust-overlay = {
|
||||
url = "github:oxalica/rust-overlay";
|
||||
inputs.flake-utils.follows = "flake-utils";
|
||||
inputs.nixpkgs.follows = "nixpkgs";
|
||||
};
|
||||
sops-nix = {
|
||||
url = "github:Mic92/sops-nix";
|
||||
inputs.nixpkgs.follows = "nixpkgs";
|
||||
|
|
|
@ -9,6 +9,7 @@ args: self: prev: let
|
|||
in {
|
||||
pandoc = self.writeScriptBin "pandoc" "true";
|
||||
inherit (pkgsX86) nix;
|
||||
inherit (args.attic.packages.x86_64-linux) attic-client;
|
||||
bind = prev.bind.overrideAttrs (_: {
|
||||
doCheck = false;
|
||||
doInstallCheck = false;
|
||||
|
|
|
@ -1,17 +0,0 @@
|
|||
{
|
||||
writeTextFile,
|
||||
python3,
|
||||
python3Packages,
|
||||
}: let
|
||||
environment = python3.buildEnv.override {
|
||||
extraLibs = with python3Packages; [
|
||||
boto3
|
||||
];
|
||||
};
|
||||
in
|
||||
writeTextFile {
|
||||
name = "clean-s3-cache.py";
|
||||
executable = true;
|
||||
destination = "/bin/clean-s3-cache.py";
|
||||
text = builtins.replaceStrings ["#SHEBANG#"] ["${environment}/bin/python"] (builtins.readFile ./clean-s3-cache.py);
|
||||
}
|
|
@ -1,180 +0,0 @@
|
|||
#!#SHEBANG#
|
||||
import asyncio
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import functools
|
||||
from typing import Any, AsyncIterable, Awaitable, Callable, Optional, TypeVar, cast
|
||||
from os import path, listdir
|
||||
import json
|
||||
|
||||
import boto3
|
||||
from botocore.response import StreamingBody
|
||||
|
||||
ENDPOINT_URL: str = "https://ams1.vultrobjects.comk"
|
||||
BUCKET_NAME: str = "cache-chir-rs"
|
||||
|
||||
executor: ThreadPoolExecutor = ThreadPoolExecutor()
|
||||
|
||||
F = TypeVar('F', bound=Callable[..., Any])
|
||||
T = TypeVar('T')
|
||||
|
||||
|
||||
def with_backoff(
|
||||
f: Callable[..., Awaitable[T]]) -> Callable[..., Awaitable[T]]:
|
||||
|
||||
async def with_backoff_wrapper(*args: Any, **kwargs: Any) -> T:
|
||||
last_delay = 2
|
||||
while True:
|
||||
try:
|
||||
return await f(*args, **kwargs)
|
||||
except Exception as e:
|
||||
print(f"{e}")
|
||||
if last_delay >= 120:
|
||||
raise
|
||||
await asyncio.sleep(last_delay)
|
||||
last_delay *= last_delay
|
||||
|
||||
return with_backoff_wrapper
|
||||
|
||||
|
||||
def aio(f: Callable[..., T]) -> Callable[..., Awaitable[T]]:
|
||||
|
||||
async def aio_wrapper(*args: Any, **kwargs: Any) -> T:
|
||||
f_bound: Callable[[], T] = functools.partial(f, *args, **kwargs)
|
||||
loop: asyncio.AbstractEventLoop = asyncio.get_running_loop()
|
||||
return await loop.run_in_executor(executor, f_bound)
|
||||
|
||||
return aio_wrapper
|
||||
|
||||
|
||||
@aio
|
||||
def exists_locally(store_path: str) -> bool:
|
||||
return path.exists(store_path)
|
||||
|
||||
|
||||
class NarInfo(object):
|
||||
|
||||
def __init__(self, narinfo: str) -> None:
|
||||
self.compression = "bzip2"
|
||||
for narinfo_line in narinfo.splitlines():
|
||||
key, value = narinfo_line.split(": ", 1)
|
||||
if key == "StorePath":
|
||||
self.store_path = value
|
||||
elif key == "URL":
|
||||
self.url = value
|
||||
elif key == "Compression":
|
||||
self.compression = value
|
||||
elif key == "FileHash":
|
||||
self.file_hash = value
|
||||
elif key == "FileSize":
|
||||
self.file_size = int(value)
|
||||
elif key == "NarHash":
|
||||
self.nar_hash = value
|
||||
elif key == "NarSize":
|
||||
self.nar_size = int(value)
|
||||
elif key == "References":
|
||||
self.references = value.split()
|
||||
elif key == "Deriver":
|
||||
self.deriver = value
|
||||
elif key == "System":
|
||||
self.system = value
|
||||
elif key == "Sig":
|
||||
self.sig = value
|
||||
elif key == "CA":
|
||||
self.ca = value
|
||||
|
||||
async def exists_locally(self) -> bool:
|
||||
return await exists_locally(self.store_path)
|
||||
|
||||
|
||||
s3 = boto3.client("s3", endpoint_url=ENDPOINT_URL)
|
||||
|
||||
|
||||
@with_backoff
|
||||
@aio
|
||||
def get_object(Key: str) -> str:
|
||||
obj = s3.get_object(Bucket=BUCKET_NAME, Key=Key)
|
||||
if "Body" not in obj:
|
||||
raise Exception("No Body")
|
||||
if isinstance(obj["Body"], StreamingBody):
|
||||
return obj["Body"].read().decode("utf-8")
|
||||
raise Exception("Not StreamingBody")
|
||||
|
||||
|
||||
async def list_cache_objects() -> AsyncIterable[str]:
|
||||
|
||||
@with_backoff
|
||||
@aio
|
||||
def list_objects_v2(ContinuationToken: Optional[str]) -> dict[str, Any]:
|
||||
if ContinuationToken != None:
|
||||
return s3.list_objects_v2(Bucket=BUCKET_NAME,
|
||||
ContinuationToken=ContinuationToken)
|
||||
else:
|
||||
return s3.list_objects_v2(Bucket=BUCKET_NAME)
|
||||
|
||||
cont_token = None
|
||||
while True:
|
||||
objs = await list_objects_v2(cont_token)
|
||||
if "Contents" not in objs:
|
||||
raise Exception("No Contents")
|
||||
if isinstance(objs["Contents"], list):
|
||||
for obj in cast(list[Any], objs["Contents"]):
|
||||
if not isinstance(obj, dict):
|
||||
raise Exception("Not dict")
|
||||
obj = cast(dict[str, Any], obj)
|
||||
yield obj["Key"]
|
||||
|
||||
if "NextContinuationToken" not in objs:
|
||||
break
|
||||
cont_token = objs["NextContinuationToken"]
|
||||
|
||||
|
||||
@with_backoff
|
||||
@aio
|
||||
def delete_object(key: str) -> None:
|
||||
s3.delete_object(Bucket=BUCKET_NAME, Key=key)
|
||||
|
||||
|
||||
def get_store_hashes() -> set[str]:
|
||||
hashes = set()
|
||||
for obj in listdir("/nix/store"):
|
||||
hashes.add(obj.split("-")[0])
|
||||
return hashes
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
store_hashes = get_store_hashes()
|
||||
nars_to_delete = set()
|
||||
nars_to_keep = set()
|
||||
async for obj_key in list_cache_objects():
|
||||
if obj_key.endswith(".narinfo"):
|
||||
# check if we have the hash locally
|
||||
narinfo = await get_object(obj_key)
|
||||
narinfo = NarInfo(narinfo)
|
||||
if not await narinfo.exists_locally():
|
||||
print(f"Found unused NAR for {narinfo.store_path}")
|
||||
await delete_object(obj_key)
|
||||
nars_to_delete.add(narinfo.url)
|
||||
else:
|
||||
nars_to_keep.add(narinfo.url)
|
||||
if obj_key.startswith("realisations/"):
|
||||
realisation = await get_object(obj_key)
|
||||
realisation = json.loads(realisation)
|
||||
if not isinstance(realisation, dict):
|
||||
continue
|
||||
if "outPath" not in realisation:
|
||||
continue
|
||||
if not await exists_locally("/nix/store/" +
|
||||
realisation["outPath"]):
|
||||
print(f"Found unused realisation for {realisation['outPath']}")
|
||||
await delete_object(obj_key)
|
||||
if obj_key.startswith("nar/"):
|
||||
nars_to_delete.add(obj_key)
|
||||
for nar in nars_to_delete:
|
||||
if nar in nars_to_keep:
|
||||
continue
|
||||
print(f"Deleting unused NAR {nar}")
|
||||
await delete_object(nar)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.get_event_loop().run_until_complete(main())
|
8
scripts/post-build-hook
Executable file
8
scripts/post-build-hook
Executable file
|
@ -0,0 +1,8 @@
|
|||
#!/usr/bin/env bash
|
||||
set -euf
|
||||
export IFS=' '
|
||||
export XDG_CONFIG_HOME=/home/runner/.config
|
||||
until /nix/var/nix/profiles/default/bin/nix run 'github:DarkKirb/nix-packages#attic-client' -- push chir-rs $OUT_PATHS; do
|
||||
sleep 5
|
||||
echo "Retrying..."
|
||||
done
|
|
@ -8,6 +8,7 @@ security:
|
|||
restic:
|
||||
password: ENC[AES256_GCM,data:80XNExfwBIG3aVNQBc8T2fdN9oA=,iv:JM/HU7vhx28VA9EppxpFc3xRVcAt+kp3JwTuHmFpL78=,tag:pC73+XCsFGTdA+MbTihD7Q==,type:str]
|
||||
services:
|
||||
attic: ENC[AES256_GCM,data:Z/XoViD/G6goXoS72sPRxwWW/biLxEUIKkw/O9Kj6/KMbxXQx8cYvkoG3bv5umE6+jcJjwPrthvOUip6jF3D0XgEPuB9dPmeNZURFxhAGkMkcCpHk6enVavhSeb1gciuCQI6hEn8FTaf8jhZ3MsBUfHeFu/rV7UqZvxf4k1/A9sGe81zR8Ez2ZUkb/nEGpTj6Z8Pg9LKUSK0QDVsPRPOcARtmjz35uiTGe8OWBodM+bVQs+F4PA6jfaqOC14STTpRxR296/Mnpu+/pZJTA4bft7JOY1iBMXTKzaK0Zxjwc6h4t0Lok5ORWPOZWCGBwJIvIB8LE/YkbT+vVMsC/rL+Qda6jwUO6tam8VdJknKi1jbboCnZjSIIqC5yD5AcGElUWaQ5Xn+F64HwkK1IgZpl5v7mXWHdiat1uznXGvESaWsgRlbxQWIrCmCvEIOw7ZF7fDArPmkbCU6AIlMOOUIEvgQiR2VWNW2mxovm9pyqPjOr22xw5i2,iv:VB8CsGDiHrGe+5/wq0bkjzyAMicExA3TtbArFQ/3abY=,tag:3WX1DEVdcOluijd1flkmXQ==,type:str]
|
||||
chir-rs:
|
||||
database-password: ENC[AES256_GCM,data:selw9JIny9dNpuMVUlLg50YC4Tqbm2I+/3EEZsDbO0I=,iv:VKmdFvQJsI0KiZNZVLSZjVLGl85Xp/P47/YrfBeUPh8=,tag:NCjXDA9JosqIYFC3sFiIPg==,type:str]
|
||||
signup-secret: ENC[AES256_GCM,data:PV4Iau9Wkvmixn8Tirvxy/fuPk94PjkovQlKyF2OqwI=,iv:UtqUFewEthfrGW2xo5rtz3lrnD7LXk1C+msPWGe3f30=,tag:9fuV1SIBK+uOoeIG3K5UhQ==,type:str]
|
||||
|
@ -52,8 +53,8 @@ sops:
|
|||
bVJUcDZLWTk3MiszOWp4enRRQmNsajQKF8QJs/Wb0SqnvsQEkRKlS1Ms9xLIdyvZ
|
||||
QCFAPclaOfaTLTiRJWXjDneBkMBduYKkRPiXCR+Bn7i4z8ixLXFmWw==
|
||||
-----END AGE ENCRYPTED FILE-----
|
||||
lastmodified: "2023-11-16T20:46:34Z"
|
||||
mac: ENC[AES256_GCM,data:UGwfSnA2x33Or70d6ZGKDxIsHR+V0nRXJlJ64J4yMNG7zhBXvfDTUp8Ewaj7a3uPh4zKdFwWmqFhTyTq2O3ra05xap1B0EsoR6vqmIUMSRoNFvISD4dFskX71SQQc1BUEEf4bonB3pP2MHClc96Xq07zx5LonVrUAXooMjjeO+M=,iv:fLVm3X9C4/YYLEhdP9QgWeLsejITtc8DLX7c/YK7kcQ=,tag:zgZso2V3ldZYMSFDIlHnAw==,type:str]
|
||||
lastmodified: "2023-12-18T09:01:54Z"
|
||||
mac: ENC[AES256_GCM,data:JfqWKmgcP1MRqfbkwKAut2ekB+sC3Da+PH6NebYiJ1/9lTMRRw2MO15wpifIEpczJb8A5DwahV0YOmwiwHuFOI/93qP6d8k+t6mRqlPJPDAkA8s5vUlzIyFqCF0oo8+OKn4n3SgEnwdgneX5VaqklRAFeVVgMNtAye9ytiZNvT8=,iv:U6Q+JjI42lf4xB3aQ9g3QYRYP0C+DMYQsfEC5Ek3gc8=,tag:K8ZXmRlNZUqyAjto1F+i1w==,type:str]
|
||||
pgp:
|
||||
- created_at: "2022-12-14T15:34:13Z"
|
||||
enc: |
|
||||
|
|
|
@ -11,6 +11,7 @@ services:
|
|||
chir-rs:
|
||||
database-password: ENC[AES256_GCM,data:6c8Ey39Lh/MoCJakEGpNFyueH+RAs//HXPKExrsiXiU=,iv:YmajjfpoaTHlbv5VhCk36jgfDetCKOTMqrmMGzXvitc=,tag:eQq7P92TR0txNk161gUUKA==,type:str]
|
||||
signup-secret: ENC[AES256_GCM,data:rLpC7HdhTSkDNeRau5iOvicDxeHJC9R3aRIVe65xysQ=,iv:Pm/+ZXWJCtN4Bq87hPaXco78C/cwD7cdCJmApDpS6iA=,tag:NL4T35lL+xauva72f8C+EQ==,type:str]
|
||||
attic: ENC[AES256_GCM,data:q75ahgZD0O56SAXxKoUaVoxqbioreMDtxLXK6K6iTzGhZYabthxmm2RSgUIWuV/sSdDPuuld3eILypr0+G+ujtckyPb8Osc8OAAiZEVgyLac09250nM59Ypuaoqm1qrouD1T2kmPwocVqbWLNz+NzWKnOgX8Yi2lwSPZGXMfhi7THOgb+vyTwSE0PYWubCktynqUM/qw2XgThqp4OYg1V3+nMo3pP3wCtjFu3Vb5HURoUIXTewag08OI0ma6jTkJUARX1ihjuBqW+2/EJ6xVCzq4f1fiWtzjmFYrsFwmScP18modQAB/yHLc0RZOdb8ujSqkJYcHx5SayZ2tBQ+A6DimUbaUfdTLqKF4nPECzZPRfSw079t1t7u1iv95yveCCWmMS9Vslb+zvXETVzg8E33d2BdNQymgYnaavlZiC++hosg9WZw8YqBFxJ7rQqcxAA5kX3VgWg8SbCuHyc2dXVUVIwRaPlOC+QETsK6iDB8+gkt/HOLv,iv:K7zh3b9i9qBTCb6UUIwrFZl3K+td0vafUw/R2JAmckE=,tag:a44Skw4jhlkOfkkkSy5ltA==,type:str]
|
||||
rspamd:
|
||||
dkim:
|
||||
darkkirb.de: ENC[AES256_GCM,data:2Af4PsmTI07QzY/++VrrENe8wovHrUNElisfufyNt4RaiAB0Uq9Bp6BnpIEqLzfDs0z4mY5D05ncVa+K6oZWnAkhTBwIwO/iD6a4v5aOSdMp1BjbAQnJ9OD41+lLw1KZe98qAbjoO0yjtr4ySo5UzQUJ2wm2i7ZHtEBH06Gc1ZIPD64u2rC3/x9m65Xf+NFe7uSn0hBlaVTnuLkA0RwcopvtFXe63PXq93cHc+YCu9O6beppp7p0K5EkOW54l0UuGVbNtHKm+AKbsjzLWBnS3w0tP6mi1GkbBB0DMJrasA+cd8SbQ02kh5t8cvisviDaroeWTGElovqNOiSYZ9PkBAaADBYuR6bDJ9riIH8RrFfmS+lT0XW6OPPGdz9b5AI5A+Xze+36u2sv/Mys+Y+/G2C+uZYgDj1EKYX3DGKAl0ytNlSw7QRa1WoobvcIOfgN5cvmVV56QkSNteEdHGGod0aoiWTgGFwuKyjZL4UBjoDXcXxiCAKeQJ9Kkk0u9IG5JEhsa6W21s1Ea3hJdCaNPXk/vF7d3gM/jzrYNiioic4S/PckYf4u/4NNJhP5QS9ys68uPXZpSxZ7cZnB7ZS1uzvY4Hq/nNBBXmSjGz/G8EaeAfcXtBeRwLnv6HgeDfz03lu+JoZJH4UKUkU0emoP8du1RpMU8lMxuOsOirwVAU2SljZn/P2+reeFsV2hca6/AmXNO8Ez663AcQsxww/xY9tady3ch6k/rixLIYZuhURzFilEwqQAye0KvcnQrwhix/i1ujGv5G2SKHTlpYiQPDDwnF/gYHC4H1kqzRCcRP0CcYeZ3hhBC/fwgkXSxhrKjRmPU9hGsB4LfNnzdbezi4K4XGiOzEy6DD6r+KqHK782U95eaBxDPaD5PTY/jcy4mn5jtalmznPiZ3C68EVTcZQJf/o84fQWPG6I9BIy5PITacqzm8EQC93ZPPB+gdLaUWv56U2ruT/3RgC4eusgBtngydbRP7MaVojasX/NWEV4uhQ6zH7ErEFQDpqwaTNBDhXQUS2DJznQH+xk9M+VxInN1tUvc7UXQ1SBdARFLPzNubRvnyhd05bx1TEGHIqKNaJN6khYOt0ES+wZPjYtx3ctdhzNjajMIjv8rjmEmNoRyyH66GMHEfDbuCkv+VoBkPPWhl5QGbF8NRmN6LxR/GokQgCitfDqBAvuCYhRahurVZmGelcXh3X/eV+aFWnC9TC8aA==,iv:LhtgzfLhkBUsZcEF5oBrUQJLeMkdSN97H9rp8fRdG2g=,tag:vTyG6L9n9LwNe94UNkLyhA==,type:str]
|
||||
|
@ -85,8 +86,8 @@ sops:
|
|||
UDRmejBFNTVxeTF6aVFta09OS25uNXcKizOsV9EUukinCAwvpZVrk9x0aXTKQckd
|
||||
gGfdCEU0HZXhZg+ikDFzy52+vPo8+gInjscXiXr/gGn6dJoctLqQXA==
|
||||
-----END AGE ENCRYPTED FILE-----
|
||||
lastmodified: "2023-11-03T13:29:28Z"
|
||||
mac: ENC[AES256_GCM,data:r1peL6D9MIP2UAuQzaX+Tj0wnVZq8ompReOuwMtVEM8yRi5tmF4X5brHOHFURyyPk8AuPVM+Bc3mMw5zoshn/eAFredhAMegA86H0HVri34mxoY8wkVeWWHTqi7QtnudeZMlXn+SPjgsC+d6WYvHEYmI7/VS1XV3cNtVaCmWqd4=,iv:2lQqsryjhMnA7sH1DPRBBYYWrxZeO9QBzRLuob/U0r0=,tag:bsVJh6FH64FoP7GWKpanDA==,type:str]
|
||||
lastmodified: "2023-09-29T12:29:47Z"
|
||||
mac: ENC[AES256_GCM,data:Rap4mZqwBEGJ3rOuMV7yGmRoERedw5hUEOF/jm34qQGIqXnkuQ0TgEK0lXEo+2W1UY0xZYlN/CPz7oej4Tl4iRL3JhVngMotSuV6gNTt5PE67fac2WOlQFQrJynZV9eD4IZahb4aOSO+Vw04RoIFgOZmle5af8vkXVflJmEJhXU=,iv:ovBe3BhEDX3V9X6kQaplYbnoGMseIuDMfX+O+keSgRc=,tag:06V+VODklrJA3VUT+Q1b4A==,type:str]
|
||||
pgp:
|
||||
- created_at: "2023-02-18T08:54:32Z"
|
||||
enc: |
|
||||
|
@ -100,4 +101,4 @@ sops:
|
|||
-----END PGP MESSAGE-----
|
||||
fp: 46C6A7E14BC7812E86C2700737FE303AAC2D06CD
|
||||
unencrypted_suffix: _unencrypted
|
||||
version: 3.8.1
|
||||
version: 3.8.0
|
||||
|
|
|
@ -144,7 +144,7 @@ with dns.lib.combinators; let
|
|||
SOA = {
|
||||
nameServer = "ns1.chir.rs.";
|
||||
adminEmail = "lotte@chir.rs";
|
||||
serial = 42;
|
||||
serial = 43;
|
||||
};
|
||||
NS = [
|
||||
"ns1.chir.rs."
|
||||
|
@ -239,6 +239,7 @@ with dns.lib.combinators; let
|
|||
peertube = createZone {};
|
||||
mediaproxy.CNAME = ["mediaproxy-chir-rs.b-cdn.net."];
|
||||
cache.CNAME = ["cache-chir-rs.b-cdn.net."];
|
||||
attic = createFullZone {};
|
||||
cloud = createZone oracleBase;
|
||||
lotte.CNAME = ["lotte-chir-rs.b-cdn.net."];
|
||||
lotte-nocdn = createZone {};
|
||||
|
|
Loading…
Reference in a new issue