From 42b1f82828eaa2a91a7f27bbca65c725a47df651 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Charlotte=20=F0=9F=A6=9D=20Delenk?= Date: Sat, 30 Nov 2024 08:19:07 +0100 Subject: [PATCH] add caching to the ca store --- Cargo.lock | 206 +++++++++++- Cargo.nix | 619 ++++++++++++++++++++++++++++++++++++- chir-rs-castore/Cargo.toml | 1 + chir-rs-castore/src/lib.rs | 64 +++- chir-rs-config/src/lib.rs | 2 + config-example.toml | 3 + 6 files changed, 883 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e4c5945..79a47ec 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -71,6 +71,48 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" +[[package]] +name = "async-channel" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89b47800b0be77592da0afd425cc03468052844aff33b84e33cc696f64e77b6a" +dependencies = [ + "concurrent-queue", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-io" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43a2b323ccce0a1d90b449fd71f2a06ca7faa7c54c2751f06c9bd851fc061059" +dependencies = [ + "async-lock", + "cfg-if", + "concurrent-queue", + "futures-io", + "futures-lite", + "parking", + "polling", + "rustix", + "slab", + "tracing", + "windows-sys 0.59.0", +] + +[[package]] +name = "async-lock" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff6e472cdea888a4bd64f342f09b3f50e1886d32afe8df3d663c01140b811b18" +dependencies = [ + "event-listener", + "event-listener-strategy", + "pin-project-lite", +] + [[package]] name = "async-trait" version = "0.1.83" @@ -91,6 +133,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "atomic" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d818003e740b63afc82337e3160717f4f63078720a810b7b903e70a5d1d2994" +dependencies = [ + "bytemuck", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -709,6 +760,12 @@ version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" +[[package]] +name = "bytemuck" +version = "1.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b37c88a63ffd85d15b406896cc343916d7cf57838a847b3a6f2ca5d39a5695a" + [[package]] name = "byteorder" version = "1.5.0" @@ -795,6 +852,7 @@ dependencies = [ "chir-rs-misc", "educe", "eyre", + "stretto", "tokio", "tracing", ] @@ -951,6 +1009,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" dependencies = [ "crossbeam-utils", + "portable-atomic", ] [[package]] @@ -1023,6 +1082,15 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-channel" +version = "0.5.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-deque" version = "0.8.5" @@ -1273,6 +1341,18 @@ dependencies = [ "concurrent-queue", "parking", "pin-project-lite", + "portable-atomic", + "portable-atomic-util", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f214dc438f977e6d4e3500aaa277f5ad94ca83fbbd9b1a15713ce2344ccc5a1" +dependencies = [ + "event-listener", + "pin-project-lite", ] [[package]] @@ -1351,6 +1431,21 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" +[[package]] +name = "futures" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + [[package]] name = "futures-channel" version = "0.3.31" @@ -1395,6 +1490,27 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" +[[package]] +name = "futures-lite" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cef40d21ae2c515b51041df9ed313ed21e572df340ea58a922a0aefe7e8891a1" +dependencies = [ + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "futures-macro" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "futures-sink" version = "0.3.31" @@ -1413,8 +1529,10 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ + "futures-channel", "futures-core", "futures-io", + "futures-macro", "futures-sink", "futures-task", "memchr", @@ -1549,6 +1667,12 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" +[[package]] +name = "hermit-abi" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbf6a919d6cf397374f7dfeeea91d974c7c0a7221d0d0f4f20d859d329e53fcc" + [[package]] name = "hex" version = "0.4.3" @@ -2248,7 +2372,7 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi", + "hermit-abi 0.3.9", "libc", ] @@ -2432,12 +2556,36 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" +[[package]] +name = "polling" +version = "3.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a604568c3202727d1507653cb121dbd627a58684eb09a820fd746bee38b4442f" +dependencies = [ + "cfg-if", + "concurrent-queue", + "hermit-abi 0.4.0", + "pin-project-lite", + "rustix", + "tracing", + "windows-sys 0.59.0", +] + [[package]] name = "portable-atomic" version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "280dc24453071f1b63954171985a0b0d30058d287960968b9b2aca264c8d4ee6" +[[package]] +name = "portable-atomic-util" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507" +dependencies = [ + "portable-atomic", +] + [[package]] name = "powerfmt" version = "0.2.0" @@ -2925,6 +3073,12 @@ dependencies = [ "untrusted", ] +[[package]] +name = "seahash" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" + [[package]] name = "sec1" version = "0.3.0" @@ -3506,6 +3660,27 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" +[[package]] +name = "stretto" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70a313e115c2cd9a88d99d60386bc88641c853d468b2c3bc454c294f385fc084" +dependencies = [ + "async-channel", + "async-io", + "atomic", + "crossbeam-channel", + "futures", + "getrandom", + "parking_lot", + "rand", + "seahash", + "thiserror 1.0.69", + "tracing", + "wg", + "xxhash-rust", +] + [[package]] name = "stringprep" version = "0.1.5" @@ -3942,6 +4117,16 @@ dependencies = [ "tracing-serde", ] +[[package]] +name = "triomphe" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef8f7726da4807b58ea5c96fdc122f80702030edc33b35aff9190a51148ccc85" +dependencies = [ + "serde", + "stable_deref_trait", +] + [[package]] name = "try-lock" version = "0.2.5" @@ -4204,6 +4389,19 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "wg" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7aafc5e81e847f05d6770e074faf7b1cd4a5dec9a0e88eac5d55e20fdfebee9a" +dependencies = [ + "event-listener", + "futures-core", + "parking_lot", + "pin-project-lite", + "triomphe", +] + [[package]] name = "which" version = "4.4.2" @@ -4472,6 +4670,12 @@ version = "0.13.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" +[[package]] +name = "xxhash-rust" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a5cbf750400958819fb6178eaa83bee5cd9c29a26a40cc241df8c70fdd46984" + [[package]] name = "yoke" version = "0.7.5" diff --git a/Cargo.nix b/Cargo.nix index d33d006..f99064b 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -31,7 +31,7 @@ args@{ ignoreLockHash, }: let - nixifiedLockHash = "b74868cc2f3f13a8548b99e3b993b2252a39400f0f9048a93634a9bf4eb8fcc2"; + nixifiedLockHash = "49bdb0babe3183dc7c6fb02633d1bd7e6b558833b0bcf158f3c28f862fdb2cc1"; workspaceSrc = if args.workspaceSrc == null then ./. else args.workspaceSrc; currentLockHash = builtins.hashFile "sha256" (workspaceSrc + /Cargo.lock); lockHashIgnored = @@ -263,6 +263,130 @@ else }; }); + "registry+https://github.com/rust-lang/crates.io-index".async-channel."2.3.1" = + overridableMkRustCrate + (profileName: rec { + name = "async-channel"; + version = "2.3.1"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { + inherit name version; + sha256 = "89b47800b0be77592da0afd425cc03468052844aff33b84e33cc696f64e77b6a"; + }; + features = builtins.concatLists [ + [ "default" ] + [ "std" ] + ]; + dependencies = { + concurrent_queue = + (rustPackages."registry+https://github.com/rust-lang/crates.io-index".concurrent-queue."2.5.0" { + inherit profileName; + }).out; + event_listener_strategy = + ( + rustPackages."registry+https://github.com/rust-lang/crates.io-index".event-listener-strategy."0.5.2" + { inherit profileName; } + ).out; + futures_core = + (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.31" { + inherit profileName; + }).out; + pin_project_lite = + (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.15" { + inherit profileName; + }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".async-io."2.4.0" = + overridableMkRustCrate + (profileName: rec { + name = "async-io"; + version = "2.4.0"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { + inherit name version; + sha256 = "43a2b323ccce0a1d90b449fd71f2a06ca7faa7c54c2751f06c9bd851fc061059"; + }; + dependencies = { + async_lock = + (rustPackages."registry+https://github.com/rust-lang/crates.io-index".async-lock."3.4.0" { + inherit profileName; + }).out; + cfg_if = + (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { + inherit profileName; + }).out; + concurrent_queue = + (rustPackages."registry+https://github.com/rust-lang/crates.io-index".concurrent-queue."2.5.0" { + inherit profileName; + }).out; + futures_io = + (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-io."0.3.31" { + inherit profileName; + }).out; + futures_lite = + (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-lite."2.5.0" { + inherit profileName; + }).out; + parking = + (rustPackages."registry+https://github.com/rust-lang/crates.io-index".parking."2.2.1" { + inherit profileName; + }).out; + polling = + (rustPackages."registry+https://github.com/rust-lang/crates.io-index".polling."3.7.4" { + inherit profileName; + }).out; + rustix = + (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rustix."0.38.41" { + inherit profileName; + }).out; + slab = + (rustPackages."registry+https://github.com/rust-lang/crates.io-index".slab."0.4.9" { + inherit profileName; + }).out; + tracing = + (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.41" { + inherit profileName; + }).out; + ${if hostPlatform.isWindows then "windows_sys" else null} = + (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows-sys."0.59.0" { + inherit profileName; + }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".async-lock."3.4.0" = + overridableMkRustCrate + (profileName: rec { + name = "async-lock"; + version = "3.4.0"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { + inherit name version; + sha256 = "ff6e472cdea888a4bd64f342f09b3f50e1886d32afe8df3d663c01140b811b18"; + }; + features = builtins.concatLists [ + [ "default" ] + [ "std" ] + ]; + dependencies = { + event_listener = + (rustPackages."registry+https://github.com/rust-lang/crates.io-index".event-listener."5.3.1" { + inherit profileName; + }).out; + event_listener_strategy = + ( + rustPackages."registry+https://github.com/rust-lang/crates.io-index".event-listener-strategy."0.5.2" + { inherit profileName; } + ).out; + pin_project_lite = + (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.15" { + inherit profileName; + }).out; + }; + }); + "registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.83" = overridableMkRustCrate (profileName: rec { @@ -311,6 +435,28 @@ else }; }); + "registry+https://github.com/rust-lang/crates.io-index".atomic."0.6.0" = + overridableMkRustCrate + (profileName: rec { + name = "atomic"; + version = "0.6.0"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { + inherit name version; + sha256 = "8d818003e740b63afc82337e3160717f4f63078720a810b7b903e70a5d1d2994"; + }; + features = builtins.concatLists [ + [ "default" ] + [ "fallback" ] + ]; + dependencies = { + bytemuck = + (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytemuck."1.20.0" { + inherit profileName; + }).out; + }; + }); + "registry+https://github.com/rust-lang/crates.io-index".atomic-waker."1.1.2" = overridableMkRustCrate (profileName: rec { @@ -2349,6 +2495,18 @@ else ]; }); + "registry+https://github.com/rust-lang/crates.io-index".bytemuck."1.20.0" = + overridableMkRustCrate + (profileName: rec { + name = "bytemuck"; + version = "1.20.0"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { + inherit name version; + sha256 = "8b37c88a63ffd85d15b406896cc343916d7cf57838a847b3a6f2ca5d39a5695a"; + }; + }); + "registry+https://github.com/rust-lang/crates.io-index".byteorder."1.5.0" = overridableMkRustCrate (profileName: rec { @@ -2559,6 +2717,10 @@ else (rustPackages."registry+https://github.com/rust-lang/crates.io-index".eyre."0.6.12" { inherit profileName; }).out; + stretto = + (rustPackages."registry+https://github.com/rust-lang/crates.io-index".stretto."0.8.4" { + inherit profileName; + }).out; tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.41.1" { inherit profileName; @@ -3029,6 +3191,8 @@ else sha256 = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973"; }; features = builtins.concatLists [ + [ "default" ] + [ "portable-atomic" ] [ "std" ] ]; dependencies = { @@ -3036,6 +3200,10 @@ else (rustPackages."registry+https://github.com/rust-lang/crates.io-index".crossbeam-utils."0.8.20" { inherit profileName; }).out; + portable_atomic = + (rustPackages."registry+https://github.com/rust-lang/crates.io-index".portable-atomic."1.10.0" { + inherit profileName; + }).out; }; }); @@ -3203,6 +3371,28 @@ else }; }); + "registry+https://github.com/rust-lang/crates.io-index".crossbeam-channel."0.5.13" = + overridableMkRustCrate + (profileName: rec { + name = "crossbeam-channel"; + version = "0.5.13"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { + inherit name version; + sha256 = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2"; + }; + features = builtins.concatLists [ + [ "default" ] + [ "std" ] + ]; + dependencies = { + crossbeam_utils = + (rustPackages."registry+https://github.com/rust-lang/crates.io-index".crossbeam-utils."0.8.20" { + inherit profileName; + }).out; + }; + }); + "registry+https://github.com/rust-lang/crates.io-index".crossbeam-deque."0.8.5" = overridableMkRustCrate (profileName: rec { @@ -3881,6 +4071,9 @@ else features = builtins.concatLists [ [ "default" ] [ "parking" ] + [ "portable-atomic" ] + [ "portable-atomic-util" ] + [ "portable_atomic_crate" ] [ "std" ] ]; dependencies = { @@ -3896,6 +4089,39 @@ else (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.15" { inherit profileName; }).out; + portable_atomic_crate = + (rustPackages."registry+https://github.com/rust-lang/crates.io-index".portable-atomic."1.10.0" { + inherit profileName; + }).out; + portable_atomic_util = + (rustPackages."registry+https://github.com/rust-lang/crates.io-index".portable-atomic-util."0.2.4" { + inherit profileName; + }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".event-listener-strategy."0.5.2" = + overridableMkRustCrate + (profileName: rec { + name = "event-listener-strategy"; + version = "0.5.2"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { + inherit name version; + sha256 = "0f214dc438f977e6d4e3500aaa277f5ad94ca83fbbd9b1a15713ce2344ccc5a1"; + }; + features = builtins.concatLists [ + [ "std" ] + ]; + dependencies = { + event_listener = + (rustPackages."registry+https://github.com/rust-lang/crates.io-index".event-listener."5.3.1" { + inherit profileName; + }).out; + pin_project_lite = + (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.15" { + inherit profileName; + }).out; }; }); @@ -4096,6 +4322,56 @@ else }; }); + "registry+https://github.com/rust-lang/crates.io-index".futures."0.3.31" = + overridableMkRustCrate + (profileName: rec { + name = "futures"; + version = "0.3.31"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { + inherit name version; + sha256 = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876"; + }; + features = builtins.concatLists [ + [ "alloc" ] + [ "async-await" ] + [ "default" ] + [ "executor" ] + [ "futures-executor" ] + [ "std" ] + ]; + dependencies = { + futures_channel = + (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-channel."0.3.31" { + inherit profileName; + }).out; + futures_core = + (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.31" { + inherit profileName; + }).out; + futures_executor = + (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-executor."0.3.31" { + inherit profileName; + }).out; + futures_io = + (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-io."0.3.31" { + inherit profileName; + }).out; + futures_sink = + (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-sink."0.3.31" { + inherit profileName; + }).out; + futures_task = + (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-task."0.3.31" { + inherit profileName; + }).out; + futures_util = + (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.31" { + inherit profileName; + }).out; + }; + }); + "registry+https://github.com/rust-lang/crates.io-index".futures-channel."0.3.31" = overridableMkRustCrate (profileName: rec { @@ -4220,6 +4496,54 @@ else ]; }); + "registry+https://github.com/rust-lang/crates.io-index".futures-lite."2.5.0" = + overridableMkRustCrate + (profileName: rec { + name = "futures-lite"; + version = "2.5.0"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { + inherit name version; + sha256 = "cef40d21ae2c515b51041df9ed313ed21e572df340ea58a922a0aefe7e8891a1"; + }; + dependencies = { + futures_core = + (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.31" { + inherit profileName; + }).out; + pin_project_lite = + (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.15" { + inherit profileName; + }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".futures-macro."0.3.31" = + overridableMkRustCrate + (profileName: rec { + name = "futures-macro"; + version = "0.3.31"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { + inherit name version; + sha256 = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650"; + }; + dependencies = { + proc_macro2 = + (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.92" { + inherit profileName; + }).out; + quote = + (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.37" { + inherit profileName; + }).out; + syn = + (rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."2.0.90" { + inherit profileName; + }).out; + }; + }); + "registry+https://github.com/rust-lang/crates.io-index".futures-sink."0.3.31" = overridableMkRustCrate (profileName: rec { @@ -4265,7 +4589,12 @@ else }; features = builtins.concatLists [ [ "alloc" ] + [ "async-await" ] + [ "async-await-macro" ] + [ "channel" ] + [ "futures-channel" ] [ "futures-io" ] + [ "futures-macro" ] [ "futures-sink" ] [ "io" ] [ "memchr" ] @@ -4274,6 +4603,10 @@ else [ "std" ] ]; dependencies = { + futures_channel = + (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-channel."0.3.31" { + inherit profileName; + }).out; futures_core = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.31" { inherit profileName; @@ -4282,6 +4615,10 @@ else (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-io."0.3.31" { inherit profileName; }).out; + futures_macro = + (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-macro."0.3.31" { + profileName = "__noProfile"; + }).out; futures_sink = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-sink."0.3.31" { inherit profileName; @@ -4670,6 +5007,21 @@ else ]; }); + "registry+https://github.com/rust-lang/crates.io-index".hermit-abi."0.4.0" = + overridableMkRustCrate + (profileName: rec { + name = "hermit-abi"; + version = "0.4.0"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { + inherit name version; + sha256 = "fbf6a919d6cf397374f7dfeeea91d974c7c0a7221d0d0f4f20d859d329e53fcc"; + }; + features = builtins.concatLists [ + [ "default" ] + ]; + }); + "registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" = overridableMkRustCrate (profileName: rec { @@ -6012,8 +6364,13 @@ else [ "elf" ] [ "errno" ] [ "general" ] + [ "if_ether" ] [ "ioctl" ] + [ "net" ] + [ "netlink" ] [ "no_std" ] + [ "prctl" ] + [ "xdp" ] ]; }); @@ -7076,6 +7433,57 @@ else }; }); + "registry+https://github.com/rust-lang/crates.io-index".polling."3.7.4" = + overridableMkRustCrate + (profileName: rec { + name = "polling"; + version = "3.7.4"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { + inherit name version; + sha256 = "a604568c3202727d1507653cb121dbd627a58684eb09a820fd746bee38b4442f"; + }; + dependencies = { + cfg_if = + (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { + inherit profileName; + }).out; + ${if hostPlatform.isWindows then "concurrent_queue" else null} = + (rustPackages."registry+https://github.com/rust-lang/crates.io-index".concurrent-queue."2.5.0" { + inherit profileName; + }).out; + ${if hostPlatform.parsed.kernel.name == "hermit" then "hermit_abi" else null} = + (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hermit-abi."0.4.0" { + inherit profileName; + }).out; + ${if hostPlatform.isWindows then "pin_project_lite" else null} = + (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.15" { + inherit profileName; + }).out; + ${ + if + hostPlatform.isUnix + || hostPlatform.parsed.kernel.name == "fuchsia" + || hostPlatform.parsed.kernel.name == "vxworks" + then + "rustix" + else + null + } = + (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rustix."0.38.41" { + inherit profileName; + }).out; + tracing = + (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.41" { + inherit profileName; + }).out; + ${if hostPlatform.isWindows then "windows_sys" else null} = + (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows-sys."0.59.0" { + inherit profileName; + }).out; + }; + }); + "registry+https://github.com/rust-lang/crates.io-index".portable-atomic."1.10.0" = overridableMkRustCrate (profileName: rec { @@ -7088,9 +7496,31 @@ else }; features = builtins.concatLists [ [ "fallback" ] + [ "require-cas" ] ]; }); + "registry+https://github.com/rust-lang/crates.io-index".portable-atomic-util."0.2.4" = + overridableMkRustCrate + (profileName: rec { + name = "portable-atomic-util"; + version = "0.2.4"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { + inherit name version; + sha256 = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507"; + }; + features = builtins.concatLists [ + [ "alloc" ] + ]; + dependencies = { + portable_atomic = + (rustPackages."registry+https://github.com/rust-lang/crates.io-index".portable-atomic."1.10.0" { + inherit profileName; + }).out; + }; + }); + "registry+https://github.com/rust-lang/crates.io-index".powerfmt."0.2.0" = overridableMkRustCrate (profileName: rec { @@ -8168,9 +8598,14 @@ else features = builtins.concatLists [ [ "alloc" ] [ "default" ] + [ "event" ] [ "fs" ] [ "libc-extra-traits" ] + [ "net" ] + [ "pipe" ] + [ "process" ] [ "std" ] + [ "time" ] [ "use-libc-auxv" ] ]; dependencies = { @@ -8675,6 +9110,21 @@ else }; }); + "registry+https://github.com/rust-lang/crates.io-index".seahash."4.1.0" = + overridableMkRustCrate + (profileName: rec { + name = "seahash"; + version = "4.1.0"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { + inherit name version; + sha256 = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b"; + }; + features = builtins.concatLists [ + [ "default" ] + ]; + }); + "registry+https://github.com/rust-lang/crates.io-index".sec1."0.3.0" = overridableMkRustCrate (profileName: rec { @@ -10485,6 +10935,81 @@ else ]; }); + "registry+https://github.com/rust-lang/crates.io-index".stretto."0.8.4" = + overridableMkRustCrate + (profileName: rec { + name = "stretto"; + version = "0.8.4"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { + inherit name version; + sha256 = "70a313e115c2cd9a88d99d60386bc88641c853d468b2c3bc454c294f385fc084"; + }; + features = builtins.concatLists [ + [ "async" ] + [ "async-channel" ] + [ "async-io" ] + [ "crossbeam-channel" ] + [ "default" ] + [ "futures" ] + [ "sync" ] + ]; + dependencies = { + async_channel = + (rustPackages."registry+https://github.com/rust-lang/crates.io-index".async-channel."2.3.1" { + inherit profileName; + }).out; + async_io = + (rustPackages."registry+https://github.com/rust-lang/crates.io-index".async-io."2.4.0" { + inherit profileName; + }).out; + atomic = + (rustPackages."registry+https://github.com/rust-lang/crates.io-index".atomic."0.6.0" { + inherit profileName; + }).out; + crossbeam_channel = + (rustPackages."registry+https://github.com/rust-lang/crates.io-index".crossbeam-channel."0.5.13" { + inherit profileName; + }).out; + futures = + (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.31" { + inherit profileName; + }).out; + ${if false then "getrandom" else null} = + (rustPackages."registry+https://github.com/rust-lang/crates.io-index".getrandom."0.2.15" { + inherit profileName; + }).out; + parking_lot = + (rustPackages."registry+https://github.com/rust-lang/crates.io-index".parking_lot."0.12.3" { + inherit profileName; + }).out; + rand = + (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { + inherit profileName; + }).out; + seahash = + (rustPackages."registry+https://github.com/rust-lang/crates.io-index".seahash."4.1.0" { + inherit profileName; + }).out; + thiserror = + (rustPackages."registry+https://github.com/rust-lang/crates.io-index".thiserror."1.0.69" { + inherit profileName; + }).out; + tracing = + (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.41" { + inherit profileName; + }).out; + wg = + (rustPackages."registry+https://github.com/rust-lang/crates.io-index".wg."0.9.2" { + inherit profileName; + }).out; + xxhash_rust = + (rustPackages."registry+https://github.com/rust-lang/crates.io-index".xxhash-rust."0.8.12" { + inherit profileName; + }).out; + }; + }); + "registry+https://github.com/rust-lang/crates.io-index".stringprep."0.1.5" = overridableMkRustCrate (profileName: rec { @@ -11717,6 +12242,34 @@ else }; }); + "registry+https://github.com/rust-lang/crates.io-index".triomphe."0.1.14" = + overridableMkRustCrate + (profileName: rec { + name = "triomphe"; + version = "0.1.14"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { + inherit name version; + sha256 = "ef8f7726da4807b58ea5c96fdc122f80702030edc33b35aff9190a51148ccc85"; + }; + features = builtins.concatLists [ + [ "default" ] + [ "serde" ] + [ "stable_deref_trait" ] + [ "std" ] + ]; + dependencies = { + serde = + (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.215" { + inherit profileName; + }).out; + stable_deref_trait = + (rustPackages."registry+https://github.com/rust-lang/crates.io-index".stable_deref_trait."1.2.0" { + inherit profileName; + }).out; + }; + }); + "registry+https://github.com/rust-lang/crates.io-index".try-lock."0.2.5" = overridableMkRustCrate (profileName: rec { @@ -12391,6 +12944,49 @@ else }; }); + "registry+https://github.com/rust-lang/crates.io-index".wg."0.9.2" = + overridableMkRustCrate + (profileName: rec { + name = "wg"; + version = "0.9.2"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { + inherit name version; + sha256 = "7aafc5e81e847f05d6770e074faf7b1cd4a5dec9a0e88eac5d55e20fdfebee9a"; + }; + features = builtins.concatLists [ + [ "default" ] + [ "event-listener" ] + [ "future" ] + [ "parking_lot" ] + [ "pin-project-lite" ] + [ "std" ] + [ "triomphe" ] + ]; + dependencies = { + event_listener = + (rustPackages."registry+https://github.com/rust-lang/crates.io-index".event-listener."5.3.1" { + inherit profileName; + }).out; + futures_core = + (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.31" { + inherit profileName; + }).out; + parking_lot = + (rustPackages."registry+https://github.com/rust-lang/crates.io-index".parking_lot."0.12.3" { + inherit profileName; + }).out; + pin_project_lite = + (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.15" { + inherit profileName; + }).out; + triomphe = + (rustPackages."registry+https://github.com/rust-lang/crates.io-index".triomphe."0.1.14" { + inherit profileName; + }).out; + }; + }); + "registry+https://github.com/rust-lang/crates.io-index".which."4.4.2" = overridableMkRustCrate (profileName: rec { @@ -12727,6 +13323,10 @@ else sha256 = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"; }; features = builtins.concatLists [ + [ "Wdk" ] + [ "Wdk_Foundation" ] + [ "Wdk_Storage" ] + [ "Wdk_Storage_FileSystem" ] [ "Win32" ] [ "Win32_Foundation" ] [ "Win32_Networking" ] @@ -12745,6 +13345,8 @@ else [ "Win32_System_LibraryLoader" ] [ "Win32_System_Memory" ] [ "Win32_System_SystemInformation" ] + [ "Win32_System_Threading" ] + [ "Win32_System_WindowsProgramming" ] [ "default" ] ]; dependencies = { @@ -13146,6 +13748,21 @@ else ]; }); + "registry+https://github.com/rust-lang/crates.io-index".xxhash-rust."0.8.12" = + overridableMkRustCrate + (profileName: rec { + name = "xxhash-rust"; + version = "0.8.12"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { + inherit name version; + sha256 = "6a5cbf750400958819fb6178eaa83bee5cd9c29a26a40cc241df8c70fdd46984"; + }; + features = builtins.concatLists [ + [ "xxh64" ] + ]; + }); + "registry+https://github.com/rust-lang/crates.io-index".yoke."0.7.5" = overridableMkRustCrate (profileName: rec { diff --git a/chir-rs-castore/Cargo.toml b/chir-rs-castore/Cargo.toml index 62ec8ba..a42dc05 100644 --- a/chir-rs-castore/Cargo.toml +++ b/chir-rs-castore/Cargo.toml @@ -12,6 +12,7 @@ chir-rs-config = { version = "0.1.0", path = "../chir-rs-config" } chir-rs-misc = { version = "0.1.0", path = "../chir-rs-misc", features = ["id-generator", "base64"] } educe = { version = "0.6.0", default-features = false, features = ["Debug"] } eyre = "0.6.12" +stretto = { version = "0.8.4", features = ["async"] } tokio = { version = "1.41.1", features = ["fs"] } tracing = "0.1.41" [lints.rust] diff --git a/chir-rs-castore/src/lib.rs b/chir-rs-castore/src/lib.rs index 739168b..1d438ac 100644 --- a/chir-rs-castore/src/lib.rs +++ b/chir-rs-castore/src/lib.rs @@ -9,12 +9,13 @@ use aws_sdk_s3::{ types::{CompletedMultipartUpload, CompletedPart}, Client, }; -use blake3::Hasher; -use bytes::BytesMut; +use blake3::{Hash, Hasher}; +use bytes::{Bytes, BytesMut}; use chir_rs_config::ChirRs; use chir_rs_misc::{id_generator, lexicographic_base64}; use educe::Educe; use eyre::{Context as _, Result}; +use stretto::{AsyncCache, AsyncCacheBuilder}; use tokio::{ fs::read_to_string, io::{AsyncRead, AsyncReadExt}, @@ -53,6 +54,9 @@ pub struct CaStore { client: Arc, /// Bucket bucket: Arc, + /// CA Value Cache + #[educe(Debug(ignore))] + cache: AsyncCache, } impl CaStore { @@ -66,6 +70,16 @@ impl CaStore { Ok(Self { client: Arc::new(Client::new(&sdk_config)), bucket: Arc::from(config.s3.bucket.as_ref()), + cache: AsyncCache::new( + (config.cache_max_size / 1_000) + .try_into() + .context("Cache size too large")?, + config + .cache_max_size + .try_into() + .context("Value too large")?, + tokio::spawn, + )?, }) } @@ -75,7 +89,7 @@ impl CaStore { /// /// This function returns an error if reading the source stream fails, uploading the source stream fails, or moving the file to its correct content-addressed position fails. #[instrument(skip(reader))] - async fn upload_inner(&self, reader: R, id: u128) -> Result + async fn upload_inner(&self, reader: R, id: u128) -> Result where R: AsyncRead + AsyncReadExt + Send, { @@ -178,11 +192,12 @@ impl CaStore { /// # Errors /// /// This function returns an error if reading the source stream fails, uploading the source stream fails, or moving the file to its correct content-addressed position fails. - pub async fn upload(&self, reader: R) -> Result + pub async fn upload(&self, reader: R) -> Result where R: AsyncRead + AsyncReadExt + Send, { let id = id_generator::generate(); + self.upload_inner(reader, id).await } @@ -192,10 +207,19 @@ impl CaStore { /// /// This function returns an error if loading file matadata fails #[instrument] - pub async fn download_bytestream( - &self, - hash: blake3::Hash, - ) -> Result<(Option, ByteStream)> { + pub async fn download_bytestream(&self, hash: Hash) -> Result<(Option, ByteStream)> { + #[allow( + clippy::significant_drop_in_scrutinee, + reason = "We are cloning like 1 arc lol" + )] + if let Some(v) = self.cache.get(&hash).await { + info!("{hash:?} found in cache. Returning"); + let data = v.as_ref(); + return Ok(( + Some(data.len().try_into().context("Bad api design")?), + data.clone().into(), + )); + } let key = lexicographic_base64::encode(hash.as_bytes()); let file = self .client @@ -205,7 +229,27 @@ impl CaStore { .send() .await .with_context(|| format!("Downloading content-addressed file {key}"))?; - Ok((file.content_length, file.body)) + let file_size_hint = file.content_length.unwrap_or(i64::MAX); + let file_size_hint: u64 = file_size_hint.try_into().unwrap_or(u64::MAX); + if file_size_hint < 1_000_000 { + // Cache this + let data = file.body.collect().await?.into_bytes(); + + self.cache + .insert( + hash, + data.clone(), + data.len().try_into().context("Bad api design")?, + ) + .await; + + Ok(( + Some(data.len().try_into().context("Bad api design")?), + data.into(), + )) + } else { + Ok((file.content_length, file.body)) + } } /// Downloads a file from the CA store backend with its hash @@ -214,7 +258,7 @@ impl CaStore { /// /// This function returns an error if loading file matadata fails #[instrument] - pub async fn download(&self, hash: blake3::Hash) -> Result<(Option, SdkBody)> { + pub async fn download(&self, hash: Hash) -> Result<(Option, SdkBody)> { let (length, body) = self.download_bytestream(hash).await?; Ok((length, body.into_inner())) } diff --git a/chir-rs-config/src/lib.rs b/chir-rs-config/src/lib.rs index de642c6..a851066 100644 --- a/chir-rs-config/src/lib.rs +++ b/chir-rs-config/src/lib.rs @@ -216,6 +216,8 @@ pub struct ChirRs { pub database: Database, /// S3 configuration pub s3: S3Config, + /// Ca Cache max size in bytes + pub cache_max_size: u64, } impl ChirRs { diff --git a/config-example.toml b/config-example.toml index cc97bd9..b454d5b 100644 --- a/config-example.toml +++ b/config-example.toml @@ -1,3 +1,6 @@ +# Maximum size of the in-memory object cache. +cache_max_size = 32_000_000 + [logging] #sentry_dsn = "…" log_level = "debug"