Lock file maintenance #8
9 changed files with 7607 additions and 15040 deletions
86
Cargo.lock
generated
86
Cargo.lock
generated
|
|
@ -875,9 +875,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.2.2"
|
||||
version = "1.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f34d93e62b03caf570cccc334cbc6c2fceca82f39211051345108adcba3eebdc"
|
||||
checksum = "27f657647bcff5394bf56c7317665bbf790a137a50eaaa5c6bfbb9e27a518f2d"
|
||||
dependencies = [
|
||||
"jobserver",
|
||||
"libc",
|
||||
|
|
@ -1011,7 +1011,7 @@ dependencies = [
|
|||
"eyre",
|
||||
"rustls 0.23.19",
|
||||
"tokio",
|
||||
"tokio-rustls 0.26.0",
|
||||
"tokio-rustls 0.26.1",
|
||||
"tracing",
|
||||
"url",
|
||||
]
|
||||
|
|
@ -1036,7 +1036,6 @@ dependencies = [
|
|||
"mime",
|
||||
"rusty_paseto",
|
||||
"sentry-tower",
|
||||
"serde",
|
||||
"tokio",
|
||||
"tokio-util",
|
||||
"tower-http 0.6.2",
|
||||
|
|
@ -1058,7 +1057,7 @@ dependencies = [
|
|||
"http 1.2.0",
|
||||
"mime",
|
||||
"serde",
|
||||
"thiserror 2.0.4",
|
||||
"thiserror 2.0.6",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
|
|
@ -1108,9 +1107,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "clap"
|
||||
version = "4.5.22"
|
||||
version = "4.5.23"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "69371e34337c4c984bbe322360c2547210bf632eb2814bbe78a6e87a2935bd2b"
|
||||
checksum = "3135e7ec2ef7b10c6ed8950f0f792ed96ee093fa088608f1c76e569722700c84"
|
||||
dependencies = [
|
||||
"clap_builder",
|
||||
"clap_derive",
|
||||
|
|
@ -1118,9 +1117,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "clap_builder"
|
||||
version = "4.5.22"
|
||||
version = "4.5.23"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6e24c1b4099818523236a8ca881d2b45db98dadfb4625cf6608c12069fcbbde1"
|
||||
checksum = "30582fc632330df2bd26877bde0c1f4470d57c582bbc070376afcd04d8cb4838"
|
||||
dependencies = [
|
||||
"anstream",
|
||||
"anstyle",
|
||||
|
|
@ -1142,9 +1141,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "clap_lex"
|
||||
version = "0.7.3"
|
||||
version = "0.7.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "afb84c814227b90d6895e01398aee0d8033c00e7466aca416fb6a8e0eb19d8a7"
|
||||
checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6"
|
||||
|
||||
[[package]]
|
||||
name = "cmake"
|
||||
|
|
@ -1564,9 +1563,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "fastrand"
|
||||
version = "2.2.0"
|
||||
version = "2.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "486f806e73c5707928240ddc295403b1b93c96a02038563881c4a2fd84b81ac4"
|
||||
checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
|
||||
|
||||
[[package]]
|
||||
name = "ff"
|
||||
|
|
@ -2056,7 +2055,7 @@ dependencies = [
|
|||
"rustls 0.23.19",
|
||||
"rustls-pki-types",
|
||||
"tokio",
|
||||
"tokio-rustls 0.26.0",
|
||||
"tokio-rustls 0.26.1",
|
||||
"tower-service",
|
||||
"webpki-roots",
|
||||
]
|
||||
|
|
@ -2314,9 +2313,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "js-sys"
|
||||
version = "0.3.74"
|
||||
version = "0.3.76"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a865e038f7f6ed956f788f0d7d60c541fff74c7bd74272c5d4cf15c63743e705"
|
||||
checksum = "6717b6b5b077764fb5966237269cb3c64edddde4b14ce42647430a78ced9e7b7"
|
||||
dependencies = [
|
||||
"once_cell",
|
||||
"wasm-bindgen",
|
||||
|
|
@ -2891,7 +2890,7 @@ dependencies = [
|
|||
"rustc-hash 2.1.0",
|
||||
"rustls 0.23.19",
|
||||
"socket2",
|
||||
"thiserror 2.0.4",
|
||||
"thiserror 2.0.6",
|
||||
"tokio",
|
||||
"tracing",
|
||||
]
|
||||
|
|
@ -2910,7 +2909,7 @@ dependencies = [
|
|||
"rustls 0.23.19",
|
||||
"rustls-pki-types",
|
||||
"slab",
|
||||
"thiserror 2.0.4",
|
||||
"thiserror 2.0.6",
|
||||
"tinyvec",
|
||||
"tracing",
|
||||
"web-time",
|
||||
|
|
@ -3058,6 +3057,7 @@ dependencies = [
|
|||
"futures-channel",
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
"h2 0.4.7",
|
||||
"http 1.2.0",
|
||||
"http-body 1.0.1",
|
||||
"http-body-util",
|
||||
|
|
@ -3080,7 +3080,7 @@ dependencies = [
|
|||
"serde_urlencoded",
|
||||
"sync_wrapper 1.0.2",
|
||||
"tokio",
|
||||
"tokio-rustls 0.26.0",
|
||||
"tokio-rustls 0.26.1",
|
||||
"tokio-util",
|
||||
"tower-service",
|
||||
"url",
|
||||
|
|
@ -4032,11 +4032,11 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "thiserror"
|
||||
version = "2.0.4"
|
||||
version = "2.0.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2f49a1853cf82743e3b7950f77e0f4d622ca36cf4317cba00c767838bac8d490"
|
||||
checksum = "8fec2a1820ebd077e2b90c4df007bebf344cd394098a13c563957d0afc83ea47"
|
||||
dependencies = [
|
||||
"thiserror-impl 2.0.4",
|
||||
"thiserror-impl 2.0.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -4052,9 +4052,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "thiserror-impl"
|
||||
version = "2.0.4"
|
||||
version = "2.0.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8381894bb3efe0c4acac3ded651301ceee58a15d47c2e34885ed1908ad667061"
|
||||
checksum = "d65750cab40f4ff1929fb1ba509e9914eb756131cef4210da8d5d700d26f6312"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
|
|
@ -4167,20 +4167,19 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "tokio-rustls"
|
||||
version = "0.26.0"
|
||||
version = "0.26.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4"
|
||||
checksum = "5f6d0975eaace0cf0fcadee4e4aaa5da15b5c079146f2cffb67c113be122bf37"
|
||||
dependencies = [
|
||||
"rustls 0.23.19",
|
||||
"rustls-pki-types",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-stream"
|
||||
version = "0.1.16"
|
||||
version = "0.1.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4f4e6ce100d0eb49a2734f8c0812bcd324cf357d21810932c5df6b96ef2b86f1"
|
||||
checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
"pin-project-lite",
|
||||
|
|
@ -4586,9 +4585,9 @@ checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b"
|
|||
|
||||
[[package]]
|
||||
name = "wasm-bindgen"
|
||||
version = "0.2.97"
|
||||
version = "0.2.99"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d15e63b4482863c109d70a7b8706c1e364eb6ea449b201a76c5b89cedcec2d5c"
|
||||
checksum = "a474f6281d1d70c17ae7aa6a613c87fce69a127e2624002df63dcb39d6cf6396"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"once_cell",
|
||||
|
|
@ -4597,13 +4596,12 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-backend"
|
||||
version = "0.2.97"
|
||||
version = "0.2.99"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8d36ef12e3aaca16ddd3f67922bc63e48e953f126de60bd33ccc0101ef9998cd"
|
||||
checksum = "5f89bb38646b4f81674e8f5c3fb81b562be1fd936d84320f3264486418519c79"
|
||||
dependencies = [
|
||||
"bumpalo",
|
||||
"log",
|
||||
"once_cell",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
|
|
@ -4612,9 +4610,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-futures"
|
||||
version = "0.4.47"
|
||||
version = "0.4.49"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9dfaf8f50e5f293737ee323940c7d8b08a66a95a419223d9f41610ca08b0833d"
|
||||
checksum = "38176d9b44ea84e9184eff0bc34cc167ed044f816accfe5922e54d84cf48eca2"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"js-sys",
|
||||
|
|
@ -4625,9 +4623,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-macro"
|
||||
version = "0.2.97"
|
||||
version = "0.2.99"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "705440e08b42d3e4b36de7d66c944be628d579796b8090bfa3471478a2260051"
|
||||
checksum = "2cc6181fd9a7492eef6fef1f33961e3695e4579b9872a6f7c83aee556666d4fe"
|
||||
dependencies = [
|
||||
"quote",
|
||||
"wasm-bindgen-macro-support",
|
||||
|
|
@ -4635,9 +4633,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-macro-support"
|
||||
version = "0.2.97"
|
||||
version = "0.2.99"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "98c9ae5a76e46f4deecd0f0255cc223cfa18dc9b261213b8aa0c7b36f61b3f1d"
|
||||
checksum = "30d7a95b763d3c45903ed6c81f156801839e5ee968bb07e534c44df0fcd330c2"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
|
|
@ -4648,9 +4646,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-shared"
|
||||
version = "0.2.97"
|
||||
version = "0.2.99"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6ee99da9c5ba11bd675621338ef6fa52296b76b83305e9b6e5c77d4c286d6d49"
|
||||
checksum = "943aab3fdaaa029a6e0271b35ea10b72b943135afe9bffca82384098ad0e06a6"
|
||||
|
||||
[[package]]
|
||||
name = "wasm-streams"
|
||||
|
|
@ -4667,9 +4665,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "web-sys"
|
||||
version = "0.3.74"
|
||||
version = "0.3.76"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a98bc3c33f0fe7e59ad7cd041b89034fa82a7c2d4365ca538dda6cdaf513863c"
|
||||
checksum = "04dd7223427d52553d3702c004d3b2fe07c148165faa56313cb00211e31c12bc"
|
||||
dependencies = [
|
||||
"js-sys",
|
||||
"wasm-bindgen",
|
||||
|
|
|
|||
|
|
@ -97,94 +97,100 @@ impl CaStore {
|
|||
let string_id = lexicographic_base64::encode(id.to_be_bytes());
|
||||
let source_fname = format!("temp/{string_id}");
|
||||
|
||||
/*info!("Starting multipart upload {id}");
|
||||
let multipart_result = self
|
||||
.client
|
||||
.create_multipart_upload()
|
||||
.bucket(&*self.bucket)
|
||||
.key(&source_fname)
|
||||
.send()
|
||||
.await
|
||||
.with_context(|| format!("Creating multipart request for Request ID{id}"))?;
|
||||
|
||||
let mut buf = BytesMut::with_capacity(16 * 1024 * 1024); // 16MiB byte buffer for the file
|
||||
let mut buf = BytesMut::with_capacity(5_000_000);
|
||||
reader.read_buf(&mut buf).await?;
|
||||
let hasher = Arc::new(Mutex::new(Hasher::new()));
|
||||
|
||||
let mut i = 1;
|
||||
let mut completed_multipart_upload_builder = CompletedMultipartUpload::builder();
|
||||
|
||||
loop {
|
||||
buf.clear();
|
||||
reader.read_buf(&mut buf).await.context("Reading chunk")?;
|
||||
if buf.is_empty() {
|
||||
break;
|
||||
}
|
||||
|
||||
debug!("Uploading part {i} for multipart upload {id}");
|
||||
|
||||
let buf2 = buf.clone();
|
||||
let hasher = Arc::clone(&hasher);
|
||||
let hasher_job = spawn_blocking(move || {
|
||||
hasher.blocking_lock().update_rayon(&buf2);
|
||||
});
|
||||
|
||||
let part_upload_fut = self
|
||||
let hash = if buf.len() >= 5_000_000 {
|
||||
info!("Starting multipart upload {id}");
|
||||
let multipart_result = self
|
||||
.client
|
||||
.upload_part()
|
||||
.create_multipart_upload()
|
||||
.bucket(&*self.bucket)
|
||||
.key(&source_fname)
|
||||
.set_upload_id(multipart_result.upload_id.clone())
|
||||
.body(ByteStream::from(buf.to_vec()))
|
||||
.part_number(i)
|
||||
.send();
|
||||
.send()
|
||||
.await
|
||||
.with_context(|| format!("Creating multipart request for Request ID{id}"))?;
|
||||
|
||||
let ((), part_upload_result) = try_join!(
|
||||
async { hasher_job.await.context("Awaiting hasher job") },
|
||||
async { part_upload_fut.await.context("Awaiting uploader job") }
|
||||
)
|
||||
.context("Awaiting job for chunk")?;
|
||||
completed_multipart_upload_builder = completed_multipart_upload_builder.parts(
|
||||
CompletedPart::builder()
|
||||
.e_tag(part_upload_result.e_tag.unwrap_or_default())
|
||||
let mut reader = buf.chain(reader);
|
||||
|
||||
let mut buf = BytesMut::with_capacity(16 * 1024 * 1024); // 16MiB byte buffer for the file
|
||||
let hasher = Arc::new(Mutex::new(Hasher::new()));
|
||||
|
||||
let mut i = 1;
|
||||
let mut completed_multipart_upload_builder = CompletedMultipartUpload::builder();
|
||||
|
||||
loop {
|
||||
buf.clear();
|
||||
reader.read_buf(&mut buf).await.context("Reading chunk")?;
|
||||
if buf.is_empty() {
|
||||
break;
|
||||
}
|
||||
|
||||
debug!("Uploading part {i} for multipart upload {id}");
|
||||
|
||||
let buf2 = buf.clone();
|
||||
let hasher = Arc::clone(&hasher);
|
||||
let hasher_job = spawn_blocking(move || {
|
||||
hasher.blocking_lock().update_rayon(&buf2);
|
||||
});
|
||||
|
||||
let part_upload_fut = self
|
||||
.client
|
||||
.upload_part()
|
||||
.bucket(&*self.bucket)
|
||||
.key(&source_fname)
|
||||
.set_upload_id(multipart_result.upload_id.clone())
|
||||
.body(ByteStream::from(buf.to_vec()))
|
||||
.part_number(i)
|
||||
.build(),
|
||||
);
|
||||
i += 1;
|
||||
}
|
||||
.send();
|
||||
|
||||
debug!("Finalizing Multipart Upload {id}");
|
||||
let ((), part_upload_result) = try_join!(
|
||||
async { hasher_job.await.context("Awaiting hasher job") },
|
||||
async { part_upload_fut.await.context("Awaiting uploader job") }
|
||||
)
|
||||
.context("Awaiting job for chunk")?;
|
||||
completed_multipart_upload_builder = completed_multipart_upload_builder.parts(
|
||||
CompletedPart::builder()
|
||||
.e_tag(part_upload_result.e_tag.unwrap_or_default())
|
||||
.part_number(i)
|
||||
.build(),
|
||||
);
|
||||
i += 1;
|
||||
}
|
||||
|
||||
let hash = hasher.lock().await.finalize();
|
||||
self.client
|
||||
.complete_multipart_upload()
|
||||
.bucket(&*self.bucket)
|
||||
.key(&source_fname)
|
||||
.multipart_upload(completed_multipart_upload_builder.build())
|
||||
.set_upload_id(multipart_result.upload_id)
|
||||
.send()
|
||||
.await
|
||||
.context("Completing multipart upload")?;*/
|
||||
debug!("Finalizing Multipart Upload {id}");
|
||||
|
||||
let hasher = Arc::new(Mutex::new(Hasher::new()));
|
||||
let mut buf = Vec::new();
|
||||
reader.read_to_end(&mut buf).await?;
|
||||
let buf = Bytes::from(buf);
|
||||
let buf2 = buf.clone();
|
||||
let hasher2 = Arc::clone(&hasher);
|
||||
spawn_blocking(move || {
|
||||
hasher2.blocking_lock().update_rayon(&buf2);
|
||||
})
|
||||
.await?;
|
||||
self.client
|
||||
.put_object()
|
||||
.bucket(&*self.bucket)
|
||||
.key(&source_fname)
|
||||
.body(ByteStream::from(buf.to_vec()))
|
||||
.send()
|
||||
.await
|
||||
.context("Uploading file")?;
|
||||
let hash = hasher.lock().await.finalize();
|
||||
self.client
|
||||
.complete_multipart_upload()
|
||||
.bucket(&*self.bucket)
|
||||
.key(&source_fname)
|
||||
.multipart_upload(completed_multipart_upload_builder.build())
|
||||
.set_upload_id(multipart_result.upload_id)
|
||||
.send()
|
||||
.await
|
||||
.context("Completing multipart upload")?;
|
||||
hash
|
||||
} else {
|
||||
let buf = Bytes::from(buf);
|
||||
let buf2 = buf.clone();
|
||||
let hasher2 = Arc::clone(&hasher);
|
||||
spawn_blocking(move || {
|
||||
hasher2.blocking_lock().update_rayon(&buf2);
|
||||
})
|
||||
.await?;
|
||||
self.client
|
||||
.put_object()
|
||||
.bucket(&*self.bucket)
|
||||
.key(&source_fname)
|
||||
.body(ByteStream::from(buf.to_vec()))
|
||||
.send()
|
||||
.await
|
||||
.context("Uploading file")?;
|
||||
|
||||
let hash = hasher.lock().await.finalize();
|
||||
hasher.lock().await.finalize()
|
||||
};
|
||||
|
||||
let target_fname = lexicographic_base64::encode(hash.as_bytes());
|
||||
|
||||
|
|
|
|||
|
|
@ -12,7 +12,11 @@ dotenvy = "0.15.7"
|
|||
eyre = "0.6.12"
|
||||
mime = "0.3.17"
|
||||
mime_guess = "2.0.5"
|
||||
reqwest = { version = "0.12.9", default-features = false, features = ["rustls-tls", "stream"] }
|
||||
reqwest = { version = "0.12.9", default-features = false, features = [
|
||||
"http2",
|
||||
"rustls-tls",
|
||||
"stream",
|
||||
] }
|
||||
tokio = { version = "1.41.1", features = ["fs", "macros", "rt-multi-thread"] }
|
||||
tracing = "0.1.41"
|
||||
tracing-subscriber = "0.3.19"
|
||||
|
|
|
|||
|
|
@ -1,20 +1,22 @@
|
|||
use std::{collections::HashSet, future::Future, path::Path, pin::Pin};
|
||||
use std::{
|
||||
collections::HashSet, future::Future, path::PathBuf, pin::Pin, sync::Arc, time::Duration,
|
||||
};
|
||||
|
||||
use chir_rs_http_api::{
|
||||
auth::{LoginRequest, PasetoToken, Scope},
|
||||
errors::APIError,
|
||||
};
|
||||
use clap::{arg, Parser, Subcommand};
|
||||
use eyre::{eyre, Context as _, OptionExt as _, Result};
|
||||
use mime_guess::{Mime, MimeGuess};
|
||||
use eyre::{eyre, Context as _, Result};
|
||||
use mime_guess::MimeGuess;
|
||||
use reqwest::Body;
|
||||
use tokio::join;
|
||||
use tracing::instrument;
|
||||
use tokio::{sync::Mutex, task::JoinSet};
|
||||
use tracing::{info, instrument};
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
#[command(version, about, long_about = None)]
|
||||
struct Args {
|
||||
#[arg(short, long, default_value = "http://localhost:5621/")]
|
||||
#[arg(short, long, default_value = "https://lotte.chir.rs/")]
|
||||
url: String,
|
||||
#[command(subcommand)]
|
||||
command: Command,
|
||||
|
|
@ -42,14 +44,18 @@ enum Command {
|
|||
},
|
||||
}
|
||||
|
||||
async fn login(url: String, username: String, password: String) -> Result<()> {
|
||||
async fn login(
|
||||
client: Arc<reqwest::Client>,
|
||||
url: String,
|
||||
username: String,
|
||||
password: String,
|
||||
) -> Result<()> {
|
||||
let request = LoginRequest {
|
||||
username,
|
||||
password,
|
||||
scopes: HashSet::from([Scope::Full]),
|
||||
};
|
||||
let request = bincode::encode_to_vec(request, bincode::config::standard())?;
|
||||
let client = reqwest::Client::new();
|
||||
let res = client
|
||||
.post(format!("{url}.api/auth/login"))
|
||||
.header("Content-Type", "application/x+bincode")
|
||||
|
|
@ -74,9 +80,13 @@ async fn login(url: String, username: String, password: String) -> Result<()> {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
#[instrument(skip(source))]
|
||||
async fn upload(url: String, source: impl AsRef<Path>, dest: String) -> Result<()> {
|
||||
let client = reqwest::Client::new();
|
||||
#[instrument(skip(client))]
|
||||
async fn upload_inner(
|
||||
client: Arc<reqwest::Client>,
|
||||
url: String,
|
||||
source: PathBuf,
|
||||
dest: String,
|
||||
) -> Result<bool> {
|
||||
let token = std::env::var("CHIR_RS_TOKEN")?;
|
||||
let file = tokio::fs::File::open(&source).await?;
|
||||
let res = client
|
||||
|
|
@ -92,17 +102,55 @@ async fn upload(url: String, source: impl AsRef<Path>, dest: String) -> Result<(
|
|||
.body(Body::from(file))
|
||||
.send()
|
||||
.await?;
|
||||
if !res.status().is_success() {
|
||||
let status = res.status();
|
||||
if status.is_client_error() {
|
||||
let response = res.bytes().await?;
|
||||
let response: APIError =
|
||||
bincode::decode_from_slice(&response, bincode::config::standard())?.0;
|
||||
Err(response).with_context(|| format!("Uploading to {dest}"))?;
|
||||
} else if status.is_server_error() {
|
||||
let response = res.bytes().await?;
|
||||
let response: APIError =
|
||||
bincode::decode_from_slice(&response, bincode::config::standard())?.0;
|
||||
println!("Server error updating {dest}: {response:?}");
|
||||
return Ok(false);
|
||||
}
|
||||
info!("Updated {dest}.");
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
#[instrument(skip(client))]
|
||||
async fn upload(
|
||||
client: Arc<reqwest::Client>,
|
||||
url: String,
|
||||
source: PathBuf,
|
||||
dest: String,
|
||||
) -> Result<()> {
|
||||
loop {
|
||||
if upload_inner(
|
||||
Arc::clone(&client),
|
||||
url.clone(),
|
||||
source.clone(),
|
||||
dest.clone(),
|
||||
)
|
||||
.await?
|
||||
{
|
||||
break;
|
||||
}
|
||||
println!("Uploading to {dest} failed. waiting 5s");
|
||||
tokio::time::sleep(Duration::from_secs(5)).await;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[instrument(skip(source))]
|
||||
async fn upload_dir(url: String, source: impl AsRef<Path>, dest: String) -> Result<()> {
|
||||
#[instrument(skip(client))]
|
||||
async fn upload_dir(
|
||||
client: Arc<reqwest::Client>,
|
||||
futs: Arc<Mutex<JoinSet<Result<()>>>>,
|
||||
url: String,
|
||||
source: PathBuf,
|
||||
dest: String,
|
||||
) -> Result<()> {
|
||||
let mut dir = tokio::fs::read_dir(source).await?;
|
||||
while let Some(ent) = dir.next_entry().await? {
|
||||
let file_type = ent.file_type().await?;
|
||||
|
|
@ -116,19 +164,39 @@ async fn upload_dir(url: String, source: impl AsRef<Path>, dest: String) -> Resu
|
|||
format!("{dest}/{file_name_str}")
|
||||
};
|
||||
if file_type.is_dir() {
|
||||
let sub_fut: Pin<Box<dyn Future<Output = Result<()>>>> =
|
||||
Box::pin(upload_dir(url.clone(), ent.path(), tgt));
|
||||
sub_fut.await?;
|
||||
let boxed: Pin<Box<dyn Future<Output = Result<()>>>> = Box::pin(upload_dir(
|
||||
Arc::clone(&client),
|
||||
Arc::clone(&futs),
|
||||
url.clone(),
|
||||
ent.path().to_path_buf(),
|
||||
tgt,
|
||||
));
|
||||
boxed.await?;
|
||||
|
||||
continue;
|
||||
}
|
||||
if file_name_str == "index.html" {
|
||||
if dest.is_empty() {
|
||||
upload(url.clone(), ent.path(), "".to_string()).await?;
|
||||
} else {
|
||||
upload(url.clone(), ent.path(), format!("{dest}/")).await?;
|
||||
if !dest.is_empty() {
|
||||
futs.lock().await.spawn(upload(
|
||||
Arc::clone(&client),
|
||||
url.clone(),
|
||||
ent.path().to_path_buf(),
|
||||
format!("{dest}/"),
|
||||
));
|
||||
}
|
||||
futs.lock().await.spawn(upload(
|
||||
Arc::clone(&client),
|
||||
url.clone(),
|
||||
ent.path().to_path_buf(),
|
||||
dest.clone(),
|
||||
));
|
||||
}
|
||||
upload(url.clone(), ent.path(), tgt).await?;
|
||||
futs.lock().await.spawn(upload(
|
||||
Arc::clone(&client),
|
||||
url.clone(),
|
||||
ent.path().to_path_buf(),
|
||||
tgt,
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -139,11 +207,22 @@ async fn main() -> Result<()> {
|
|||
dotenvy::dotenv().ok();
|
||||
tracing_subscriber::fmt::init();
|
||||
let matches = Args::parse();
|
||||
let client = Arc::new(reqwest::Client::new());
|
||||
|
||||
match matches.command {
|
||||
Command::Login { username, password } => login(matches.url, username, password).await?,
|
||||
Command::Upload { source, dest } => upload(matches.url, source, dest).await?,
|
||||
Command::UploadDir { source, dest } => upload_dir(matches.url, source, dest).await?,
|
||||
Command::Login { username, password } => {
|
||||
login(client, matches.url, username, password).await?
|
||||
}
|
||||
Command::Upload { source, dest } => {
|
||||
upload(client, matches.url, source.into(), dest).await?
|
||||
}
|
||||
Command::UploadDir { source, dest } => {
|
||||
let futs = Arc::new(Mutex::new(JoinSet::new()));
|
||||
upload_dir(client, Arc::clone(&futs), matches.url, source.into(), dest).await?;
|
||||
while let Some(v) = futs.lock().await.join_next().await {
|
||||
v??;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ use tracing::{error, info, instrument};
|
|||
#[instrument]
|
||||
pub async fn expire(db: &Database) -> Result<()> {
|
||||
let id = id_generator::generate();
|
||||
let oldest_acceptable_id = id - ((24 * 60 * 60) << 48);
|
||||
let oldest_acceptable_id = id - ((24 * 60 * 60 * 1_000_000_000) << 48);
|
||||
let oldest_acceptable_id = oldest_acceptable_id.to_be_bytes();
|
||||
#[expect(clippy::panic, reason = "sqlx moment")]
|
||||
query!(
|
||||
|
|
|
|||
|
|
@ -22,9 +22,11 @@ chrono = "0.4.38"
|
|||
eyre = "0.6.12"
|
||||
futures = "0.3.31"
|
||||
mime = "0.3.17"
|
||||
rusty_paseto = { version = "0.7.1", default-features = false, features = ["batteries_included", "v4_local"] }
|
||||
rusty_paseto = { version = "0.7.1", default-features = false, features = [
|
||||
"batteries_included",
|
||||
"v4_local",
|
||||
] }
|
||||
sentry-tower = { version = "0.34.0", features = ["axum", "axum-matched-path"] }
|
||||
serde = { version = "1.0.215", features = ["derive"] }
|
||||
tokio = { version = "1.41.1", features = ["fs", "net"] }
|
||||
tokio-util = { version = "0.7.13", features = ["compat"] }
|
||||
tower-http = { version = "0.6.2", features = ["trace"] }
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
use axum::{
|
||||
body::Body,
|
||||
extract::{Query, State},
|
||||
extract::State,
|
||||
http::{
|
||||
header::{ACCEPT, CACHE_CONTROL, CONTENT_LENGTH, CONTENT_TYPE, ETAG, IF_NONE_MATCH},
|
||||
HeaderMap, Request, StatusCode, Uri,
|
||||
|
|
@ -15,7 +15,6 @@ use chir_rs_misc::lexicographic_base64;
|
|||
use eyre::Context as _;
|
||||
use futures::TryStreamExt;
|
||||
use mime::MimeIter;
|
||||
use serde::Deserialize;
|
||||
use tokio_util::compat::FuturesAsyncReadCompatExt as _;
|
||||
use tracing::{debug, error, info};
|
||||
|
||||
|
|
|
|||
12
flake.lock
12
flake.lock
|
|
@ -78,11 +78,11 @@
|
|||
},
|
||||
"nixpkgs": {
|
||||
"locked": {
|
||||
"lastModified": 1733132332,
|
||||
"narHash": "sha256-8DTh3irBPko0ooHaeGJ5kPwBD8mOwiMX4eBjFuicLsA=",
|
||||
"lastModified": 1733682815,
|
||||
"narHash": "sha256-8DFHnrHprF8u95VOFgMAgJe2OBD70pd1g0NFtIEElAY=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "b9ffe0d2ef589f4d75a62f908f00dfd6344b6b6e",
|
||||
"rev": "73b9f2d20bea97d5bd6c8158f3a388ef40ac97c5",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
|
@ -128,11 +128,11 @@
|
|||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1733106880,
|
||||
"narHash": "sha256-aJmAIjZfWfPSWSExwrYBLRgXVvgF5LP1vaeUGOOIQ98=",
|
||||
"lastModified": 1733625333,
|
||||
"narHash": "sha256-tIML2axjm4AnlKP29upVJxzBpj4Cy4ak+PKonqQtXmc=",
|
||||
"owner": "oxalica",
|
||||
"repo": "rust-overlay",
|
||||
"rev": "e66c0d43abf5bdefb664c3583ca8994983c332ae",
|
||||
"rev": "430c8b054e45ea44fd2c9521a378306ada507a6c",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
|
|
|||
Loading…
Reference in a new issue