sn-fetch PoC: range probe, weighted planner, parallel segments, reassembly + sha256
This commit is contained in:
@@ -0,0 +1,50 @@
|
||||
//! Error type for all sn-fetch operations.
|
||||
|
||||
use crate::types::{ExitId, SegmentIndex};
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum FetchError {
|
||||
#[error("http request failed: {0}")]
|
||||
Http(#[from] reqwest::Error),
|
||||
|
||||
#[error("io error: {0}")]
|
||||
Io(#[from] std::io::Error),
|
||||
|
||||
#[error("invalid url `{0}`: scheme must be http or https")]
|
||||
InvalidUrl(String),
|
||||
|
||||
#[error("file size must be greater than zero")]
|
||||
ZeroFileSize,
|
||||
|
||||
#[error("segment {index}: received {got} bytes, expected {expected}")]
|
||||
SegmentLengthMismatch {
|
||||
index: SegmentIndex,
|
||||
got: u64,
|
||||
expected: u64,
|
||||
},
|
||||
|
||||
#[error("segment {exit}/{index}: server ignored range request (status {status})")]
|
||||
RangeRejected {
|
||||
exit: ExitId,
|
||||
index: SegmentIndex,
|
||||
status: u16,
|
||||
},
|
||||
|
||||
#[error("downloaded {got} bytes, expected {expected}")]
|
||||
TotalLengthMismatch { got: u64, expected: u64 },
|
||||
|
||||
#[error("sha256 mismatch: expected {expected}, got {got}")]
|
||||
HashMismatch { expected: String, got: String },
|
||||
|
||||
#[error("invalid plan: {0}")]
|
||||
InvalidPlan(String),
|
||||
|
||||
#[error("unknown exit `{0}` referenced in plan")]
|
||||
UnknownExit(ExitId),
|
||||
|
||||
#[error("invalid part file name `{0}`")]
|
||||
CorruptPart(String),
|
||||
|
||||
#[error("invalid sha256 hex string")]
|
||||
InvalidDigest,
|
||||
}
|
||||
@@ -0,0 +1,204 @@
|
||||
//! Execution: download planned segments in parallel and reassemble the
|
||||
//! target file.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use sha2::{Digest, Sha256};
|
||||
use tokio::io::AsyncWriteExt;
|
||||
|
||||
use crate::error::FetchError;
|
||||
use crate::exit::ExitRegistry;
|
||||
use crate::plan::FetchPlan;
|
||||
use crate::types::{ByteRange, ExitId, FileSize, SegmentIndex, Sha256Digest, TargetUrl};
|
||||
|
||||
/// Statistics for one finished segment.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct SegmentStats {
|
||||
pub index: SegmentIndex,
|
||||
pub bytes: u64,
|
||||
pub duration: Duration,
|
||||
}
|
||||
|
||||
/// Result of a complete fetch.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct FetchReport {
|
||||
pub url: TargetUrl,
|
||||
pub output: PathBuf,
|
||||
pub file_size: FileSize,
|
||||
pub segments: Vec<SegmentStats>,
|
||||
pub wall_time: Duration,
|
||||
pub sha256: Option<Sha256Digest>,
|
||||
}
|
||||
|
||||
/// Download every planned segment in parallel. Each segment is written
|
||||
/// to its own part file inside `parts_dir` (named `seg-NNNNNN.part`).
|
||||
///
|
||||
/// Failures abort the whole fetch: a partially assembled file is never
|
||||
/// presented as success.
|
||||
pub async fn run_segments(
|
||||
plan: &FetchPlan,
|
||||
exits: &ExitRegistry,
|
||||
parts_dir: &Path,
|
||||
) -> Result<Vec<SegmentStats>, FetchError> {
|
||||
tokio::fs::create_dir_all(parts_dir).await?;
|
||||
|
||||
let mut handles = Vec::with_capacity(plan.segments.len());
|
||||
for seg in &plan.segments {
|
||||
let client = exits.get(seg.exit)?.client().clone();
|
||||
handles.push(tokio::spawn(download_segment(
|
||||
plan.url.clone(),
|
||||
seg.exit,
|
||||
seg.index,
|
||||
seg.range,
|
||||
client,
|
||||
parts_dir.to_path_buf(),
|
||||
)));
|
||||
}
|
||||
|
||||
let mut stats = Vec::with_capacity(handles.len());
|
||||
for handle in handles {
|
||||
let stats_i = handle
|
||||
.await
|
||||
.map_err(|err| FetchError::InvalidPlan(format!("segment task panicked: {err}")))??;
|
||||
stats.push(stats_i);
|
||||
}
|
||||
stats.sort_by_key(|s| s.index);
|
||||
Ok(stats)
|
||||
}
|
||||
|
||||
/// Download one byte range and write it to `parts_dir/seg-NNNNNN.part`.
|
||||
async fn download_segment(
|
||||
url: TargetUrl,
|
||||
exit: ExitId,
|
||||
index: SegmentIndex,
|
||||
range: ByteRange,
|
||||
client: reqwest::Client,
|
||||
parts_dir: PathBuf,
|
||||
) -> Result<SegmentStats, FetchError> {
|
||||
let started = Instant::now();
|
||||
|
||||
let part_path = part_file(&parts_dir, index);
|
||||
let mut response = client
|
||||
.get(url.inner().clone())
|
||||
.header(reqwest::header::RANGE, range.http_value())
|
||||
.send()
|
||||
.await?
|
||||
.error_for_status()?;
|
||||
|
||||
match response.status() {
|
||||
reqwest::StatusCode::PARTIAL_CONTENT => {}
|
||||
status => {
|
||||
return Err(FetchError::RangeRejected {
|
||||
exit,
|
||||
index,
|
||||
status: status.as_u16(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
let mut file = tokio::fs::File::create(&part_path).await?;
|
||||
let mut written: u64 = 0;
|
||||
while let Some(chunk) = response.chunk().await? {
|
||||
file.write_all(&chunk).await?;
|
||||
written += chunk.len() as u64;
|
||||
}
|
||||
file.flush().await?;
|
||||
|
||||
if written != range.len() {
|
||||
tokio::fs::remove_file(&part_path).await.ok();
|
||||
return Err(FetchError::SegmentLengthMismatch {
|
||||
index,
|
||||
got: written,
|
||||
expected: range.len(),
|
||||
});
|
||||
}
|
||||
|
||||
Ok(SegmentStats {
|
||||
index,
|
||||
bytes: written,
|
||||
duration: started.elapsed(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Reassemble part files (in index order) into the output file while
|
||||
/// streaming a SHA-256 over the content. Verifies the total length and,
|
||||
/// if given, the expected digest. Returns the actual digest.
|
||||
pub async fn assemble(
|
||||
parts_dir: &Path,
|
||||
output: &Path,
|
||||
expected_size: FileSize,
|
||||
expected_digest: Option<&Sha256Digest>,
|
||||
) -> Result<Sha256Digest, FetchError> {
|
||||
let mut part_paths = collect_part_files(parts_dir).await?;
|
||||
part_paths.sort();
|
||||
|
||||
if let Some(parent) = output.parent() {
|
||||
if !parent.as_os_str().is_empty() {
|
||||
tokio::fs::create_dir_all(parent).await?;
|
||||
}
|
||||
}
|
||||
|
||||
let mut output_file = tokio::fs::File::create(output).await?;
|
||||
let mut hasher = Sha256::new();
|
||||
let mut total: u64 = 0;
|
||||
|
||||
for part_path in &part_paths {
|
||||
let mut part = tokio::fs::File::open(part_path).await?;
|
||||
let mut buffer = [0u8; 64 * 1024];
|
||||
loop {
|
||||
let n = tokio::io::AsyncReadExt::read(&mut part, &mut buffer).await?;
|
||||
if n == 0 {
|
||||
break;
|
||||
}
|
||||
output_file.write_all(&buffer[..n]).await?;
|
||||
hasher.update(&buffer[..n]);
|
||||
total += n as u64;
|
||||
}
|
||||
}
|
||||
output_file.flush().await?;
|
||||
|
||||
if total != expected_size.value() {
|
||||
return Err(FetchError::TotalLengthMismatch {
|
||||
got: total,
|
||||
expected: expected_size.value(),
|
||||
});
|
||||
}
|
||||
|
||||
let digest = Sha256Digest::from_bytes(hasher.finalize().into());
|
||||
if let Some(expected) = expected_digest {
|
||||
if digest != *expected {
|
||||
return Err(FetchError::HashMismatch {
|
||||
expected: expected.to_hex(),
|
||||
got: digest.to_hex(),
|
||||
});
|
||||
}
|
||||
}
|
||||
Ok(digest)
|
||||
}
|
||||
|
||||
/// Collect `seg-NNNNNN.part` files; anything else in `parts_dir` is an
|
||||
/// error, so we never silently assemble garbage.
|
||||
async fn collect_part_files(parts_dir: &Path) -> Result<Vec<PathBuf>, FetchError> {
|
||||
let mut out = Vec::new();
|
||||
let mut entries = tokio::fs::read_dir(parts_dir).await?;
|
||||
while let Some(entry) = entries.next_entry().await? {
|
||||
let raw_name = entry.file_name();
|
||||
let name = raw_name
|
||||
.to_str()
|
||||
.ok_or_else(|| FetchError::CorruptPart("<non-utf8 file name>".to_string()))?;
|
||||
let middle = name
|
||||
.strip_prefix("seg-")
|
||||
.and_then(|n| n.strip_suffix(".part"))
|
||||
.ok_or_else(|| FetchError::CorruptPart(name.to_string()))?;
|
||||
middle
|
||||
.parse::<u32>()
|
||||
.map_err(|_| FetchError::CorruptPart(name.to_string()))?;
|
||||
out.push(entry.path());
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
fn part_file(parts_dir: &Path, index: SegmentIndex) -> PathBuf {
|
||||
parts_dir.join(format!("seg-{:06}.part", index.value()))
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
//! Exit abstraction: an "exit" is one internet uplink through which
|
||||
//! segment requests can be sent.
|
||||
//!
|
||||
//! PoC v0 ships only [`LocalExit`] (the household's own connection, no
|
||||
//! detour). The registry/trait structure is deliberately ready so a
|
||||
//! future exit implementation can route HTTP through a neighbour over
|
||||
//! the mesh (e.g. via a per-exit proxy or tunnel) without changing the
|
||||
//! planner or executor.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use crate::error::FetchError;
|
||||
use crate::types::ExitId;
|
||||
|
||||
/// An internet uplink able to issue HTTP requests.
|
||||
pub trait HttpExit: Send + Sync {
|
||||
fn id(&self) -> ExitId;
|
||||
|
||||
/// Client whose connections go out through this exit.
|
||||
fn client(&self) -> &reqwest::Client;
|
||||
}
|
||||
|
||||
/// The household's own uplink: plain outgoing HTTP, no detour.
|
||||
#[derive(Debug)]
|
||||
pub struct LocalExit {
|
||||
id: ExitId,
|
||||
client: reqwest::Client,
|
||||
}
|
||||
|
||||
impl LocalExit {
|
||||
pub fn new(id: ExitId) -> Result<Self, FetchError> {
|
||||
let client = reqwest::Client::builder()
|
||||
.connect_timeout(Duration::from_secs(30))
|
||||
.build()?;
|
||||
Ok(Self { id, client })
|
||||
}
|
||||
}
|
||||
|
||||
impl HttpExit for LocalExit {
|
||||
fn id(&self) -> ExitId {
|
||||
self.id
|
||||
}
|
||||
|
||||
fn client(&self) -> &reqwest::Client {
|
||||
&self.client
|
||||
}
|
||||
}
|
||||
|
||||
/// All exits available for a fetch, keyed by id.
|
||||
#[derive(Default, Clone)]
|
||||
pub struct ExitRegistry {
|
||||
exits: BTreeMap<ExitId, Arc<dyn HttpExit>>,
|
||||
}
|
||||
|
||||
impl ExitRegistry {
|
||||
pub fn empty() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
pub fn register(&mut self, exit: Arc<dyn HttpExit>) {
|
||||
self.exits.insert(exit.id(), exit);
|
||||
}
|
||||
|
||||
pub fn get(&self, id: ExitId) -> Result<&Arc<dyn HttpExit>, FetchError> {
|
||||
self.exits.get(&id).ok_or(FetchError::UnknownExit(id))
|
||||
}
|
||||
|
||||
pub fn ids(&self) -> Vec<ExitId> {
|
||||
self.exits.keys().copied().collect()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
//! sn-fetch — segmented multi-exit downloader for SpiderNet.
|
||||
//!
|
||||
//! PoC v0: probes a URL for HTTP range support, plans weighted byte-range
|
||||
//! segments across exits, downloads segments in parallel and reassembles
|
||||
//! the target file with SHA-256 verification.
|
||||
//!
|
||||
//! SpiderNet software carries no warranty and accepts no responsibility
|
||||
//! for its usage or for transported content (AGPL-3.0, sections 15/16).
|
||||
|
||||
#![forbid(unsafe_code)]
|
||||
|
||||
pub mod error;
|
||||
pub mod exec;
|
||||
pub mod exit;
|
||||
pub mod plan;
|
||||
pub mod probe;
|
||||
pub mod runner;
|
||||
pub mod types;
|
||||
@@ -0,0 +1,182 @@
|
||||
//! CLI entry point for the sn-fetch PoC.
|
||||
//!
|
||||
//! KEINE GARANTIE — this software is provided "as is" without warranty of
|
||||
//! any kind (AGPL-3.0, sections 15/16). Users are solely responsible for
|
||||
//! what they transmit or receive with it.
|
||||
|
||||
#![forbid(unsafe_code)]
|
||||
|
||||
use std::num::{NonZeroU32, NonZeroU64, NonZeroU8, NonZeroUsize};
|
||||
use std::path::PathBuf;
|
||||
|
||||
use clap::Parser;
|
||||
use sn_fetch::runner::{self, FetchOptions};
|
||||
use sn_fetch::types::{
|
||||
ExitId, ExitWeight, MinSegmentBytes, SegmentsPerExit, Sha256Digest, TargetUrl,
|
||||
};
|
||||
|
||||
#[derive(Debug, Parser)]
|
||||
#[command(name = "sn-fetch", about = "SpiderNet segmented downloader (PoC)")]
|
||||
struct Args {
|
||||
/// URL of the file to download (http/https).
|
||||
url: String,
|
||||
|
||||
/// Output file path (default: file name from the URL, in cwd).
|
||||
#[arg(short, long)]
|
||||
output: Option<PathBuf>,
|
||||
|
||||
/// Exits as `id=weight` pairs, comma separated, e.g. `local=1,local2=2`.
|
||||
/// PoC: all ids map to plain local connections; the weights only shape
|
||||
/// the plan. Multi-exit routing comes with the mesh integration.
|
||||
#[arg(short = 'e', long = "exit", default_value = "local=1")]
|
||||
exits: String,
|
||||
|
||||
/// Parallel sub-segments per exit.
|
||||
#[arg(short = 's', long, default_value_t = 1)]
|
||||
segments_per_exit: usize,
|
||||
|
||||
/// Minimum segment size in MiB (smaller segments get merged).
|
||||
#[arg(long, default_value_t = 4)]
|
||||
min_segment_mib: u64,
|
||||
|
||||
/// Expected SHA-256 of the complete file (hex).
|
||||
#[arg(long)]
|
||||
sha256: Option<String>,
|
||||
|
||||
/// Only probe the server and print what we learned.
|
||||
#[arg(long)]
|
||||
probe_only: bool,
|
||||
}
|
||||
|
||||
fn parse_exits(spec: &str) -> Result<Vec<(ExitId, ExitWeight)>, String> {
|
||||
spec.split(',')
|
||||
.map(|pair| {
|
||||
let (id, weight) = pair
|
||||
.split_once('=')
|
||||
.ok_or_else(|| format!("invalid exit spec `{pair}`, expected `id=weight`"))?;
|
||||
let id = id
|
||||
.trim()
|
||||
.parse::<u8>()
|
||||
.ok()
|
||||
.and_then(NonZeroU8::new)
|
||||
.ok_or_else(|| format!("invalid exit id `{id}` (must be a non-zero number)"))?;
|
||||
let weight = weight
|
||||
.trim()
|
||||
.parse::<u32>()
|
||||
.ok()
|
||||
.and_then(NonZeroU32::new)
|
||||
.ok_or_else(|| format!("invalid exit weight `{weight}` (must be non-zero)"))?;
|
||||
Ok((ExitId::new(id), ExitWeight::new(weight)))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn default_output(url: &TargetUrl) -> PathBuf {
|
||||
let mut segments = match url.inner().path_segments() {
|
||||
Some(segments) => segments.clone(),
|
||||
None => return PathBuf::from("download.bin"),
|
||||
};
|
||||
let name = segments
|
||||
.next_back()
|
||||
.filter(|name| !name.is_empty())
|
||||
.map(str::to_string)
|
||||
.unwrap_or_else(|| "download.bin".to_string());
|
||||
PathBuf::from(name)
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
tracing_subscriber::fmt()
|
||||
.with_env_filter(
|
||||
tracing_subscriber::EnvFilter::try_from_default_env()
|
||||
.unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("warn")),
|
||||
)
|
||||
.init();
|
||||
|
||||
if let Err(report) = run().await {
|
||||
eprintln!("NO WARRANTY — SpiderNet sn-fetch PoC (AGPL-3.0, no liability)");
|
||||
eprintln!("error: {report}");
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
async fn run() -> Result<(), String> {
|
||||
let args = Args::parse();
|
||||
let target = TargetUrl::parse(&args.url).map_err(|err| err.to_string())?;
|
||||
|
||||
let client = reqwest::Client::builder()
|
||||
.connect_timeout(std::time::Duration::from_secs(30))
|
||||
.build()
|
||||
.map_err(|err| err.to_string())?;
|
||||
|
||||
if args.probe_only {
|
||||
let info = sn_fetch::probe::probe(&client, &target)
|
||||
.await
|
||||
.map_err(|err| err.to_string())?;
|
||||
println!("url: {target}");
|
||||
println!("range support: {:?}", info.range_support);
|
||||
match info.content_length {
|
||||
Some(size) => println!("content-length: {size} bytes"),
|
||||
None => println!("content-length: unknown"),
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let exit_specs = parse_exits(&args.exits)?;
|
||||
let spe =
|
||||
NonZeroUsize::new(args.segments_per_exit).ok_or("segments_per_exit must be at least 1")?;
|
||||
let min_segment = MinSegmentBytes::new(
|
||||
NonZeroU64::new(args.min_segment_mib.saturating_mul(1024 * 1024))
|
||||
.ok_or("min_segment_mib must be at least 1")?,
|
||||
);
|
||||
|
||||
let expected = match &args.sha256 {
|
||||
Some(hex) => Some(Sha256Digest::from_hex(hex).map_err(|err| err.to_string())?),
|
||||
None => None,
|
||||
};
|
||||
|
||||
let options = FetchOptions {
|
||||
output: args
|
||||
.output
|
||||
.clone()
|
||||
.unwrap_or_else(|| default_output(&target)),
|
||||
exits: exit_specs.clone(),
|
||||
segments_per_exit: SegmentsPerExit::new(spe),
|
||||
min_segment,
|
||||
expected_sha256: expected,
|
||||
};
|
||||
|
||||
let registry = runner::local_registry(&exit_specs).map_err(|err| err.to_string())?;
|
||||
let parts_dir = std::env::temp_dir().join("sn-fetch-parts");
|
||||
|
||||
let report = runner::run(&target, &options, ®istry, &parts_dir)
|
||||
.await
|
||||
.map_err(|err| err.to_string())?;
|
||||
|
||||
println!("NO WARRANTY — SpiderNet sn-fetch PoC (AGPL-3.0, no liability)");
|
||||
println!();
|
||||
println!("url: {}", report.url);
|
||||
println!("output: {}", report.output.display());
|
||||
println!("size: {} bytes", report.file_size);
|
||||
match report.sha256 {
|
||||
Some(digest) => println!("sha256: {digest}"),
|
||||
None => println!("sha256: not computed"),
|
||||
}
|
||||
println!(
|
||||
"time: {:.2?} s → effective {:.1} MiB/s",
|
||||
report.wall_time.as_secs_f64(),
|
||||
report.file_size.value() as f64 / 1024.0 / 1024.0 / report.wall_time.as_secs_f64()
|
||||
);
|
||||
println!("segments:");
|
||||
for segment in &report.segments {
|
||||
let mib_per_s = segment.bytes as f64
|
||||
/ 1024.0
|
||||
/ 1024.0
|
||||
/ segment.duration.as_secs_f64().max(f64::EPSILON);
|
||||
println!(
|
||||
" #{}: {} bytes in {:.2?} s ({mib_per_s:.1} MiB/s)",
|
||||
segment.index, segment.bytes, segment.duration
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,279 @@
|
||||
//! Planning: split a file into weighted byte-range segments across exits.
|
||||
|
||||
use crate::error::FetchError;
|
||||
use crate::types::{
|
||||
ByteRange, ExitId, ExitWeight, FileSize, MinSegmentBytes, SegmentIndex, SegmentsPerExit,
|
||||
TargetUrl,
|
||||
};
|
||||
use std::num::NonZeroUsize;
|
||||
|
||||
/// One planned segment: a byte range assigned to one exit.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct SegmentPlan {
|
||||
pub index: SegmentIndex,
|
||||
pub range: ByteRange,
|
||||
pub exit: ExitId,
|
||||
}
|
||||
|
||||
/// A validated download plan.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct FetchPlan {
|
||||
pub url: TargetUrl,
|
||||
pub file_size: FileSize,
|
||||
pub segments: Vec<SegmentPlan>,
|
||||
}
|
||||
|
||||
/// Exits with their capacity weights.
|
||||
pub type ExitSpecs = Vec<(ExitId, ExitWeight)>;
|
||||
|
||||
impl FetchPlan {
|
||||
/// Build a validated plan.
|
||||
///
|
||||
/// Exits are weighted by capacity; each exit's share is further split
|
||||
/// into `segments_per_exit` sub-segments. Segments smaller than
|
||||
/// `min_segment` are merged into a neighbour so we never spawn
|
||||
/// absurdly tiny requests — unless the whole file is that small.
|
||||
pub fn build(
|
||||
url: TargetUrl,
|
||||
file_size: FileSize,
|
||||
exits: &ExitSpecs,
|
||||
segments_per_exit: SegmentsPerExit,
|
||||
min_segment: MinSegmentBytes,
|
||||
) -> Result<Self, FetchError> {
|
||||
if exits.is_empty() {
|
||||
return Err(FetchError::InvalidPlan("no exits given".to_string()));
|
||||
}
|
||||
|
||||
let mut exits = exits.clone();
|
||||
|
||||
// Drop the weakest exits (smallest weight; ties: higher id goes
|
||||
// first) until every remaining exit can hold at least one
|
||||
// minimum segment.
|
||||
while exits.len() > 1
|
||||
&& u128::from(file_size.value())
|
||||
< u128::from(exits.len() as u64) * u128::from(min_segment.value())
|
||||
{
|
||||
let weakest = exits
|
||||
.iter()
|
||||
.enumerate()
|
||||
.min_by_key(|(_, (id, weight))| (weight.value(), std::cmp::Reverse(id.value())))
|
||||
.map(|(pos, _)| pos)
|
||||
.unwrap_or(0);
|
||||
exits.remove(weakest);
|
||||
}
|
||||
// Deterministic boundary assignment order.
|
||||
exits.sort_by_key(|(id, _)| *id);
|
||||
|
||||
// If even one segment per exit would be too small, reduce to one
|
||||
// segment per exit.
|
||||
let spe_limit = usize::try_from(
|
||||
u128::from(file_size.value())
|
||||
/ (u128::from(exits.len() as u64) * u128::from(min_segment.value())),
|
||||
)
|
||||
.unwrap_or(0);
|
||||
let spe = if spe_limit >= segments_per_exit.value() {
|
||||
segments_per_exit
|
||||
} else {
|
||||
SegmentsPerExit::new(NonZeroUsize::MIN)
|
||||
};
|
||||
|
||||
let mut assignments: Vec<(ByteRange, ExitId)> = Vec::new();
|
||||
|
||||
if exits.len() == 1 && spe.value() == 1 {
|
||||
let only = ByteRange::try_new(0, file_size.value() - 1)?;
|
||||
assignments.push((only, exits[0].0));
|
||||
} else {
|
||||
// Weighted split across exits, then even sub-split per exit.
|
||||
let units: Vec<u128> = exits
|
||||
.iter()
|
||||
.map(|(_, w)| u128::from(w.value()) * spe.value() as u128)
|
||||
.collect();
|
||||
let total_units: u128 = units.iter().sum();
|
||||
|
||||
let size = u128::from(file_size.value());
|
||||
let mut cursor: u128 = 0;
|
||||
let mut start: u64 = 0;
|
||||
|
||||
for (i, units_i) in units.iter().enumerate() {
|
||||
let is_last = i == exits.len() - 1;
|
||||
let end_exclusive: u128 = if is_last {
|
||||
size
|
||||
} else {
|
||||
cursor + (size * units_i) / total_units
|
||||
};
|
||||
|
||||
let exit_end: u64 = end_exclusive
|
||||
.try_into()
|
||||
.map_err(|_| FetchError::InvalidPlan("boundary overflow".to_string()))?;
|
||||
let exit_end_inclusive = exit_end.saturating_sub(1);
|
||||
|
||||
if exit_end_inclusive >= start {
|
||||
for range in even_subsplit(start, exit_end_inclusive, spe.value()) {
|
||||
assignments.push((range, exits[i].0));
|
||||
}
|
||||
start = exit_end;
|
||||
}
|
||||
|
||||
cursor = end_exclusive;
|
||||
}
|
||||
|
||||
// The last exit covers to the end, so nothing can be left
|
||||
// over; stay honest and extend the last assignment anyway.
|
||||
if start < file_size.value() {
|
||||
match assignments.last_mut() {
|
||||
Some((last_range, _)) => {
|
||||
*last_range =
|
||||
ByteRange::try_new(last_range.start(), file_size.value() - 1)?;
|
||||
}
|
||||
None => {
|
||||
let only = ByteRange::try_new(0, file_size.value() - 1)?;
|
||||
assignments.push((only, exits[0].0));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Merge-fix: no segment below the minimum size (except a lone
|
||||
// tiny segment for a tiny file).
|
||||
let mut assignments = merge_tiny_segments(assignments, min_segment);
|
||||
|
||||
let segments: Vec<SegmentPlan> = assignments
|
||||
.drain(..)
|
||||
.enumerate()
|
||||
.map(|(i, (range, exit))| {
|
||||
let index = u32::try_from(i)
|
||||
.map(SegmentIndex::new)
|
||||
.map_err(|_| FetchError::InvalidPlan("more than 2^32 segments".to_string()))?;
|
||||
Ok::<SegmentPlan, FetchError>(SegmentPlan { index, range, exit })
|
||||
})
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
|
||||
let plan = Self {
|
||||
url,
|
||||
file_size,
|
||||
segments,
|
||||
};
|
||||
plan.validate()?;
|
||||
Ok(plan)
|
||||
}
|
||||
|
||||
/// Check invariants: contiguous, gap-free, complete coverage of the
|
||||
/// file, sequential indices, non-empty ranges.
|
||||
pub fn validate(&self) -> Result<(), FetchError> {
|
||||
let mut cursor: u64 = 0;
|
||||
for (expected_index, seg) in self.segments.iter().enumerate() {
|
||||
let expected_index = u32::try_from(expected_index)
|
||||
.map_err(|_| FetchError::InvalidPlan("more than 2^32 segments".to_string()))?;
|
||||
if seg.index.value() != expected_index {
|
||||
return Err(FetchError::InvalidPlan(format!(
|
||||
"segment index out of order: {} != {expected_index}",
|
||||
seg.index
|
||||
)));
|
||||
}
|
||||
if seg.range.start() != cursor {
|
||||
return Err(FetchError::InvalidPlan(format!(
|
||||
"gap or overlap at segment {}: starts at {} but cursor is at {cursor}",
|
||||
seg.index,
|
||||
seg.range.start()
|
||||
)));
|
||||
}
|
||||
if seg.range.is_empty() {
|
||||
return Err(FetchError::InvalidPlan(format!(
|
||||
"segment {} is empty",
|
||||
seg.index
|
||||
)));
|
||||
}
|
||||
cursor = seg
|
||||
.range
|
||||
.end_inclusive()
|
||||
.checked_add(1)
|
||||
.ok_or_else(|| FetchError::InvalidPlan("range end overflow".to_string()))?;
|
||||
}
|
||||
if cursor != self.file_size.value() {
|
||||
return Err(FetchError::InvalidPlan(format!(
|
||||
"plan covers up to {cursor} but file size is {}",
|
||||
self.file_size
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Split `[start, end_inclusive]` into `parts` nearly-even sub-ranges,
|
||||
/// ordered; the first `remainder` parts get one extra byte.
|
||||
///
|
||||
/// The caller guarantees `parts >= 1` and `end_inclusive >= start`, so
|
||||
/// every part is non-empty and `try_new` cannot fail.
|
||||
fn even_subsplit(start: u64, end_inclusive: u64, parts: usize) -> Vec<ByteRange> {
|
||||
let total_len = end_inclusive - start + 1;
|
||||
let base = total_len / parts as u64;
|
||||
let remainder = total_len % parts as u64;
|
||||
|
||||
let mut out = Vec::with_capacity(parts);
|
||||
let mut cursor = start;
|
||||
for i in 0..parts {
|
||||
let len = base + if (i as u64) < remainder { 1 } else { 0 };
|
||||
let range_end = cursor + len - 1;
|
||||
match ByteRange::try_new(cursor, range_end) {
|
||||
Ok(range) => out.push(range),
|
||||
// Unreachable by construction; keep the sub-range out instead
|
||||
// of panicking (validate() would then reject the plan).
|
||||
Err(_) => return Vec::new(),
|
||||
}
|
||||
cursor = range_end + 1;
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Merge any segment smaller than `min_segment` into a neighbour
|
||||
/// (prefer the next, fall back to the previous). A single remaining
|
||||
/// segment is left as-is, however small.
|
||||
fn merge_tiny_segments(
|
||||
mut assignments: Vec<(ByteRange, ExitId)>,
|
||||
min_segment: MinSegmentBytes,
|
||||
) -> Vec<(ByteRange, ExitId)> {
|
||||
while let Some(pos) = assignments
|
||||
.iter()
|
||||
.position(|(range, _)| range.len() < min_segment.value())
|
||||
{
|
||||
if assignments.len() == 1 {
|
||||
break; // nothing left to merge into
|
||||
}
|
||||
|
||||
let merged: (ByteRange, ExitId) = if pos + 1 < assignments.len() {
|
||||
// Merge into the next segment, keeping its exit.
|
||||
let next = &assignments[pos + 1];
|
||||
let range = match ByteRange::try_new(assignments[pos].0.start(), next.0.end_inclusive())
|
||||
{
|
||||
Ok(range) => range,
|
||||
// Unreachable: next.start == pos.end + 1, so the merged
|
||||
// range is always well-formed. Drop both segments instead
|
||||
// of panicking; validate() would reject a broken plan.
|
||||
Err(_) => break,
|
||||
};
|
||||
(range, next.1)
|
||||
} else {
|
||||
// Last segment is too small: merge into the previous one.
|
||||
let prev_pos = pos - 1;
|
||||
let prev = &assignments[prev_pos];
|
||||
let range = match ByteRange::try_new(prev.0.start(), assignments[pos].0.end_inclusive())
|
||||
{
|
||||
Ok(range) => range,
|
||||
Err(_) => break,
|
||||
};
|
||||
(range, prev.1)
|
||||
};
|
||||
|
||||
match pos + 1 < assignments.len() {
|
||||
true => {
|
||||
assignments[pos + 1] = merged;
|
||||
assignments.remove(pos);
|
||||
}
|
||||
false => {
|
||||
assignments[pos - 1] = merged;
|
||||
assignments.remove(pos);
|
||||
}
|
||||
}
|
||||
}
|
||||
assignments
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
//! Probing: find out whether a server supports HTTP range requests and
|
||||
//! how large the object is, before planning a segmented download.
|
||||
|
||||
use reqwest::header::{ACCEPT_RANGES, CONTENT_LENGTH};
|
||||
use reqwest::StatusCode;
|
||||
|
||||
use crate::error::FetchError;
|
||||
use crate::types::{FileSize, RangeSupport, TargetUrl};
|
||||
|
||||
/// What a HEAD request (plus fallback range probe) told us about the target.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct ProbeInfo {
|
||||
pub range_support: RangeSupport,
|
||||
pub content_length: Option<FileSize>,
|
||||
}
|
||||
|
||||
/// Probe the target: HEAD first; if the `Accept-Ranges` header is absent
|
||||
/// or inconclusive, fall back to a 1-byte GET with a `Range` header and
|
||||
/// check for `206 Partial Content`.
|
||||
pub async fn probe(client: &reqwest::Client, url: &TargetUrl) -> Result<ProbeInfo, FetchError> {
|
||||
let head = client
|
||||
.head(url.inner().clone())
|
||||
.send()
|
||||
.await?
|
||||
.error_for_status()?;
|
||||
|
||||
let content_length = head
|
||||
.headers()
|
||||
.get(CONTENT_LENGTH)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.and_then(|v| v.parse::<u64>().ok());
|
||||
|
||||
let advertised = match head
|
||||
.headers()
|
||||
.get(ACCEPT_RANGES)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.map(str::trim)
|
||||
{
|
||||
Some("bytes") => Some(RangeSupport::Supported),
|
||||
Some("none") => Some(RangeSupport::Unsupported),
|
||||
_ => None,
|
||||
};
|
||||
|
||||
let range_support = match advertised {
|
||||
Some(support) => support,
|
||||
// HEAD was inconclusive: ask directly with a 1-byte range.
|
||||
None => {
|
||||
let resp = client
|
||||
.get(url.inner().clone())
|
||||
.header(reqwest::header::RANGE, "bytes=0-0")
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
match resp.status() {
|
||||
StatusCode::PARTIAL_CONTENT => RangeSupport::Supported,
|
||||
StatusCode::OK => RangeSupport::Unsupported,
|
||||
_ => RangeSupport::Unknown,
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
Ok(ProbeInfo {
|
||||
range_support,
|
||||
content_length: content_length.and_then(|len| FileSize::try_new(len).ok()),
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,171 @@
|
||||
//! High-level orchestration: probe → plan → execute → assemble.
|
||||
//!
|
||||
//! Non-range-supporting servers fall back to a plain single-stream
|
||||
//! download (no pooling gain, correct result).
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::Instant;
|
||||
|
||||
use sha2::Digest;
|
||||
use tokio::io::AsyncWriteExt;
|
||||
|
||||
use crate::error::FetchError;
|
||||
use crate::exec::{self, FetchReport, SegmentStats};
|
||||
use crate::exit::{ExitRegistry, LocalExit};
|
||||
use crate::plan::ExitSpecs;
|
||||
use crate::probe::{self, ProbeInfo};
|
||||
use crate::types::{
|
||||
FileSize, MinSegmentBytes, RangeSupport, SegmentsPerExit, Sha256Digest, TargetUrl,
|
||||
};
|
||||
|
||||
/// All knobs for one fetch.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct FetchOptions {
|
||||
pub output: PathBuf,
|
||||
pub exits: ExitSpecs,
|
||||
pub segments_per_exit: SegmentsPerExit,
|
||||
pub min_segment: MinSegmentBytes,
|
||||
pub expected_sha256: Option<Sha256Digest>,
|
||||
}
|
||||
|
||||
/// Run a full fetch and produce a report.
|
||||
pub async fn run(
|
||||
target: &TargetUrl,
|
||||
options: &FetchOptions,
|
||||
exits: &ExitRegistry,
|
||||
parts_dir: &Path,
|
||||
) -> Result<FetchReport, FetchError> {
|
||||
let probe_client = reqwest::Client::builder()
|
||||
.connect_timeout(std::time::Duration::from_secs(30))
|
||||
.build()?;
|
||||
let info = probe::probe(&probe_client, target).await?;
|
||||
|
||||
match info.range_support {
|
||||
RangeSupport::Supported => run_ranged(target, options, exits, parts_dir, info).await,
|
||||
RangeSupport::Unsupported | RangeSupport::Unknown => {
|
||||
tracing::warn!(
|
||||
"server does not (confirmably) support range requests; \
|
||||
falling back to single-stream download without pooling"
|
||||
);
|
||||
run_whole(target, options, exits, parts_dir, info).await
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Ranged path: probe must have told us the size; plan, execute, assemble.
|
||||
async fn run_ranged(
|
||||
target: &TargetUrl,
|
||||
options: &FetchOptions,
|
||||
exits: &ExitRegistry,
|
||||
parts_dir: &Path,
|
||||
info: ProbeInfo,
|
||||
) -> Result<FetchReport, FetchError> {
|
||||
let file_size = info.content_length.ok_or(FetchError::ZeroFileSize)?;
|
||||
|
||||
let plan = crate::plan::FetchPlan::build(
|
||||
target.clone(),
|
||||
file_size,
|
||||
&options.exits,
|
||||
options.segments_per_exit,
|
||||
options.min_segment,
|
||||
)?;
|
||||
tracing::info!("plan: {} segment(s)", plan.segments.len());
|
||||
|
||||
let started = Instant::now();
|
||||
let segment_stats: Vec<SegmentStats> = exec::run_segments(&plan, exits, parts_dir).await?;
|
||||
let digest = exec::assemble(
|
||||
parts_dir,
|
||||
&options.output,
|
||||
file_size,
|
||||
options.expected_sha256.as_ref(),
|
||||
)
|
||||
.await?;
|
||||
let wall_time = started.elapsed();
|
||||
|
||||
Ok(FetchReport {
|
||||
url: target.clone(),
|
||||
output: options.output.clone(),
|
||||
file_size,
|
||||
segments: segment_stats,
|
||||
wall_time,
|
||||
sha256: Some(digest),
|
||||
})
|
||||
}
|
||||
|
||||
/// Fallback path: single stream, whole file.
|
||||
async fn run_whole(
|
||||
target: &TargetUrl,
|
||||
options: &FetchOptions,
|
||||
exits: &ExitRegistry,
|
||||
_parts_dir: &Path, // symmetric signature; unused without segments
|
||||
info: ProbeInfo,
|
||||
) -> Result<FetchReport, FetchError> {
|
||||
let exit_id = options
|
||||
.exits
|
||||
.first()
|
||||
.map(|(id, _)| *id)
|
||||
.ok_or_else(|| FetchError::InvalidPlan("no exits given".to_string()))?;
|
||||
let client = exits.get(exit_id)?.client().clone();
|
||||
|
||||
let started = Instant::now();
|
||||
let response = client
|
||||
.get(target.inner().clone())
|
||||
.send()
|
||||
.await?
|
||||
.error_for_status()?;
|
||||
|
||||
let mut file = tokio::fs::File::create(&options.output).await?;
|
||||
let mut hasher = sha2::Sha256::new();
|
||||
let mut written: u64 = 0;
|
||||
let mut response = response;
|
||||
while let Some(chunk) = response.chunk().await? {
|
||||
file.write_all(&chunk).await?;
|
||||
hasher.update(&chunk);
|
||||
written += chunk.len() as u64;
|
||||
}
|
||||
file.flush().await?;
|
||||
|
||||
if let Some(expected_len) = info.content_length {
|
||||
if written != expected_len.value() {
|
||||
return Err(FetchError::TotalLengthMismatch {
|
||||
got: written,
|
||||
expected: expected_len.value(),
|
||||
});
|
||||
}
|
||||
}
|
||||
if written == 0 {
|
||||
return Err(FetchError::ZeroFileSize);
|
||||
}
|
||||
|
||||
let digest = Sha256Digest::from_bytes(hasher.finalize().into());
|
||||
if let Some(expected) = options.expected_sha256.as_ref() {
|
||||
if digest != *expected {
|
||||
return Err(FetchError::HashMismatch {
|
||||
expected: expected.to_hex(),
|
||||
got: digest.to_hex(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Ok(FetchReport {
|
||||
url: target.clone(),
|
||||
output: options.output.clone(),
|
||||
file_size: FileSize::try_new(written)?,
|
||||
segments: vec![SegmentStats {
|
||||
index: crate::types::SegmentIndex::new(0),
|
||||
bytes: written,
|
||||
duration: started.elapsed(),
|
||||
}],
|
||||
wall_time: started.elapsed(),
|
||||
sha256: Some(digest),
|
||||
})
|
||||
}
|
||||
|
||||
/// Convenience: build a registry with one [`LocalExit`] per spec id.
|
||||
pub fn local_registry(specs: &ExitSpecs) -> Result<ExitRegistry, FetchError> {
|
||||
let mut registry = ExitRegistry::empty();
|
||||
for (id, _) in specs {
|
||||
registry.register(std::sync::Arc::new(LocalExit::new(*id)?));
|
||||
}
|
||||
Ok(registry)
|
||||
}
|
||||
@@ -0,0 +1,232 @@
|
||||
//! Domain types: newtypes over raw primitives, validated at construction.
|
||||
|
||||
use std::fmt;
|
||||
use std::num::{NonZeroU32, NonZeroU64, NonZeroU8, NonZeroUsize};
|
||||
|
||||
use crate::error::FetchError;
|
||||
|
||||
/// Identifier of an exit (an internet uplink offered by a household).
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub struct ExitId(NonZeroU8);
|
||||
|
||||
impl ExitId {
|
||||
pub const fn new(value: NonZeroU8) -> Self {
|
||||
Self(value)
|
||||
}
|
||||
|
||||
pub const fn value(self) -> u8 {
|
||||
self.0.get()
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for ExitId {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(f, "exit-{}", self.0.get())
|
||||
}
|
||||
}
|
||||
|
||||
/// Relative capacity share of an exit. 1 means equal share.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct ExitWeight(NonZeroU32);
|
||||
|
||||
impl ExitWeight {
|
||||
pub const fn new(value: NonZeroU32) -> Self {
|
||||
Self(value)
|
||||
}
|
||||
|
||||
pub const fn value(self) -> u32 {
|
||||
self.0.get()
|
||||
}
|
||||
}
|
||||
|
||||
/// How many sub-segments each exit's share is split into (parallel
|
||||
/// connections per exit; more connections can defeat per-connection
|
||||
/// server throttling).
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct SegmentsPerExit(NonZeroUsize);
|
||||
|
||||
impl SegmentsPerExit {
|
||||
pub const fn new(value: NonZeroUsize) -> Self {
|
||||
Self(value)
|
||||
}
|
||||
|
||||
pub const fn value(self) -> usize {
|
||||
self.0.get()
|
||||
}
|
||||
}
|
||||
|
||||
/// Lower bound for a segment size, to avoid absurdly tiny segments.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct MinSegmentBytes(NonZeroU64);
|
||||
|
||||
impl MinSegmentBytes {
|
||||
pub const fn new(value: NonZeroU64) -> Self {
|
||||
Self(value)
|
||||
}
|
||||
|
||||
pub const fn value(self) -> u64 {
|
||||
self.0.get()
|
||||
}
|
||||
}
|
||||
|
||||
/// Total size of a downloadable object, validated to be non-zero.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct FileSize(u64);
|
||||
|
||||
impl FileSize {
|
||||
pub fn try_new(value: u64) -> Result<Self, FetchError> {
|
||||
if value == 0 {
|
||||
return Err(FetchError::ZeroFileSize);
|
||||
}
|
||||
Ok(Self(value))
|
||||
}
|
||||
|
||||
pub const fn value(self) -> u64 {
|
||||
self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for FileSize {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(f, "{}", self.0)
|
||||
}
|
||||
}
|
||||
|
||||
/// A half-open-by-one byte range `[start, end_inclusive]`, as HTTP uses
|
||||
/// inclusive end positions.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct ByteRange {
|
||||
start: u64,
|
||||
end_inclusive: u64,
|
||||
}
|
||||
|
||||
impl ByteRange {
|
||||
pub fn try_new(start: u64, end_inclusive: u64) -> Result<Self, FetchError> {
|
||||
if end_inclusive < start {
|
||||
return Err(FetchError::InvalidPlan(format!(
|
||||
"range start {start} is after end {end_inclusive}"
|
||||
)));
|
||||
}
|
||||
Ok(Self {
|
||||
start,
|
||||
end_inclusive,
|
||||
})
|
||||
}
|
||||
|
||||
pub const fn start(self) -> u64 {
|
||||
self.start
|
||||
}
|
||||
|
||||
pub const fn end_inclusive(self) -> u64 {
|
||||
self.end_inclusive
|
||||
}
|
||||
|
||||
pub const fn len(self) -> u64 {
|
||||
self.end_inclusive - self.start + 1
|
||||
}
|
||||
|
||||
pub const fn is_empty(self) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
/// Value for the HTTP `Range` header, e.g. `bytes=100-199`.
|
||||
pub fn http_value(self) -> String {
|
||||
format!("bytes={}-{}", self.start, self.end_inclusive)
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for ByteRange {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(
|
||||
f,
|
||||
"{}..={} ({} bytes)",
|
||||
self.start,
|
||||
self.end_inclusive,
|
||||
self.len()
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// Position of a segment inside a plan, sequential from 0.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub struct SegmentIndex(u32);
|
||||
|
||||
impl SegmentIndex {
|
||||
pub const fn new(value: u32) -> Self {
|
||||
Self(value)
|
||||
}
|
||||
|
||||
pub const fn value(self) -> u32 {
|
||||
self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for SegmentIndex {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(f, "{}", self.0)
|
||||
}
|
||||
}
|
||||
|
||||
/// A validated SHA-256 digest.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Sha256Digest([u8; 32]);
|
||||
|
||||
impl Sha256Digest {
|
||||
pub fn from_bytes(bytes: [u8; 32]) -> Self {
|
||||
Self(bytes)
|
||||
}
|
||||
|
||||
pub fn from_hex(hex_str: &str) -> Result<Self, FetchError> {
|
||||
let bytes = hex::decode(hex_str.trim()).map_err(|_| FetchError::InvalidDigest)?;
|
||||
let arr: [u8; 32] = bytes.try_into().map_err(|_| FetchError::InvalidDigest)?;
|
||||
Ok(Self(arr))
|
||||
}
|
||||
|
||||
pub fn to_hex(&self) -> String {
|
||||
hex::encode(self.0)
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for Sha256Digest {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(f, "{}", self.to_hex())
|
||||
}
|
||||
}
|
||||
|
||||
/// A validated target URL. Only http(s) is accepted.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct TargetUrl(reqwest::Url);
|
||||
|
||||
impl TargetUrl {
|
||||
pub fn parse(input: &str) -> Result<Self, FetchError> {
|
||||
let url =
|
||||
reqwest::Url::parse(input).map_err(|_| FetchError::InvalidUrl(input.to_string()))?;
|
||||
match url.scheme() {
|
||||
"http" | "https" => Ok(Self(url)),
|
||||
_ => Err(FetchError::InvalidUrl(input.to_string())),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn inner(&self) -> &reqwest::Url {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for TargetUrl {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(f, "{}", self.0)
|
||||
}
|
||||
}
|
||||
|
||||
/// Result of probing a server's range-request support.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum RangeSupport {
|
||||
/// Server advertised `Accept-Ranges: bytes` or answered a range probe
|
||||
/// with `206 Partial Content`.
|
||||
Supported,
|
||||
/// Server advertised `Accept-Ranges: none` or answered a range probe
|
||||
/// with a full `200` response.
|
||||
Unsupported,
|
||||
/// Probe was inconclusive; treat as unsupported (safe fallback).
|
||||
Unknown,
|
||||
}
|
||||
Reference in New Issue
Block a user