sn-fetch: per-exit egress (Direct vs Proxy) for neighbor exits over the mesh

server/crates/sn-fetch/src/exit.rs

@@ -48,6 +48,37 @@ impl HttpExit for LocalExit {
     }
 }
 
+/// A neighbor's uplink, reached through an HTTP proxy over the mesh.
+/// The proxy is expected to listen on the neighbor's Yggdrasil address,
+/// so the transport between the households is encrypted end-to-end by
+/// the overlay.
+#[derive(Debug)]
+pub struct MeshExit {
+    id: ExitId,
+    client: reqwest::Client,
+}
+
+impl MeshExit {
+    pub fn new(id: ExitId, proxy: reqwest::Url) -> Result<Self, FetchError> {
+        let proxy = reqwest::Proxy::all(proxy)?;
+        let client = reqwest::Client::builder()
+            .connect_timeout(Duration::from_secs(30))
+            .proxy(proxy)
+            .build()?;
+        Ok(Self { id, client })
+    }
+}
+
+impl HttpExit for MeshExit {
+    fn id(&self) -> ExitId {
+        self.id
+    }
+
+    fn client(&self) -> &reqwest::Client {
+        &self.client
+    }
+}
+
 /// All exits available for a fetch, keyed by id.
 #[derive(Default, Clone)]
 pub struct ExitRegistry {

server/crates/sn-fetch/src/main.rs

@@ -15,7 +15,7 @@ use std::path::PathBuf;
 use clap::Parser;
 use sn_fetch::runner::{self, FetchOptions};
 use sn_fetch::types::{
-    ExitId, ExitWeight, MinSegmentBytes, SegmentsPerExit, Sha256Digest, TargetUrl,
+    Egress, ExitId, ExitWeight, MinSegmentBytes, SegmentsPerExit, Sha256Digest, TargetUrl,
 };
 
 #[derive(Debug, Parser)]
@@ -28,12 +28,20 @@ struct Args {
     #[arg(short, long)]
     output: Option<PathBuf>,
 
-    /// Exits as `id=weight` pairs, comma separated, e.g. `local=1,local2=2`.
-    /// `PoC`: all ids map to plain local connections; the weights only shape
-    /// the plan. Multi-exit routing comes with the mesh integration.
-    #[arg(short = 'e', long = "exit", default_value = "local=1")]
+    /// Exits as `id=weight` pairs, comma separated, e.g. `1=2,2=1`.
+    /// A proxy route turns an exit into a neighbor's uplink over the
+    /// mesh overlay: `id=weight@http://[ygg-addr]:8080`. Without a
+    /// proxy, the exit is this machine's own connection.
+    /// `PoC`: weights only shape the plan; a real proxy service
+    /// (`sn-exit`) comes with the mesh integration.
+    #[arg(short = 'e', long = "exit", default_value = "1=1")]
     exits: String,
 
+    /// Total tries per segment (initial try + retries; failed segments
+    /// are retried on a different exit when available).
+    #[arg(long, default_value_t = 3)]
+    max_attempts: u32,
+
     /// Parallel sub-segments per exit.
     #[arg(short = 's', long, default_value_t = 1)]
     segments_per_exit: usize,
@@ -51,10 +59,10 @@ struct Args {
     probe_only: bool,
 }
 
-fn parse_exits(spec: &str) -> Result<Vec<(ExitId, ExitWeight)>, String> {
+fn parse_exits(spec: &str) -> Result<Vec<(ExitId, ExitWeight, Egress)>, String> {
     spec.split(',')
         .map(|pair| {
-            let (id, weight) = pair
+            let (id, rest) = pair
                 .split_once('=')
                 .ok_or_else(|| format!("invalid exit spec `{pair}`, expected `id=weight`"))?;
             let id = id
@@ -63,13 +71,21 @@ fn parse_exits(spec: &str) -> Result<Vec<(ExitId, ExitWeight)>, String> {
                 .ok()
                 .and_then(NonZeroU8::new)
                 .ok_or_else(|| format!("invalid exit id `{id}` (must be a non-zero number)"))?;
+            let (weight, egress) = match rest.split_once('@') {
+                Some((weight, proxy)) => {
+                    let egress = Egress::proxy(proxy.trim())
+                        .map_err(|err| format!("invalid proxy in exit spec `{pair}`: {err}"))?;
+                    (weight, egress)
+                }
+                None => (rest, Egress::Direct),
+            };
             let weight = weight
                 .trim()
                 .parse::<u32>()
                 .ok()
                 .and_then(NonZeroU32::new)
                 .ok_or_else(|| format!("invalid exit weight `{weight}` (must be non-zero)"))?;
-            Ok((ExitId::new(id), ExitWeight::new(weight)))
+            Ok((ExitId::new(id), ExitWeight::new(weight), egress))
         })
         .collect()
 }
@@ -131,6 +147,8 @@ async fn run() -> Result<(), String> {
         NonZeroU64::new(args.min_segment_mib.saturating_mul(1024 * 1024))
             .ok_or("min_segment_mib must be at least 1")?,
     );
+    let max_attempts =
+        NonZeroU32::new(args.max_attempts).ok_or("max_attempts must be at least 1")?;
 
     let expected = match &args.sha256 {
         Some(hex) => Some(Sha256Digest::from_hex(hex).map_err(|err| err.to_string())?),
@@ -146,9 +164,10 @@ async fn run() -> Result<(), String> {
         segments_per_exit: SegmentsPerExit::new(spe),
         min_segment,
         expected_sha256: expected,
+        retries: sn_fetch::exec::RetryPolicy::new(max_attempts),
     };
 
-    let registry = runner::local_registry(&exit_specs).map_err(|err| err.to_string())?;
+    let registry = runner::registry_from_specs(&exit_specs).map_err(|err| err.to_string())?;
     let parts_dir = std::env::temp_dir().join("sn-fetch-parts");
 
     let report = runner::run(&target, &options, &registry, &parts_dir)

server/crates/sn-fetch/src/plan.rs

@@ -2,8 +2,8 @@
 
 use crate::error::FetchError;
 use crate::types::{
-    ByteRange, ExitId, ExitWeight, FileSize, MinSegmentBytes, SegmentIndex, SegmentsPerExit,
-    TargetUrl,
+    ByteRange, Egress, ExitId, ExitWeight, FileSize, MinSegmentBytes, SegmentIndex,
+    SegmentsPerExit, TargetUrl,
 };
 use std::num::NonZeroUsize;
 
@@ -23,8 +23,9 @@ pub struct FetchPlan {
     pub segments: Vec<SegmentPlan>,
 }
 
-/// Exits with their capacity weights.
-pub type ExitSpecs = Vec<(ExitId, ExitWeight)>;
+/// exit description, including how traffic leaves (`Direct` = own
+/// uplink, `Proxy` = a neighbor's exit over the mesh).
+pub type ExitSpecs = Vec<(ExitId, ExitWeight, Egress)>;
 
 impl FetchPlan {
     /// Build a validated plan.
@@ -56,12 +57,12 @@ impl FetchPlan {
             let weakest = exits
                 .iter()
                 .enumerate()
-                .min_by_key(|(_, (id, weight))| (weight.value(), std::cmp::Reverse(id.value())))
+                .min_by_key(|(_, (id, weight, _))| (weight.value(), std::cmp::Reverse(id.value())))
                 .map_or(0, |(pos, _)| pos);
             exits.remove(weakest);
         }
         // Deterministic boundary assignment order.
-        exits.sort_by_key(|(id, _)| *id);
+        exits.sort_by_key(|(id, _, _)| *id);
 
         // If even one segment per exit would be too small, reduce to one
         // segment per exit.
@@ -85,7 +86,7 @@ impl FetchPlan {
             // Weighted split across exits, then even sub-split per exit.
             let units: Vec<u128> = exits
                 .iter()
-                .map(|(_, w)| u128::from(w.value()) * spe.value() as u128)
+                .map(|(_, w, _)| u128::from(w.value()) * spe.value() as u128)
                 .collect();
             let total_units: u128 = units.iter().sum();
 

server/crates/sn-fetch/src/runner.rs

@@ -10,12 +10,13 @@ use sha2::Digest;
 use tokio::io::AsyncWriteExt;
 
 use crate::error::FetchError;
-use crate::exec::{self, FetchReport, SegmentStats};
-use crate::exit::{ExitRegistry, LocalExit};
+use crate::exec::{self, FetchReport, RetryPolicy, SegmentStats};
+use crate::exit::{ExitRegistry, HttpExit, LocalExit, MeshExit};
 use crate::plan::ExitSpecs;
 use crate::probe::{self, ProbeInfo};
+use crate::progress::ProgressSink;
 use crate::types::{
-    FileSize, MinSegmentBytes, RangeSupport, SegmentsPerExit, Sha256Digest, TargetUrl,
+    Egress, FileSize, MinSegmentBytes, RangeSupport, SegmentsPerExit, Sha256Digest, TargetUrl,
 };
 
 /// All knobs for one fetch.
@@ -26,6 +27,8 @@ pub struct FetchOptions {
     pub segments_per_exit: SegmentsPerExit,
     pub min_segment: MinSegmentBytes,
     pub expected_sha256: Option<Sha256Digest>,
+    /// How often a segment is retried (preferably on another exit).
+    pub retries: RetryPolicy,
 }
 
 /// Run a full fetch and produce a report.
@@ -74,7 +77,17 @@ async fn run_ranged(
     tracing::info!("plan: {} segment(s)", plan.segments.len());
 
     let started = Instant::now();
-    let segment_stats: Vec<SegmentStats> = exec::run_segments(&plan, exits, parts_dir).await?;
+    let segment_stats: Vec<SegmentStats> = exec::run_segments_with_policy(
+        &plan,
+        exits,
+        parts_dir,
+        &options.retries,
+        &ProgressSink::tracing_log(),
+    )
+    .await?
+    .into_iter()
+    .map(std::convert::From::from)
+    .collect();
     let digest = exec::assemble(
         parts_dir,
         &options.output,
@@ -105,7 +118,7 @@ async fn run_whole(
     let exit_id = options
         .exits
         .first()
-        .map(|(id, _)| *id)
+        .map(|(id, _, _)| *id)
         .ok_or_else(|| FetchError::InvalidPlan("no exits given".to_string()))?;
     let client = exits.get(exit_id)?.client().clone();
 
@@ -163,11 +176,18 @@ async fn run_whole(
     })
 }
 
-/// Convenience: build a registry with one [`LocalExit`] per spec id.
-pub fn local_registry(specs: &ExitSpecs) -> Result<ExitRegistry, FetchError> {
+/// Build a registry from exit specs: `Egress::Direct` exits connect
+/// locally (own uplink), `Egress::Proxy` exits route through the given
+/// proxy — typically a neighbor's exit service over the Yggdrasil
+/// overlay.
+pub fn registry_from_specs(specs: &ExitSpecs) -> Result<ExitRegistry, FetchError> {
     let mut registry = ExitRegistry::empty();
-    for (id, _) in specs {
-        registry.register(std::sync::Arc::new(LocalExit::new(*id)?));
+    for (id, _, egress) in specs {
+        let exit: std::sync::Arc<dyn HttpExit> = match egress {
+            Egress::Direct => std::sync::Arc::new(LocalExit::new(*id)?),
+            Egress::Proxy(url) => std::sync::Arc::new(MeshExit::new(*id, url.clone())?),
+        };
+        registry.register(exit);
     }
     Ok(registry)
 }

server/crates/sn-fetch/src/types.rs

@@ -230,3 +230,36 @@ pub enum RangeSupport {
     /// Probe was inconclusive; treat as unsupported (safe fallback).
     Unknown,
 }
+
+/// How a request physically leaves through an exit.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum Egress {
+    /// The household's own uplink: plain outgoing HTTP, no detour.
+    Direct,
+    /// Through an HTTP proxy — e.g. a neighbor's exit service reachable
+    /// over the mesh overlay (typically bound to their Yggdrasil
+    /// address).
+    Proxy(reqwest::Url),
+}
+
+impl Egress {
+    /// Parse a proxy URL for [`Egress::Proxy`]; only `http`/`https`
+    /// upstreams are accepted.
+    pub fn proxy(input: &str) -> Result<Self, FetchError> {
+        let url =
+            reqwest::Url::parse(input).map_err(|_| FetchError::InvalidUrl(input.to_string()))?;
+        match url.scheme() {
+            "http" | "https" => Ok(Self::Proxy(url)),
+            _ => Err(FetchError::InvalidUrl(input.to_string())),
+        }
+    }
+}
+
+impl fmt::Display for Egress {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::Direct => write!(f, "direct"),
+            Self::Proxy(url) => write!(f, "proxy:{url}"),
+        }
+    }
+}