diff --git a/.gitignore b/.gitignore index f4fd1f2..1a243a3 100644 --- a/.gitignore +++ b/.gitignore @@ -18,4 +18,12 @@ state.json state.json.tmp failed_urls.txt manifest.jsonl +catalogue.md +catalogue.jsonl +catalogue_moves.jsonl +sorter_applied.jsonl +sorter_low_confidence.txt +subscriptions.json +*.bak +_logs/ *.log diff --git a/README.md b/README.md index 742f94d..fb7a107 100644 --- a/README.md +++ b/README.md @@ -50,11 +50,16 @@ All original Python entry points are ported and live-validated (see below). ## CLI ``` -dlphelper dlp [--subs FILE] [--urls-file FILE] [--token] +dlphelper dlp [--since WHEN] [--urls-file FILE] [--token] [--password] [--retries N] [--retry-delay N] [--cooldown-failures N] [--cooldown-seconds N] [--only gofile|0807] [--limit N] [--shuffle] [--hash] [--extract] [--fresh|--no-fresh] [--dry-run] - [--global-state FILE] [--no-global] [--assume-done FILE] + [--global-state FILE] [--no-global] + +dlphelper dlp --subs FILE # alternative mode: iterate every channel + # in the list (channel_id/out_dir unused) + +dlphelper dlp --assume-done FILE # alternative mode: mark URLs as done, exit dlphelper catalogue [run_dir] [--scan DIR]... [--manifest FILE] [--overrides FILE] [--out-md FILE] [--out-jsonl FILE] [--apply --library DIR] @@ -78,6 +83,9 @@ Notes: (bookkeeping stays out of the library, which may be a seeded torrent). - The sorter implements the rules in `docs/SORTING.md` — that document is the working agreement; change it first, then the code. +- `dlp --since` limits a run to recent messages (e.g. `5d`, `12h`, + `2026-10-04`); `--subs` and `--assume-done` are alternative modes that ignore + ``/``. - Retry loops, cooldowns and the "one gofile guest account per run" rule are ported verbatim from `dlp.py`; a failure in one URL never aborts the run. - Ctrl-C: state.json is saved after every URL; re-run to resume (the python diff --git a/docs/SORTING.md b/docs/SORTING.md index f759bbe..18a075b 100644 --- a/docs/SORTING.md +++ b/docs/SORTING.md @@ -70,14 +70,14 @@ graph LR (`data/gwshop_catalog.json`) entries within the same faction; best sequence-match above `--threshold` (default 0.62) wins. The catalog provides canonical names, sub-factions and classes. -4. **Plan** — default output: a dry-run table `source → target` plus - `sorter_plan.jsonl`. Nothing is touched. +4. **Plan** — default output: a dry-run table `source → target` on stdout + (`--json` prints the same plan as JSON). Nothing is touched. 5. **Apply** — create the target and **hardlink** the file there - (`os.link`); if the filesystems differ, fall back to a real copy with a - warning. Originals are never modified or deleted — that is what keeps the - *arr-style workflow possible (remux/seed/sort independently). Every - applied action is appended to `sorter_applied.jsonl` and re-runs skip - targets that already exist. + (`std::fs::hard_link`); if the filesystems differ, fall back to a real + copy with a warning. Originals are never modified or deleted — that is + what keeps the *arr-style workflow possible (remux/seed/sort + independently). Every applied action is appended to `sorter_applied.jsonl` + and re-runs skip targets that already exist. ## Bookkeeping location @@ -95,8 +95,8 @@ confidence doc always lands next to the audit file. sees. Limitation: hardlinks need source and library on the same filesystem; the sorter detects this and copies instead (warned). - Zip contents **can** be read without extracting (that is how the sorter - peeks). Modifying a member in place is technically possible with - `zipfile` but is not useful here. + peeks). Modifying a member in place is technically possible with the `zip` + crate but is not useful here. - Endgame (later stage): unpack archives into the unit folder so the library is plain files, easier to sort per-model and to torrent. The unpack step will live in the sorter (`--unpack`, 7z binary) but stays off @@ -142,7 +142,8 @@ already-present duplicates, 0 copied): (`noserverino` behavior): two paths to the same hardlinked file show different `st_ino`, and the values are stable across calls. To verify a link, compare `st_nlink` (and content/size) — never inodes. Server-side - links are real: a fresh `os.link` bumps nlink (2→3 in the checked case). + links are real: a fresh `std::fs::hard_link` bumps nlink (2→3 in the + checked case). - Filenames with raw cp437 bytes crash 7z direct-to-CIFS with exit 2 ("Cannot open output file"); the extractor then extracts to a `/var/tmp` temp dir, repairs the names and moves the result over. diff --git a/src/catalogue.rs b/src/catalogue.rs index 194f634..c778c53 100644 --- a/src/catalogue.rs +++ b/src/catalogue.rs @@ -607,7 +607,7 @@ pub fn write_reports( format_size(files_size) )); let mut sorted_files = files.clone(); - sorted_files.sort_by(|a, b| a.name.to_lowercase().cmp(&b.name.to_lowercase())); + sorted_files.sort_by_key(|a| a.name.to_lowercase()); for e in sorted_files { lines.push(format!( " - {} ({}){}", @@ -768,7 +768,8 @@ mod tests { /// All 17 CASES from tests/test_catalogue.py; `None` = "don't care". #[test] fn classify_matches_python_test_catalogue() { - let cases: &[(&str, Option<&str>, Option<&str>, Option<&str>)] = &[ + type Case<'a> = (&'a str, Option<&'a str>, Option<&'a str>, Option<&'a str>); + let cases: &[Case] = &[ // (filename, expected system, expected faction, expected class or None-any) // GW naming: chapters are factions, classes are GW unit types. ( diff --git a/src/difflib.rs b/src/difflib.rs index 0b022c1..54cce80 100644 --- a/src/difflib.rs +++ b/src/difflib.rs @@ -16,27 +16,32 @@ pub struct Match { size: usize, } -pub struct SequenceMatcher<'a> { - a: &'a str, - b: &'a str, +pub struct SequenceMatcher { + /// `a`/`b` as chars, collected once (python works on code points; the + /// matching routines index them repeatedly, so collecting here avoids + /// re-allocating on every `find_longest_match` call). + a_chars: Vec, + b_chars: Vec, /// b[j] -> indices (purged of junk and popular elements, like `b2j`). b2j: std::collections::HashMap>, bjunk: std::collections::HashSet, } -impl<'a> SequenceMatcher<'a> { - pub fn new(a: &'a str, b: &'a str) -> Self { +impl SequenceMatcher { + pub fn new(a: &str, b: &str) -> Self { // __chain_b with isjunk=None: no junk, but autojunk purges // "popular" characters from long sequences (len(b) >= 200). + let a_chars: Vec = a.chars().collect(); + let b_chars: Vec = b.chars().collect(); let mut b2j: std::collections::HashMap> = std::collections::HashMap::new(); - for (idx, elt) in b.chars().enumerate() { + for (idx, &elt) in b_chars.iter().enumerate() { b2j.entry(elt).or_default().push(idx); } let bjunk = std::collections::HashSet::new(); // isjunk is None - let mut bpopular = std::collections::HashSet::new(); - let n = b.chars().count(); + let n = b_chars.len(); if n >= 200 { let ntest = n / 100 + 1; + let mut bpopular = std::collections::HashSet::new(); for (&elt, idxs) in &b2j { if idxs.len() > ntest { bpopular.insert(elt); @@ -46,7 +51,12 @@ impl<'a> SequenceMatcher<'a> { b2j.remove(&elt); } } - SequenceMatcher { a, b, b2j, bjunk } + SequenceMatcher { + a_chars, + b_chars, + b2j, + bjunk, + } } fn isbjunk(&self, elt: char) -> bool { @@ -55,9 +65,10 @@ impl<'a> SequenceMatcher<'a> { /// CPython SequenceMatcher.find_longest_match(alo, ahi, blo, bhi) over /// char indices. + #[allow(clippy::needless_range_loop)] // `i` feeds the besti/bestj arithmetic fn find_longest_match(&self, alo: usize, ahi: usize, blo: usize, bhi: usize) -> Match { - let a_chars: Vec = self.a.chars().collect(); - let b_chars: Vec = self.b.chars().collect(); + let a_chars = &self.a_chars; + let b_chars = &self.b_chars; let nothing: Vec = Vec::new(); let mut besti = alo; @@ -129,8 +140,8 @@ impl<'a> SequenceMatcher<'a> { /// All matching blocks, in order (recursive queue algorithm). pub fn get_matching_blocks(&self) -> Vec { - let la = self.a.chars().count(); - let lb = self.b.chars().count(); + let la = self.a_chars.len(); + let lb = self.b_chars.len(); let mut blocks: Vec = Vec::new(); let mut queue: Vec<(usize, usize, usize, usize)> = vec![(0, la, 0, lb)]; while let Some((alo, ahi, blo, bhi)) = queue.pop() { @@ -163,12 +174,12 @@ impl<'a> SequenceMatcher<'a> { pub fn quick_ratio(&self) -> f64 { let mut fullbcount: std::collections::HashMap = std::collections::HashMap::new(); - for elt in self.b.chars() { + for &elt in &self.b_chars { *fullbcount.entry(elt).or_default() += 1; } let mut avail: std::collections::HashMap = std::collections::HashMap::new(); let mut matches = 0usize; - for elt in self.a.chars() { + for &elt in &self.a_chars { let numb = avail .get(&elt) .copied() @@ -178,13 +189,13 @@ impl<'a> SequenceMatcher<'a> { matches += 1; } } - Self::calculate_ratio(matches, self.a.chars().count(), self.b.chars().count()) + Self::calculate_ratio(matches, self.a_chars.len(), self.b_chars.len()) } /// Trivial upper bound (python `real_quick_ratio`). pub fn real_quick_ratio(&self) -> f64 { - let la = self.a.chars().count(); - let lb = self.b.chars().count(); + let la = self.a_chars.len(); + let lb = self.b_chars.len(); Self::calculate_ratio(la.min(lb), la, lb) } @@ -192,7 +203,7 @@ impl<'a> SequenceMatcher<'a> { /// `SequenceMatcher(None, a, b).ratio()`. pub fn ratio(&self) -> f64 { let matches: usize = self.get_matching_blocks().iter().map(|m| m.size).sum(); - Self::calculate_ratio(matches, self.a.chars().count(), self.b.chars().count()) + Self::calculate_ratio(matches, self.a_chars.len(), self.b_chars.len()) } } diff --git a/src/extract.rs b/src/extract.rs index b0c65c7..16fe2e2 100644 --- a/src/extract.rs +++ b/src/extract.rs @@ -183,10 +183,10 @@ fn walk_files_sorted(root: &Path) -> Vec { Ok(meta) if meta.is_dir() => dirs.push(path), Ok(meta) if meta.is_file() => files.push(path), Ok(meta) if meta.is_symlink() => { - if let Ok(target) = fs::metadata(&path) { - if target.is_file() { - files.push(path); - } + if let Ok(target) = fs::metadata(&path) + && target.is_file() + { + files.push(path); } } _ => {} @@ -369,9 +369,12 @@ impl Extractor { // Container formats: trust the magic bytes over the extension (python's // 7z CLI sniffs content too). Compressed streams (.gz/.bz2/.xz) and tar // stay extension-driven, where the tar-inside logic needs the name. + // The OUTERMOST suffix decides how the file is wrapped: for + // `x.tar.gz` that is ".gz" (decompress, then untar). `archive_suffixes` + // returns outer-first, so the outer suffix is `first()`, not `last()`. let kind: Option = sniff_container(archive) .map(str::to_string) - .or_else(|| sfx.last().cloned()); + .or_else(|| sfx.first().cloned()); let res: Result<(), String> = match kind.as_deref() { Some(".7z") => extract_7z(archive, &actual), Some(".zip") => extract_zip(archive, &actual), @@ -674,10 +677,10 @@ pub fn load_done(manifest_path: &Path) -> HashSet { if line.is_empty() { continue; } - if let Ok(value) = serde_json::from_str::(line) { - if let Some(dest) = value.get("dest").and_then(|d| d.as_str()) { - done.insert(dest.to_string()); - } + if let Ok(value) = serde_json::from_str::(line) + && let Some(dest) = value.get("dest").and_then(|d| d.as_str()) + { + done.insert(dest.to_string()); } } done @@ -899,7 +902,7 @@ mod tests { block[148..156].copy_from_slice(chk.as_bytes()); let mut out = block; out.extend_from_slice(data); - while out.len() % 512 != 0 { + while !out.len().is_multiple_of(512) { out.push(0); } out.extend(std::iter::repeat_n(0u8, 1024)); @@ -1044,7 +1047,7 @@ mod tests { let extractor = Extractor { out_root: out.clone(), }; - extractor.run(&[src.clone()]).unwrap(); + extractor.run(std::slice::from_ref(&src)).unwrap(); assert_eq!( fs::read_to_string(out.join("src/Pack/unit/part.txt")).unwrap(), "payload" @@ -1062,7 +1065,7 @@ mod tests { let extractor = Extractor { out_root: out.clone(), }; - extractor.run(&[src.clone()]).unwrap(); + extractor.run(std::slice::from_ref(&src)).unwrap(); let s = src.as_path(); let o = out.as_path(); @@ -1192,13 +1195,13 @@ mod tests { let extractor = Extractor { out_root: out.clone(), }; - extractor.run(&[src.clone()]).unwrap(); + extractor.run(std::slice::from_ref(&src)).unwrap(); let manifest_before = fs::read_to_string(out.join("extract_manifest.jsonl")).unwrap(); let inv_before = inventory(&out); assert!(!manifest_before.is_empty()); - extractor.run(&[src.clone()]).unwrap(); + extractor.run(std::slice::from_ref(&src)).unwrap(); let manifest_after = fs::read_to_string(out.join("extract_manifest.jsonl")).unwrap(); assert_eq!( diff --git a/src/globalstore.rs b/src/globalstore.rs index 04aeb44..a6e23ca 100644 --- a/src/globalstore.rs +++ b/src/globalstore.rs @@ -40,10 +40,10 @@ impl GlobalRecord { /// `~/.local/share/dlphelper/urls.json`. An empty XDG_DATA_HOME falls back to /// home, matching the python version. pub fn default_global_path() -> Option { - if let Ok(xdg) = std::env::var("XDG_DATA_HOME") { - if !xdg.is_empty() { - return Some(PathBuf::from(xdg).join("dlphelper").join("urls.json")); - } + if let Ok(xdg) = std::env::var("XDG_DATA_HOME") + && !xdg.is_empty() + { + return Some(PathBuf::from(xdg).join("dlphelper").join("urls.json")); } dirs::home_dir().map(|home| { home.join(".local") diff --git a/src/gwshop.rs b/src/gwshop.rs index a46797d..f00678d 100644 --- a/src/gwshop.rs +++ b/src/gwshop.rs @@ -10,7 +10,7 @@ //! Credentials are read from (first hit wins): //! 1. env ALGOLIA_APP_ID + ALGOLIA_API_KEY //! 2. algolia.json next to the repo root (gitignored): -//! {"app_id": "...", "api_key": "..."} +//! {"app_id": "...", "api_key": "..."} //! //! The key is the *search-only* public key (never an admin key). Treat any //! key found in algolia.json as still secret: never log or commit it. @@ -125,10 +125,10 @@ impl GwShopClient { return r.json().context("decoding Algolia response"); } let text = r.text().unwrap_or_default(); - last_err = Some(format!( - "Algolia {method} {path} -> {status}: {}", - &text[..text.len().min(200)] - )); + // Truncate by chars, not bytes: a byte slice could split a + // multi-byte character and panic (python sliced `text[:200]`). + let snippet: String = text.chars().take(200).collect(); + last_err = Some(format!("Algolia {method} {path} -> {status}: {snippet}")); if status == 401 || status == 403 { // credentials: no point retrying break; @@ -143,6 +143,9 @@ impl GwShopClient { )) } + /// Enumerate the application's indices (python `list_indices`). Kept as + /// a client capability even though the CLI does not use it yet. + #[allow(dead_code)] pub fn list_indices(&self) -> anyhow::Result> { let v = self.request("GET", "/1/indexes", None)?; Ok(v.get("items") diff --git a/src/main.rs b/src/main.rs index 36d1f8a..c6f43e1 100644 --- a/src/main.rs +++ b/src/main.rs @@ -88,7 +88,7 @@ struct DlpArgs { #[arg(long, default_value_t = 60)] cooldown_seconds: u64, /// process only one host class - #[arg(long)] + #[arg(long, value_parser = ["gofile", "0807"])] only: Option, /// only URLs posted at/after this point: e.g. 5d, 12h, 2026-10-04 #[arg(long, value_name = "WHEN")] @@ -219,10 +219,10 @@ fn resolve_token(cli_value: Option<&str>) -> anyhow::Result { if let Some(t) = cli_value { return Ok(t.to_string()); } - if let Ok(t) = std::env::var("DISCORD_TOKEN") { - if !t.is_empty() { - return Ok(t); - } + if let Ok(t) = std::env::var("DISCORD_TOKEN") + && !t.is_empty() + { + return Ok(t); } log::info!("No Discord token given ($DISCORD_TOKEN); prompting (input hidden)."); if !std::io::stdin().is_terminal() { @@ -354,11 +354,11 @@ fn cmd_catalogue(args: &CatalogueArgs) -> anyhow::Result<()> { .unwrap_or_default() }; let mut scan_dirs = args.scan.clone(); - if let Some(rd) = run_dir { - if args.manifest.is_none() { - // run dir is scanned as a whole (covers files not yet in the manifest) - scan_dirs.push(rd.to_path_buf()); - } + if let Some(rd) = run_dir + && args.manifest.is_none() + { + // run dir is scanned as a whole (covers files not yet in the manifest) + scan_dirs.push(rd.to_path_buf()); } if manifests.is_empty() && scan_dirs.is_empty() { @@ -416,7 +416,7 @@ fn cmd_catalogue(args: &CatalogueArgs) -> anyhow::Result<()> { let Some(library) = &args.library else { parser_error("--apply needs --library DIR"); }; - let Some(run_dir) = run_dir else { + let Some(_run_dir) = run_dir else { parser_error("--apply needs a run_dir (files are moved out of it)"); }; let moved = catalogue::apply_moves( @@ -431,7 +431,6 @@ fn cmd_catalogue(args: &CatalogueArgs) -> anyhow::Result<()> { ); // rewrite reports with updated paths catalogue::write_reports(&entries, &out_md, &out_jsonl)?; - let _ = run_dir; } Ok(()) } @@ -474,7 +473,7 @@ fn cmd_sorter(args: &SorterArgs) -> anyhow::Result<()> { } else { // hidden trees like .broken/ must never reach the plan let mut out = Vec::new(); - collect_visible(source, source, &mut out)?; + collect_visible(source, &mut out)?; out.sort(); out }; @@ -568,9 +567,9 @@ fn cmd_sorter(args: &SorterArgs) -> anyhow::Result<()> { Ok(()) } -/// Recursively collect files under `root`, skipping any path whose parts -/// (relative to root) start with '.' (python rglob filter). -fn collect_visible(root: &Path, dir: &Path, out: &mut Vec) -> std::io::Result<()> { +/// Recursively collect files under `dir`, skipping any path whose file name +/// starts with '.' (python rglob filter). +fn collect_visible(dir: &Path, out: &mut Vec) -> std::io::Result<()> { for entry in std::fs::read_dir(dir)? { let entry = entry?; let path = entry.path(); @@ -579,7 +578,7 @@ fn collect_visible(root: &Path, dir: &Path, out: &mut Vec) -> std::io:: continue; } if path.is_dir() { - collect_visible(root, &path, out)?; + collect_visible(&path, out)?; } else { out.push(path); } diff --git a/src/orchestrate.rs b/src/orchestrate.rs index b0cec91..9506581 100644 --- a/src/orchestrate.rs +++ b/src/orchestrate.rs @@ -224,17 +224,15 @@ pub struct SourceInfo { pub message_id: Option, pub timestamp: Option, pub author: Option, - pub manifest: PathBuf, } -fn source_of(channel_id: &str, f: &FoundUrl, manifest: &Path) -> SourceInfo { +fn source_of(channel_id: &str, f: &FoundUrl) -> SourceInfo { SourceInfo { channel: channel_id.to_string(), host: Some(f.host.to_string()), message_id: f.message_id.clone().map(Value::String), timestamp: f.timestamp.clone(), author: f.author.clone(), - manifest: manifest.to_path_buf(), } } @@ -386,13 +384,11 @@ pub fn process_channel( .filter(|f| { if is_torrent_url(&f.url) { log::info!("Skip torrent (never downloaded): {}", f.url); - if let Err(e) = state.mark(&f.url, |r| { + state.mark_logged(&f.url, |r| { r.status = "ok".into(); r.extra .insert("note".into(), Value::from("torrent link skipped (policy)")); - }) { - log::error!("Could not record torrent skip for {}: {e}", f.url); - } + }); return false; } true @@ -440,20 +436,20 @@ pub fn process_channel( // classify() already filtered, but be defensive _ => { log::warn!("Skipping unsupported URL: {}", f.url); - state.mark(&f.url, |r| { + state.mark_logged(&f.url, |r| { r.status = "failed".into(); r.error = Some("unsupported host".into()); - })?; + }); } } } let mut source_by_url: HashMap = HashMap::new(); for f in &gofile_urls { - source_by_url.insert(f.url.clone(), source_of(channel_id, f, &manifest)); + source_by_url.insert(f.url.clone(), source_of(channel_id, f)); } for f in &z0807_urls { - source_by_url.insert(f.url.clone(), source_of(channel_id, f, &manifest)); + source_by_url.insert(f.url.clone(), source_of(channel_id, f)); } // --limit applies per host class to the *pending* URLs, so a test run @@ -527,7 +523,7 @@ pub fn process_channel( out_dir, &mut state, &source_by_url, - gstore.as_deref_mut(), + gstore, &manifest, )?; } @@ -613,11 +609,11 @@ pub fn run_gofile( if is_done(state, gstore.as_deref(), &f.url) { continue; } - state.mark(&f.url, |r| { + state.mark_logged(&f.url, |r| { r.status = "failed".into(); r.error = Some(exc.to_string()); r.files = Vec::new(); - })?; + }); } return Ok(()); } @@ -680,21 +676,23 @@ pub fn run_gofile( match files { Ok(files) => { log::info!("OK {} ({} files)", f.url, files.len()); - record_files( + if let Err(e) = record_files( state, gstore.as_deref_mut(), &f.url, &files, &source, manifest, - )?; + ) { + log::error!("Could not record download for {}: {e:#}", f.url); + } if files.is_empty() { - state.mark(&f.url, |r| { + state.mark_logged(&f.url, |r| { r.extra.insert( "note".into(), Value::from("only .torrent files - skipped"), ); - })?; + }); } if opts.extract { extract_zips(&files); @@ -708,10 +706,10 @@ pub fn run_gofile( // a bug must not stop the run log::error!("Unexpected error for {}: {err:#}", f.url); let msg = format!("unexpected: {err}"); - state.mark(&f.url, |r| { + state.mark_logged(&f.url, |r| { r.error = Some(msg); r.files = Vec::new(); - })?; + }); if attempt < opts.retries { thread::sleep(Duration::from_secs(opts.retry_delay * attempt as u64)); } @@ -721,43 +719,43 @@ pub fn run_gofile( GoFileError::PasswordRequired(msg) => { // Retrying without a password never helps log::error!("PASSWORD REQUIRED {}: {msg}", f.url); - state.mark(&f.url, |r| { + state.mark_logged(&f.url, |r| { r.status = "failed".into(); r.error = Some(msg.clone()); r.files = Vec::new(); - })?; + }); resolved = true; break; } GoFileError::NotFound(msg) => { // Dead links never come back - no retry log::error!("DEAD LINK {}: {msg}", f.url); - state.mark(&f.url, |r| { + state.mark_logged(&f.url, |r| { r.status = "failed".into(); r.error = Some(msg.clone()); r.files = Vec::new(); - })?; + }); resolved = true; break; } GoFileError::Empty(msg) => { // Emptied content is stable - no retry log::error!("EMPTY CONTENT {}: {msg}", f.url); - state.mark(&f.url, |r| { + state.mark_logged(&f.url, |r| { r.status = "failed".into(); r.error = Some(msg.clone()); r.files = Vec::new(); - })?; + }); resolved = true; break; } GoFileError::AuthError(msg) => { // Token may have died mid-run (rate limit); re-auth log::warn!("Auth error for {}: {msg} - re-authenticating", f.url); - state.mark(&f.url, |r| { + state.mark_logged(&f.url, |r| { r.error = Some(msg.clone()); r.files = Vec::new(); - })?; + }); client.reset(); if let Err(reauth) = client.ensure_token() { log::error!("Re-authentication failed: {reauth}"); @@ -775,10 +773,10 @@ pub fn run_gofile( opts.retries, f.url ); - state.mark(&f.url, |r| { + state.mark_logged(&f.url, |r| { r.error = Some(msg.clone()); r.files = Vec::new(); - })?; + }); if attempt < opts.retries { thread::sleep(Duration::from_secs( opts.retry_delay * attempt as u64, @@ -795,11 +793,11 @@ pub fn run_gofile( .get(&f.url) .and_then(|r| r.error.clone()) .unwrap_or_else(|| "unknown".into()); - state.mark(&f.url, |r| { + state.mark_logged(&f.url, |r| { r.status = "failed".into(); r.error = Some(err); r.files = Vec::new(); - })?; + }); consecutive_failures += 1; log::error!("FAILED {} after {} attempts", f.url, opts.retries); if consecutive_failures >= opts.cooldown_failures { @@ -890,14 +888,16 @@ pub fn run_0807( format_size(size), entries.len() ); - record_files( + if let Err(e) = record_files( state, gstore.as_deref_mut(), &f.url, &entries, &source, manifest, - )?; + ) { + log::error!("Could not record download for {}: {e:#}", f.url); + } if opts.extract { extract_zips(&entries); } @@ -910,10 +910,10 @@ pub fn run_0807( // a bug must not stop the run log::error!("Unexpected error for {}: {err:#}", f.url); let msg = format!("unexpected: {err}"); - state.mark(&f.url, |r| { + state.mark_logged(&f.url, |r| { r.error = Some(msg); r.files = Vec::new(); - })?; + }); if attempt < opts.retries { thread::sleep(Duration::from_secs(opts.retry_delay * attempt as u64)); } @@ -923,11 +923,11 @@ pub fn run_0807( Zero807Error::NotFound(msg) => { // Dead links never come back - no retry log::error!("DEAD LINK {}: {msg}", f.url); - state.mark(&f.url, |r| { + state.mark_logged(&f.url, |r| { r.status = "failed".into(); r.error = Some(msg.clone()); r.files = Vec::new(); - })?; + }); resolved = true; break; } @@ -938,10 +938,10 @@ pub fn run_0807( opts.retries, f.url ); - state.mark(&f.url, |r| { + state.mark_logged(&f.url, |r| { r.error = Some(msg.clone()); r.files = Vec::new(); - })?; + }); if attempt < opts.retries { thread::sleep(Duration::from_secs( opts.retry_delay * attempt as u64, @@ -958,11 +958,11 @@ pub fn run_0807( .get(&f.url) .and_then(|r| r.error.clone()) .unwrap_or_else(|| "unknown".into()); - state.mark(&f.url, |r| { + state.mark_logged(&f.url, |r| { r.status = "failed".into(); r.error = Some(err); r.files = Vec::new(); - })?; + }); consecutive_failures += 1; log::error!("FAILED {} after {} attempts", f.url, opts.retries); if consecutive_failures >= opts.cooldown_failures { @@ -1107,7 +1107,6 @@ mod tests { message_id: Some(Value::from(42)), timestamp: Some("2024-01-01".into()), author: Some("ben".into()), - manifest: manifest.clone(), }; let files = vec![serde_json::json!({"path": "/tmp/x.7z", "size": 10, "name": "x.7z"})]; record_files( diff --git a/src/sorter.rs b/src/sorter.rs index aabe518..c42227a 100644 --- a/src/sorter.rs +++ b/src/sorter.rs @@ -181,7 +181,7 @@ pub fn list_zip(path: &Path, limit: usize) -> Vec { return Vec::new(); } }; - let mut zf = match zip::ZipArchive::new(file) { + let zf = match zip::ZipArchive::new(file) { Ok(zf) => zf, Err(exc) => { log::debug!("zip listing failed for {}: {exc}", path.display()); @@ -190,10 +190,10 @@ pub fn list_zip(path: &Path, limit: usize) -> Vec { }; let mut names = Vec::new(); for i in 0..zf.len() { - if let Some(name) = zf.name_for_index(i) { - if !name.ends_with('/') { - names.push(name.to_string()); - } + if let Some(name) = zf.name_for_index(i) + && !name.ends_with('/') + { + names.push(name.to_string()); } if names.len() >= limit { break; @@ -228,12 +228,6 @@ pub fn list_7z(path: &Path, limit: usize) -> Vec { names } -fn is_archive_name(name: &str) -> bool { - ARCHIVE_SUFFIXES - .iter() - .any(|s| py_suffix(name).eq_ignore_ascii_case(s)) -} - /// python `Path.suffix`: the last dot-suffix, or "" (dotfiles have none). fn py_suffix(name: &str) -> &str { let stem_end = py_stem_split(name); @@ -241,6 +235,9 @@ fn py_suffix(name: &str) -> &str { } /// Byte index where python's stem ends (start of the suffix, or len). +/// +/// Matches pathlib: the split dot must satisfy `0 < i < len(name) - 1`, so a +/// leading dot (dotfile) and a trailing dot are not suffixes. fn py_stem_split(name: &str) -> usize { let base = match name.rfind('/') { Some(i) => i + 1, @@ -248,8 +245,8 @@ fn py_stem_split(name: &str) -> usize { }; let base_str = &name[base..]; match base_str.rfind('.') { - Some(0) | None => name.len(), - Some(i) => base + i, + Some(i) if i > 0 && i < base_str.len() - 1 => base + i, + _ => name.len(), } } @@ -280,7 +277,7 @@ pub fn inspect(path: &Path) -> SortItem { Vec::new() }; if item.inner_names.is_empty() && size > 0 { - let head_is_html = fs::File::open(path).map_or(false, |mut fh| { + let head_is_html = fs::File::open(path).is_ok_and(|mut fh| { use std::io::Read; let mut head = Vec::with_capacity(512); let _ = fh.by_ref().take(512).read_to_end(&mut head); @@ -309,7 +306,7 @@ pub fn inspect(path: &Path) -> SortItem { /// fellaz catachan jungle fighters x") - the file stem alone loses the /// archive's own folder structure. pub fn clean_inner(name: &str) -> String { - clean_name(&name.replace('/', " ").replace('\\', " ")) + clean_name(&name.replace(['/', '\\'], " ")) } /// True for directory names that must never be match candidates. @@ -728,15 +725,15 @@ fn apply_entry( item.match_kind = kind.to_string(); item.confidence = confidence.to_string(); let entry_faction = entry.faction.clone(); - if let Some(f) = entry_faction { - if !matches!( + if let Some(f) = entry_faction + && !matches!( f.as_str(), "Unit Type" | "Heroes" | "Warriors" | "Character" - ) { - let e = &ctx.catalog[entry_idx]; - item.system = e.system.clone().unwrap_or_else(|| item.system.clone()); - item.faction = f; - } + ) + { + let e = &ctx.catalog[entry_idx]; + item.system = e.system.clone().unwrap_or_else(|| item.system.clone()); + item.faction = f; } } @@ -846,10 +843,10 @@ fn match_name( ) -> Option<(f64, usize, bool)> { let key = (clean.to_string(), system.to_string(), faction.to_string()); if let Some(hit) = cache.dir_matches.get(&key) { - return hit.clone(); + return *hit; } let result = compute_match_name(clean, ctx, system, faction, threshold, cache); - cache.dir_matches.insert(key, result.clone()); + cache.dir_matches.insert(key, result); result } @@ -864,17 +861,17 @@ fn compute_match_name( if system == "Unknown" { let pool: Vec = (0..ctx.catalog.len()).collect(); let full = best_in_pool_str(clean, &pool, ctx, false, cache); - if let Some((score, idx)) = full { - if score >= CROSS_SYSTEM_MIN { - let entry = &ctx.catalog[idx]; - let contained = token_contained(clean, &entry.clean) - || entry - .clean_aliases - .iter() - .any(|a| token_contained(clean, a)); - if contained { - return Some((score, idx, false)); - } + if let Some((score, idx)) = full + && score >= CROSS_SYSTEM_MIN + { + let entry = &ctx.catalog[idx]; + let contained = token_contained(clean, &entry.clean) + || entry + .clean_aliases + .iter() + .any(|a| token_contained(clean, a)); + if contained { + return Some((score, idx, false)); } } None @@ -918,10 +915,10 @@ fn compute_match_name( continue; } if winner.as_ref().is_none_or(|w| c.0 > w.0) { - winner = Some(c.clone()); + winner = Some(*c); } } - return winner; + winner } else { pool_best(clean, &pool, ctx, threshold, cache) } @@ -933,20 +930,16 @@ fn best_in_pool_str( clean: &str, pool: &[usize], ctx: &SorterCtx, - use_inner: bool, + _use_inner: bool, cache: &mut MatchCache, ) -> Option<(f64, usize)> { let mut best: Option<(f64, usize)> = None; for &idx in pool { - let mut score = best_ratio(clean, ctx, idx, true, cache); - if use_inner { - // bare-string variant never has inner names; kept for shape parity - } + let score = best_ratio(clean, ctx, idx, true, cache); if best.is_none_or(|b| score > b.0) { best = Some((score, idx)); } } - let _ = use_inner; best } @@ -1011,10 +1004,7 @@ fn rehome_from_dirs( } let own = item.match_kind == "catalog" || item.match_kind == "weak"; let own_score = if own { item.match_score } else { 0.0 }; - let Some((score, idx, weak, real_dir)) = best_dir_match(item, dirs, ctx, threshold, cache) - else { - return None; - }; + let (score, idx, weak, real_dir) = best_dir_match(item, dirs, ctx, threshold, cache)?; if own && score < own_score { return None; } @@ -1140,12 +1130,10 @@ fn find_best( if best.is_none_or(|b| b.0 < threshold) { best = best_in_pool(item, &all, ctx, true, cache); } - if let Some((score, idx)) = best { - if score < CROSS_SYSTEM_MIN { - best = None; - } else if !contained_somewhere(item, idx, ctx) { - best = None; - } + if let Some((score, idx)) = best + && (score < CROSS_SYSTEM_MIN || !contained_somewhere(item, idx, ctx)) + { + best = None; } best } @@ -1191,11 +1179,11 @@ fn apply_best( Some("Unit Type") | Some("Heroes") | Some("Warriors") | Some("Character") ); if item.faction == UNSORTED { - if let Some(ef) = &entry.faction { - if !generic { - item.system = entry.system.clone().unwrap_or_else(|| item.system.clone()); - item.faction = ef.clone(); - } + if let Some(ef) = &entry.faction + && !generic + { + item.system = entry.system.clone().unwrap_or_else(|| item.system.clone()); + item.faction = ef.clone(); } if let Some(g) = &entry.group { if g != "Unit Type" { @@ -1238,10 +1226,10 @@ fn apply_best( .unwrap_or_else(|| UNSORTED.into()); } } - } else if let Some(g) = &entry.group { - if g != "Unit Type" { - item.group = g.clone(); - } + } else if let Some(g) = &entry.group + && g != "Unit Type" + { + item.group = g.clone(); } } else if *score >= WEAK_MIN && weak_tier_shared(item, *idx, ctx) { // Weak tier: a same-system hit below the threshold with at least one @@ -1298,9 +1286,9 @@ pub fn build_target(item: &SortItem, library: &Path, inner_rel: Option<&Path>) - let mut parts: Vec = Vec::new(); if item.system == "Unknown" { parts.push(UNSORTED.into()); - if inner_rel.is_some() && item.unit.is_some() { + if let (Some(_), Some(unit)) = (inner_rel, &item.unit) { // extracted archives keep their unit folder even when unplaced - parts.push(display_name(item.unit.as_ref().unwrap())); + parts.push(display_name(unit)); } } else { parts.push(item.system.clone()); @@ -1384,28 +1372,27 @@ pub fn sort_plan( let mut inner_rel: Option = None; let prov = provenance; let rootp = root; - if let (Some(rootp), Some(prov)) = (rootp, prov) { - if let Ok(rel) = f.strip_prefix(rootp) { - let mut p = rel.parent().unwrap_or_else(|| Path::new("")); - loop { - if p.as_os_str().is_empty() { - break; - } - if let Some(unit) = prov.get(&p.to_string_lossy().into_owned()) { - dest_key = Some(p.to_string_lossy().into_owned()); - unit_name = Some(unit.clone()); - // dirs-only relative path (the file name itself is - // appended by build_target; keeping it here would - // duplicate it in the target path) - let rest = rel.strip_prefix(p).unwrap(); - inner_rel = - Some(rest.parent().unwrap_or_else(|| Path::new("")).to_path_buf()); - break; - } - match p.parent() { - Some(par) => p = par, - None => break, - } + if let (Some(rootp), Some(prov)) = (rootp, prov) + && let Ok(rel) = f.strip_prefix(rootp) + { + let mut p = rel.parent().unwrap_or_else(|| Path::new("")); + loop { + if p.as_os_str().is_empty() { + break; + } + if let Some(unit) = prov.get(&p.to_string_lossy().into_owned()) { + dest_key = Some(p.to_string_lossy().into_owned()); + unit_name = Some(unit.clone()); + // dirs-only relative path (the file name itself is + // appended by build_target; keeping it here would + // duplicate it in the target path) + let rest = rel.strip_prefix(p).unwrap(); + inner_rel = Some(rest.parent().unwrap_or_else(|| Path::new("")).to_path_buf()); + break; + } + match p.parent() { + Some(par) => p = par, + None => break, } } } @@ -1435,14 +1422,13 @@ pub fn sort_plan( dirs.push((d, dir_parts[i].to_string_lossy().into_owned())); } let winner = rehome_from_dirs(&mut item, &dirs, &ctx, threshold, &mut cache, false); - if let Some(winner) = winner { - if let Ok(inner) = f + if let Some(winner) = winner + && let Ok(inner) = f .parent() .unwrap_or_else(|| Path::new("")) .strip_prefix(&winner) - { - inner_rel = Some(inner.to_path_buf()); - } + { + inner_rel = Some(inner.to_path_buf()); } } item.target = Some(build_target(&item, library, inner_rel.as_deref())); @@ -1451,8 +1437,9 @@ pub fn sort_plan( } else { // one classification per extraction directory, reused for every // file inside it - let dk = dest_key.unwrap(); - let un = unit_name.unwrap(); + let (Some(dk), Some(un)) = (dest_key, unit_name) else { + unreachable!("dest_key and unit_name are set together") + }; if !unit_cache.contains_key(&dk) { let listing: Vec = { let mut all: Vec = Vec::new(); @@ -1492,30 +1479,27 @@ pub fn sort_plan( item.match_score = fields.match_score; item.match_kind = fields.match_kind; item.confidence = fields.confidence; - if let Some(ir) = &inner_rel { - if ir.iter().count() > 0 { - // extracted archives: inner dir names may claim subtrees - let mut dirs: Vec<(PathBuf, String)> = Vec::new(); - let parts: Vec = - ir.iter().map(|p| p.to_os_string()).collect(); - for i in (0..parts.len()).rev() { - let mut d = rootp.unwrap().join(&dk); - for part in &parts[..=i] { - d = d.join(part); - } - dirs.push((d, parts[i].to_string_lossy().into_owned())); - } - let winner = - rehome_from_dirs(&mut item, &dirs, &ctx, threshold, &mut cache, true); - if let Some(winner) = winner { - if let Ok(inner) = f - .parent() - .unwrap_or_else(|| Path::new("")) - .strip_prefix(&winner) - { - inner_rel = Some(inner.to_path_buf()); - } + if let Some(ir) = &inner_rel + && ir.iter().count() > 0 + { + // extracted archives: inner dir names may claim subtrees + let mut dirs: Vec<(PathBuf, String)> = Vec::new(); + let parts: Vec = ir.iter().map(|p| p.to_os_string()).collect(); + for i in (0..parts.len()).rev() { + let mut d = rootp.unwrap().join(&dk); + for part in &parts[..=i] { + d = d.join(part); } + dirs.push((d, parts[i].to_string_lossy().into_owned())); + } + let winner = rehome_from_dirs(&mut item, &dirs, &ctx, threshold, &mut cache, true); + if let Some(winner) = winner + && let Ok(inner) = f + .parent() + .unwrap_or_else(|| Path::new("")) + .strip_prefix(&winner) + { + inner_rel = Some(inner.to_path_buf()); } } item.target = Some(build_target(&item, library, inner_rel.as_deref())); @@ -1551,10 +1535,10 @@ fn collect_relative_files(dir: &Path, out: &mut Vec) { if p.is_dir() { let rel = p.strip_prefix(dir).unwrap(); collect_relative_files_prefixed(&p, rel, out); - } else if p.is_file() { - if let Ok(rel) = p.strip_prefix(dir) { - out.push(rel.to_string_lossy().into_owned()); - } + } else if p.is_file() + && let Ok(rel) = p.strip_prefix(dir) + { + out.push(rel.to_string_lossy().into_owned()); } } } @@ -1624,21 +1608,39 @@ pub fn apply_plan( skipped += 1; continue; } - if let Some(parent) = dest.parent() { - fs::create_dir_all(parent)?; + if let Some(parent) = dest.parent() + && let Err(e) = fs::create_dir_all(parent) + { + log::error!("Cannot create {}: {e}", parent.display()); + skipped += 1; + continue; } - if same_device(src, dest.parent().unwrap()) { - fs::hard_link(src, &dest)?; - item.action = "link".into(); - linked += 1; + // A single unplaceable file must not abort the whole apply: log it, + // count it as skipped, and keep going (the download pipeline has the + // same never-abort contract). + let placed = if same_device(src, dest.parent().unwrap()) { + fs::hard_link(src, &dest).map(|()| true) } else { log::warn!( "Different filesystem - COPYING instead of hardlinking {}", item.name ); - fs::copy(src, &dest)?; - item.action = "copy".into(); - copied += 1; + fs::copy(src, &dest).map(|_| false) + }; + match placed { + Ok(true) => { + item.action = "link".into(); + linked += 1; + } + Ok(false) => { + item.action = "copy".into(); + copied += 1; + } + Err(e) => { + log::error!("Cannot place {} -> {}: {e}", src.display(), dest.display()); + skipped += 1; + continue; + } } if item.match_kind == "weak" { weak += 1; diff --git a/src/state.rs b/src/state.rs index 585ae97..1d02a8e 100644 --- a/src/state.rs +++ b/src/state.rs @@ -95,6 +95,7 @@ impl State { } /// The URLs from `urls` whose status is not "ok". + #[allow(dead_code)] // python `pending`; kept as a store capability pub fn pending<'a>(&self, urls: &'a [String]) -> Vec<&'a str> { urls.iter() .map(String::as_str) @@ -131,6 +132,16 @@ impl State { Ok(self.records.get(url).expect("entry just inserted")) } + /// Like [`mark`](Self::mark), but a save failure is logged instead of + /// propagated: recording one URL's outcome must never abort the whole run + /// (AGENTS.md §6.5). The in-memory record is still updated, so a later + /// successful save persists it. + pub fn mark_logged(&mut self, url: &str, f: impl FnOnce(&mut Record)) { + if let Err(e) = self.mark(url, f) { + log::error!("Could not persist state for {url}: {e}"); + } + } + /// Atomic save: write `*.json.tmp`, then rename over state.json. pub fn save(&self) -> anyhow::Result<()> { save_sorted(&self.path, &self.records) diff --git a/src/warhammer_data.rs b/src/warhammer_data.rs index a89e9eb..2e51e35 100644 --- a/src/warhammer_data.rs +++ b/src/warhammer_data.rs @@ -37,6 +37,9 @@ struct Raw { } pub struct Data { + /// Parsed SYSTEMS table (kept for tests/inspection; matching uses + /// `faction_aliases` below). + #[allow(dead_code)] pub systems: Systems, /// system -> (alias, faction), insertion order of first occurrence, /// value = last faction seen (python dict semantics).