feat(rust): port dlp.py orchestration + unified CLI (clap subcommands)
- orchestrate.rs: process_channel, run_gofile (re-auth on AuthError, no-retry branches for dead/password/empty, cooldowns), run_0807, subscriptions, --assume-done (with note), failed_urls export - main.rs: dlp / catalogue / sorter / extract / gwshop subcommands, token resolution (--token -> $DISCORD_TOKEN -> hidden prompt) - globalstore: mark() gains a note field (python note= kwarg) - README: port status, CLI, deviations, untested live paths 77 tests green.
This commit is contained in:
@@ -101,13 +101,15 @@ impl GlobalStore {
|
||||
}
|
||||
|
||||
/// Record a URL as done (plus attribution fields when known), then save.
|
||||
/// No-ops when the store is disabled.
|
||||
/// No-ops when the store is disabled. `note` mirrors the python `note=`
|
||||
/// kwarg (e.g. "assumed done via --assume-done") and lands in `extra`.
|
||||
pub fn mark(
|
||||
&mut self,
|
||||
url: &str,
|
||||
channel: Option<&str>,
|
||||
host: Option<&str>,
|
||||
file_count: Option<u64>,
|
||||
note: Option<&str>,
|
||||
) -> anyhow::Result<()> {
|
||||
if self.path.is_none() {
|
||||
return Ok(());
|
||||
@@ -123,6 +125,9 @@ impl GlobalStore {
|
||||
if let Some(n) = file_count {
|
||||
rec.extra.insert("file_count".into(), Value::from(n));
|
||||
}
|
||||
if let Some(note) = note {
|
||||
rec.extra.insert("note".into(), Value::from(note));
|
||||
}
|
||||
rec.updated = Some(now_local_iso());
|
||||
self.save()
|
||||
}
|
||||
@@ -143,7 +148,8 @@ mod tests {
|
||||
#[test]
|
||||
fn disabled_store_is_a_noop() {
|
||||
let mut gs = GlobalStore::disabled();
|
||||
gs.mark("u", Some("c"), Some("gofile"), Some(2)).unwrap();
|
||||
gs.mark("u", Some("c"), Some("gofile"), Some(2), None)
|
||||
.unwrap();
|
||||
assert!(gs.records.is_empty());
|
||||
}
|
||||
|
||||
@@ -153,9 +159,9 @@ mod tests {
|
||||
let path = dir.path().join("urls.json");
|
||||
{
|
||||
let mut gs = GlobalStore::load(&path);
|
||||
gs.mark("u1", Some("chan"), Some("gofile"), Some(3))
|
||||
gs.mark("u1", Some("chan"), Some("gofile"), Some(3), None)
|
||||
.unwrap();
|
||||
gs.mark("u2", None, None, None).unwrap();
|
||||
gs.mark("u2", None, None, None, None).unwrap();
|
||||
}
|
||||
let gs = GlobalStore::load(&path);
|
||||
assert!(gs.is_done("u1"));
|
||||
|
||||
+615
-2
@@ -1,3 +1,21 @@
|
||||
//! dlphelper CLI — Rust rewrite of the python entry points:
|
||||
//!
|
||||
//! - `dlphelper dlp ...` <- version-python/dlp.py
|
||||
//! - `dlphelper catalogue ...` <- version-python/catalogue.py
|
||||
//! - `dlphelper sorter ...` <- version-python/sorter.py
|
||||
//! - `dlphelper extract ...` <- version-python/tools/extract_all.py
|
||||
//! - `dlphelper gwshop ...` <- version-python/dlphelper/gwshop.py
|
||||
//!
|
||||
//! Secrets rule: the Discord token is resolved via --token / $DISCORD_TOKEN /
|
||||
//! hidden prompt and only ever handed to the DCE container via env.
|
||||
|
||||
use std::io::IsTerminal;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::exit;
|
||||
|
||||
use anyhow::Context as _;
|
||||
use clap::{Parser, Subcommand};
|
||||
|
||||
mod catalogue;
|
||||
mod difflib;
|
||||
mod discord;
|
||||
@@ -13,7 +31,602 @@ mod util;
|
||||
mod warhammer_data;
|
||||
mod zero807;
|
||||
|
||||
#[derive(Parser)]
|
||||
#[command(
|
||||
name = "dlphelper",
|
||||
version,
|
||||
about = "Collect, catalogue and sort Warhammer downloads (Rust rewrite)."
|
||||
)]
|
||||
struct Cli {
|
||||
#[command(subcommand)]
|
||||
cmd: Cmd,
|
||||
}
|
||||
|
||||
#[derive(Subcommand)]
|
||||
enum Cmd {
|
||||
/// Collect downloads from Discord channels (gofile + 0807).
|
||||
Dlp(DlpArgs),
|
||||
/// Classify the collection into system / faction / class.
|
||||
Catalogue(CatalogueArgs),
|
||||
/// Fuzzysort collection files into the library (plan/apply).
|
||||
Sorter(SorterArgs),
|
||||
/// Extract archives into per-source directories.
|
||||
Extract(ExtractArgs),
|
||||
/// Dump the GW webshop catalog via Algolia.
|
||||
Gwshop(GwshopArgs),
|
||||
}
|
||||
|
||||
#[derive(clap::Args)]
|
||||
struct DlpArgs {
|
||||
/// Discord channel or thread id
|
||||
channel_id: Option<String>,
|
||||
/// directory for downloads and state
|
||||
out_dir: Option<PathBuf>,
|
||||
/// process a subscription list instead of one channel
|
||||
/// (entries: channel_id, out_dir, optional name)
|
||||
#[arg(long, value_name = "FILE")]
|
||||
subs: Option<PathBuf>,
|
||||
/// Discord token (else $DISCORD_TOKEN or prompt)
|
||||
#[arg(long)]
|
||||
token: Option<String>,
|
||||
/// skip the Discord export; read URLs from this file (e.g. failed_urls.txt)
|
||||
#[arg(long, value_name = "FILE")]
|
||||
urls_file: Option<PathBuf>,
|
||||
/// password for password-protected gofile links
|
||||
#[arg(long)]
|
||||
password: Option<String>,
|
||||
/// attempts per URL (default 3)
|
||||
#[arg(long, default_value_t = 3)]
|
||||
retries: u32,
|
||||
/// base seconds between retries, multiplied per attempt (default 5)
|
||||
#[arg(long, default_value_t = 5)]
|
||||
retry_delay: u64,
|
||||
/// consecutive failures before a cooldown pause (default 3)
|
||||
#[arg(long, default_value_t = 3)]
|
||||
cooldown_failures: u32,
|
||||
/// length of the cooldown pause (default 60)
|
||||
#[arg(long, default_value_t = 60)]
|
||||
cooldown_seconds: u64,
|
||||
/// process only one host class
|
||||
#[arg(long)]
|
||||
only: Option<String>,
|
||||
/// download at most N pending URLs per host class (test runs)
|
||||
#[arg(long, value_name = "N")]
|
||||
limit: Option<usize>,
|
||||
/// randomize which URLs a --limit picks
|
||||
#[arg(long)]
|
||||
shuffle: bool,
|
||||
/// record sha256 of each downloaded file
|
||||
#[arg(long)]
|
||||
hash: bool,
|
||||
/// unzip downloaded archives into per-archive folders
|
||||
#[arg(long)]
|
||||
extract: bool,
|
||||
/// re-export the Discord channel even if export.json exists
|
||||
/// (subs mode does this by default)
|
||||
#[arg(long)]
|
||||
fresh: bool,
|
||||
/// subs mode: reuse cached exports instead of re-exporting
|
||||
#[arg(long)]
|
||||
no_fresh: bool,
|
||||
/// export/extract/classify only, download nothing
|
||||
#[arg(long)]
|
||||
dry_run: bool,
|
||||
/// cross-run URL store FILE (default: $XDG_DATA_HOME/dlphelper/urls.json
|
||||
/// or ~/.local/share/dlphelper/urls.json)
|
||||
#[arg(long, value_name = "FILE")]
|
||||
global_state: Option<PathBuf>,
|
||||
/// disable the cross-run URL store for this run
|
||||
#[arg(long)]
|
||||
no_global: bool,
|
||||
/// mark every URL in the file as already downloaded, then exit
|
||||
/// (manual import of old download lists)
|
||||
#[arg(long, value_name = "FILE")]
|
||||
assume_done: Option<PathBuf>,
|
||||
}
|
||||
|
||||
#[derive(clap::Args)]
|
||||
struct CatalogueArgs {
|
||||
/// a dlp.py run dir (uses its manifest.jsonl and downloads/)
|
||||
run_dir: Option<PathBuf>,
|
||||
/// additional directory to scan for archive files (repeatable)
|
||||
#[arg(long, value_name = "DIR")]
|
||||
scan: Vec<PathBuf>,
|
||||
/// explicit manifest.jsonl (default: <run_dir>/manifest.jsonl)
|
||||
#[arg(long, value_name = "FILE")]
|
||||
manifest: Option<PathBuf>,
|
||||
/// manual classification overrides (JSON)
|
||||
#[arg(long, value_name = "FILE")]
|
||||
overrides: Option<PathBuf>,
|
||||
#[arg(long, value_name = "FILE")]
|
||||
out_md: Option<PathBuf>,
|
||||
#[arg(long, value_name = "FILE")]
|
||||
out_jsonl: Option<PathBuf>,
|
||||
/// move files into the library structure (default: preview)
|
||||
#[arg(long)]
|
||||
apply: bool,
|
||||
/// library root for --apply (required with --apply)
|
||||
#[arg(long, value_name = "DIR")]
|
||||
library: Option<PathBuf>,
|
||||
}
|
||||
|
||||
#[derive(clap::Args)]
|
||||
struct SorterArgs {
|
||||
/// a directory to sort or a single file
|
||||
source: PathBuf,
|
||||
/// library root the files will be linked into
|
||||
#[arg(long, value_name = "DIR")]
|
||||
library: PathBuf,
|
||||
/// catalog FILE (default: version-python/data/gwshop_catalog.json,
|
||||
/// falling back to data/gwshop_catalog.json, both relative to the cwd)
|
||||
#[arg(long, value_name = "FILE")]
|
||||
catalog: Option<PathBuf>,
|
||||
/// fuzzy match threshold (default 0.62)
|
||||
#[arg(long, default_value_t = sorter::DEFAULT_THRESHOLD)]
|
||||
threshold: f64,
|
||||
/// create the targets (hardlink, fallback copy)
|
||||
#[arg(long)]
|
||||
apply: bool,
|
||||
/// audit FILE (default: <library>/../_logs/sorter_applied.jsonl)
|
||||
#[arg(long, value_name = "FILE")]
|
||||
audit: Option<PathBuf>,
|
||||
/// extract_manifest.jsonl from extract: sort whole extraction
|
||||
/// directories as units
|
||||
#[arg(long, value_name = "FILE")]
|
||||
provenance: Option<PathBuf>,
|
||||
/// print the plan as JSON instead of text
|
||||
#[arg(long)]
|
||||
json: bool,
|
||||
/// never place into this game system (default-dropped: Middle-Earth)
|
||||
#[arg(long, value_name = "NAME")]
|
||||
drop_system: Vec<String>,
|
||||
/// re-enable a default-dropped game system
|
||||
#[arg(long, value_name = "NAME")]
|
||||
keep_system: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(clap::Args)]
|
||||
struct ExtractArgs {
|
||||
/// archives or directories containing archives
|
||||
sources: Vec<PathBuf>,
|
||||
/// directory to extract into
|
||||
#[arg(long, required = true, value_name = "DIR")]
|
||||
out: PathBuf,
|
||||
}
|
||||
|
||||
#[derive(clap::Args)]
|
||||
struct GwshopArgs {
|
||||
/// output catalog JSON path
|
||||
out: PathBuf,
|
||||
/// Algolia index name
|
||||
#[arg(long, default_value = gwshop::DEFAULT_INDEX)]
|
||||
index: String,
|
||||
/// also write the raw Algolia hits here
|
||||
#[arg(long, value_name = "FILE")]
|
||||
raw: Option<PathBuf>,
|
||||
}
|
||||
|
||||
/// argparse parser.error equivalent: message to stderr, exit code 2.
|
||||
fn parser_error(msg: &str) -> ! {
|
||||
eprintln!("error: {msg}");
|
||||
exit(2);
|
||||
}
|
||||
|
||||
/// python resolve_token: --token, then $DISCORD_TOKEN, then hidden prompt.
|
||||
fn resolve_token(cli_value: Option<&str>) -> anyhow::Result<String> {
|
||||
if let Some(t) = cli_value {
|
||||
return Ok(t.to_string());
|
||||
}
|
||||
if let Ok(t) = std::env::var("DISCORD_TOKEN") {
|
||||
if !t.is_empty() {
|
||||
return Ok(t);
|
||||
}
|
||||
}
|
||||
log::info!("No Discord token given ($DISCORD_TOKEN); prompting (input hidden).");
|
||||
if !std::io::stdin().is_terminal() {
|
||||
anyhow::bail!(
|
||||
"No Discord token available and stdin is not a terminal; \
|
||||
use --token or $DISCORD_TOKEN."
|
||||
);
|
||||
}
|
||||
let token = rpassword::prompt_password("Discord token: ").context("reading the token")?;
|
||||
if token.is_empty() {
|
||||
anyhow::bail!("No token provided, aborting.");
|
||||
}
|
||||
Ok(token)
|
||||
}
|
||||
|
||||
/// The global store for this run, honoring --no-global / --global-state.
|
||||
fn make_gstore(args: &DlpArgs) -> globalstore::GlobalStore {
|
||||
if args.no_global {
|
||||
globalstore::GlobalStore::disabled()
|
||||
} else if let Some(p) = &args.global_state {
|
||||
globalstore::GlobalStore::load(p)
|
||||
} else {
|
||||
match globalstore::default_global_path() {
|
||||
Some(p) => globalstore::GlobalStore::load(p),
|
||||
None => globalstore::GlobalStore::disabled(), // no home dir: like python gstore=None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn default_catalog_path() -> PathBuf {
|
||||
let v = PathBuf::from("version-python/data/gwshop_catalog.json");
|
||||
if v.exists() {
|
||||
return v;
|
||||
}
|
||||
let alt = PathBuf::from("data/gwshop_catalog.json");
|
||||
if alt.exists() {
|
||||
return alt;
|
||||
}
|
||||
v // surfaces a clear error in load_catalog
|
||||
}
|
||||
|
||||
fn cmd_dlp(args: &DlpArgs) -> anyhow::Result<()> {
|
||||
if let Some(assume) = &args.assume_done {
|
||||
if args.channel_id.is_some() || args.subs.is_some() {
|
||||
parser_error("--assume-done cannot be combined with a channel or --subs");
|
||||
}
|
||||
let mut gstore = make_gstore(args);
|
||||
orchestrate::mark_assume_done(assume, &mut gstore)?;
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if let Some(subs_path) = &args.subs {
|
||||
if args.channel_id.is_some() || args.out_dir.is_some() || args.urls_file.is_some() {
|
||||
parser_error("--subs cannot be combined with channel_id/out_dir/--urls-file");
|
||||
}
|
||||
let subs = orchestrate::load_subscriptions(subs_path)?;
|
||||
let token = resolve_token(args.token.as_deref())?;
|
||||
let mut gstore = make_gstore(args);
|
||||
let mut total_failed = 0u64;
|
||||
for (channel_id, out_dir, name) in &subs {
|
||||
log::info!(
|
||||
"=== subscription: {} ===",
|
||||
name.clone().unwrap_or_else(|| channel_id.clone())
|
||||
);
|
||||
let opts = orchestrate::RunOpts::from(args);
|
||||
// subs mode re-exports by default (python force_export=not --no-fresh)
|
||||
total_failed += orchestrate::process_channel(
|
||||
channel_id,
|
||||
Path::new(out_dir),
|
||||
&opts,
|
||||
Some(&token),
|
||||
Some(&mut gstore),
|
||||
None,
|
||||
!args.no_fresh,
|
||||
)?;
|
||||
}
|
||||
if total_failed > 0 {
|
||||
log::warn!("{total_failed} URL(s) failed in total across subscriptions.");
|
||||
exit(1);
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let (Some(channel_id), Some(out_dir)) = (&args.channel_id, &args.out_dir) else {
|
||||
parser_error("channel_id and out_dir are required (or use --subs/--assume-done)");
|
||||
};
|
||||
// A token is only needed when we will export a channel (--urls-file skips that)
|
||||
let token = if args.urls_file.is_some() {
|
||||
None
|
||||
} else {
|
||||
Some(resolve_token(args.token.as_deref())?)
|
||||
};
|
||||
let opts = orchestrate::RunOpts::from(args);
|
||||
let mut gstore = make_gstore(args);
|
||||
let failures = orchestrate::process_channel(
|
||||
channel_id,
|
||||
out_dir,
|
||||
&opts,
|
||||
token.as_deref(),
|
||||
Some(&mut gstore),
|
||||
args.urls_file.as_deref(),
|
||||
false,
|
||||
)?;
|
||||
if failures > 0 {
|
||||
exit(1);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
impl From<&DlpArgs> for orchestrate::RunOpts {
|
||||
fn from(a: &DlpArgs) -> Self {
|
||||
orchestrate::RunOpts {
|
||||
retries: a.retries,
|
||||
retry_delay: a.retry_delay,
|
||||
cooldown_failures: a.cooldown_failures,
|
||||
cooldown_seconds: a.cooldown_seconds,
|
||||
hash: a.hash,
|
||||
extract: a.extract,
|
||||
password: a.password.clone(),
|
||||
limit: a.limit,
|
||||
shuffle: a.shuffle,
|
||||
only: a.only.clone(),
|
||||
dry_run: a.dry_run,
|
||||
fresh: a.fresh,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn cmd_catalogue(args: &CatalogueArgs) -> anyhow::Result<()> {
|
||||
let run_dir = args.run_dir.as_deref();
|
||||
let manifests: Vec<PathBuf> = if let Some(m) = &args.manifest {
|
||||
vec![m.clone()]
|
||||
} else {
|
||||
run_dir
|
||||
.map(|d| vec![d.join("manifest.jsonl")])
|
||||
.unwrap_or_default()
|
||||
};
|
||||
let mut scan_dirs = args.scan.clone();
|
||||
if let Some(rd) = run_dir {
|
||||
if args.manifest.is_none() {
|
||||
// run dir is scanned as a whole (covers files not yet in the manifest)
|
||||
scan_dirs.push(rd.to_path_buf());
|
||||
}
|
||||
}
|
||||
|
||||
if manifests.is_empty() && scan_dirs.is_empty() {
|
||||
parser_error("give a run_dir or at least one --scan directory");
|
||||
}
|
||||
|
||||
let out_base: PathBuf = run_dir
|
||||
.map(Path::to_path_buf)
|
||||
.unwrap_or_else(|| std::env::current_dir().unwrap_or_else(|_| PathBuf::from(".")));
|
||||
let out_md = args
|
||||
.out_md
|
||||
.clone()
|
||||
.unwrap_or_else(|| out_base.join("catalogue.md"));
|
||||
let out_jsonl = args
|
||||
.out_jsonl
|
||||
.clone()
|
||||
.unwrap_or_else(|| out_base.join("catalogue.jsonl"));
|
||||
|
||||
let mut entries =
|
||||
catalogue::build_catalogue(&manifests, &scan_dirs, args.overrides.as_deref())?;
|
||||
catalogue::write_reports(&entries, &out_md, &out_jsonl)?;
|
||||
|
||||
let classified = entries
|
||||
.iter()
|
||||
.filter(|e| e.faction != catalogue::UNCLASSIFIED_FACTION)
|
||||
.count();
|
||||
log::info!(
|
||||
"Catalogued {} files ({} classified) -> {}",
|
||||
entries.len(),
|
||||
classified,
|
||||
out_md.display()
|
||||
);
|
||||
|
||||
// compact per-faction summary to the console
|
||||
let mut summary: std::collections::BTreeMap<(String, String), (u64, u64)> =
|
||||
std::collections::BTreeMap::new();
|
||||
for e in &entries {
|
||||
let entry = summary
|
||||
.entry((e.system.clone(), e.faction.clone()))
|
||||
.or_insert((0, 0));
|
||||
entry.0 += 1;
|
||||
entry.1 += e.size;
|
||||
}
|
||||
for ((system, faction), (n, size)) in &summary {
|
||||
log::info!(
|
||||
" {:<18} {:<32} {:>4} files {}",
|
||||
system,
|
||||
faction,
|
||||
n,
|
||||
catalogue::format_size(*size)
|
||||
);
|
||||
}
|
||||
|
||||
if args.apply {
|
||||
let Some(library) = &args.library else {
|
||||
parser_error("--apply needs --library DIR");
|
||||
};
|
||||
let Some(run_dir) = run_dir else {
|
||||
parser_error("--apply needs a run_dir (files are moved out of it)");
|
||||
};
|
||||
let moved = catalogue::apply_moves(
|
||||
&mut entries,
|
||||
library,
|
||||
&library.join("catalogue_moves.jsonl"),
|
||||
)?;
|
||||
log::info!(
|
||||
"Moved {} files into {} (audit: catalogue_moves.jsonl)",
|
||||
moved,
|
||||
library.display()
|
||||
);
|
||||
// rewrite reports with updated paths
|
||||
catalogue::write_reports(&entries, &out_md, &out_jsonl)?;
|
||||
let _ = run_dir;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn load_provenance(path: &Path) -> anyhow::Result<sorter::Provenance> {
|
||||
use std::io::BufRead;
|
||||
let mut prov = sorter::Provenance::new();
|
||||
let fh = std::fs::File::open(path).with_context(|| format!("reading {}", path.display()))?;
|
||||
for line in std::io::BufReader::new(fh).lines() {
|
||||
let rec: serde_json::Value = serde_json::from_str(&line?)?;
|
||||
let dest = rec["dest"]
|
||||
.as_str()
|
||||
.context("provenance row lacks 'dest'")?;
|
||||
let unit = rec["unit"]
|
||||
.as_str()
|
||||
.context("provenance row lacks 'unit'")?;
|
||||
prov.insert(dest.to_string(), unit.to_string());
|
||||
}
|
||||
Ok(prov)
|
||||
}
|
||||
|
||||
fn cmd_sorter(args: &SorterArgs) -> anyhow::Result<()> {
|
||||
let dropped = sorter::dropped_systems_default(&args.keep_system, &args.drop_system);
|
||||
if !args.keep_system.is_empty() || !args.drop_system.is_empty() {
|
||||
log::info!(
|
||||
"System filter adjusted: dropping {}",
|
||||
if dropped.is_empty() {
|
||||
"nothing".to_string()
|
||||
} else {
|
||||
let mut s = dropped.clone();
|
||||
s.sort();
|
||||
s.join(", ")
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
let source = &args.source;
|
||||
let files: Vec<PathBuf> = if source.is_file() {
|
||||
vec![source.clone()]
|
||||
} else {
|
||||
// hidden trees like .broken/ must never reach the plan
|
||||
let mut out = Vec::new();
|
||||
collect_visible(source, source, &mut out)?;
|
||||
out.sort();
|
||||
out
|
||||
};
|
||||
let catalog_path = args.catalog.clone().unwrap_or_else(default_catalog_path);
|
||||
let catalog = sorter::load_catalog(&catalog_path, &dropped)?;
|
||||
if !dropped.is_empty() {
|
||||
let mut s = dropped.clone();
|
||||
s.sort();
|
||||
log::info!("Dropped systems: {}", s.join(", "));
|
||||
}
|
||||
// Bookkeeping must never land inside the library itself (it may be a
|
||||
// seeded torrent whose file set is frozen): default to a _logs folder
|
||||
// beside it. An explicit --audit overrides this.
|
||||
let audit = args.audit.clone().unwrap_or_else(|| {
|
||||
args.library
|
||||
.parent()
|
||||
.unwrap_or(Path::new("."))
|
||||
.join("_logs")
|
||||
.join("sorter_applied.jsonl")
|
||||
});
|
||||
log::info!("Audit file: {}", audit.display());
|
||||
|
||||
let provenance = match &args.provenance {
|
||||
Some(p) => {
|
||||
let prov = load_provenance(p)?;
|
||||
log::info!("Provenance: {} extraction directories", prov.len());
|
||||
Some(prov)
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
|
||||
let mut items = sorter::sort_plan(
|
||||
&files,
|
||||
&args.library,
|
||||
&catalog,
|
||||
args.threshold,
|
||||
provenance.as_ref(),
|
||||
Some(source),
|
||||
);
|
||||
|
||||
if args.json {
|
||||
let rows: Vec<serde_json::Value> = items
|
||||
.iter()
|
||||
.map(|i| {
|
||||
serde_json::json!({
|
||||
"source": i.path.display().to_string(),
|
||||
"name": i.name,
|
||||
"size": i.size,
|
||||
"system": i.system,
|
||||
"group": i.group,
|
||||
"faction": i.faction,
|
||||
"subfaction": i.subfaction,
|
||||
"class": i.klass,
|
||||
"unit": i.unit,
|
||||
"score": (i.match_score * 1000.0).round() / 1000.0,
|
||||
"kind": i.match_kind,
|
||||
"inner_count": i.inner_names.len(),
|
||||
"target": i.target
|
||||
.as_ref()
|
||||
.map(|t| t.display().to_string())
|
||||
.unwrap_or_else(|| "None".into()),
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
println!("{}", serde_json::to_string_pretty(&rows)?);
|
||||
} else {
|
||||
println!("{}", sorter::format_plan(&items));
|
||||
}
|
||||
|
||||
let n_catalog = items.iter().filter(|i| i.match_kind == "catalog").count();
|
||||
let n_implicit = items.iter().filter(|i| i.match_kind == "implicit").count();
|
||||
let n_none = items.iter().filter(|i| i.match_kind == "none").count();
|
||||
log::info!(
|
||||
"Plan: {} files -> {} catalog matches [+] / {} implicit [~] / {} unclassifiable [?]",
|
||||
items.len(),
|
||||
n_catalog,
|
||||
n_implicit,
|
||||
n_none
|
||||
);
|
||||
|
||||
if args.apply {
|
||||
let (linked, copied, skipped) = sorter::apply_plan(&mut items, &audit)?;
|
||||
log::info!(
|
||||
"Applied: {} hardlinked, {} copied, {} skipped (audit: {})",
|
||||
linked,
|
||||
copied,
|
||||
skipped,
|
||||
audit.display()
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Recursively collect files under `root`, skipping any path whose parts
|
||||
/// (relative to root) start with '.' (python rglob filter).
|
||||
fn collect_visible(root: &Path, dir: &Path, out: &mut Vec<PathBuf>) -> std::io::Result<()> {
|
||||
for entry in std::fs::read_dir(dir)? {
|
||||
let entry = entry?;
|
||||
let path = entry.path();
|
||||
let name = entry.file_name().to_string_lossy().into_owned();
|
||||
if name.starts_with('.') {
|
||||
continue;
|
||||
}
|
||||
if path.is_dir() {
|
||||
collect_visible(root, &path, out)?;
|
||||
} else {
|
||||
out.push(path);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn cmd_extract(args: &ExtractArgs) -> anyhow::Result<()> {
|
||||
let ex = extract::Extractor {
|
||||
out_root: args.out.clone(),
|
||||
};
|
||||
ex.run(&args.sources)
|
||||
}
|
||||
|
||||
fn cmd_gwshop(args: &GwshopArgs) -> anyhow::Result<()> {
|
||||
gwshop::run_cli(&args.out, &args.index, args.raw.as_deref())
|
||||
}
|
||||
|
||||
fn main() {
|
||||
env_logger::init();
|
||||
println!("dlphelper {}", env!("CARGO_PKG_VERSION"));
|
||||
env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info"))
|
||||
.format(|buf, record| {
|
||||
use std::io::Write as _;
|
||||
writeln!(
|
||||
buf,
|
||||
"{} {:<7} {}",
|
||||
chrono::Local::now().format("%H:%M:%S"),
|
||||
record.level(),
|
||||
record.args()
|
||||
)
|
||||
})
|
||||
.init();
|
||||
|
||||
let cli = Cli::parse();
|
||||
let res = match &cli.cmd {
|
||||
Cmd::Dlp(args) => cmd_dlp(args),
|
||||
Cmd::Catalogue(args) => cmd_catalogue(args),
|
||||
Cmd::Sorter(args) => cmd_sorter(args),
|
||||
Cmd::Extract(args) => cmd_extract(args),
|
||||
Cmd::Gwshop(args) => cmd_gwshop(args),
|
||||
};
|
||||
if let Err(err) = res {
|
||||
log::error!("{err:#}");
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -19,6 +19,7 @@ use std::fs;
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use anyhow::Context as _;
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::catalogue::{clean_name, match_class, match_faction};
|
||||
@@ -446,7 +447,8 @@ pub fn drop_dropped_system(item: &mut SortItem) {
|
||||
/// (pass an empty dropped set to keep everything).
|
||||
/// Accepts the catalog document ({..., "entries": [...]}) or a bare list.
|
||||
pub fn load_catalog(path: &Path, dropped: &[String]) -> anyhow::Result<Vec<CatalogEntry>> {
|
||||
let text = fs::read_to_string(path)?;
|
||||
let text =
|
||||
fs::read_to_string(path).with_context(|| format!("reading catalog {}", path.display()))?;
|
||||
let data: Value = serde_json::from_str(&text)?;
|
||||
let entries = match &data {
|
||||
Value::Object(o) => o.get("entries").cloned().unwrap_or(Value::Array(vec![])),
|
||||
|
||||
Reference in New Issue
Block a user