Warhammer_DLP_Helper/version-rust/src/main.rs

//! dlphelper CLI — Rust rewrite of the python entry points:
//!
//! - `dlphelper dlp ...`          <- version-python/dlp.py
//! - `dlphelper catalogue ...`    <- version-python/catalogue.py
//! - `dlphelper sorter ...`       <- version-python/sorter.py
//! - `dlphelper extract ...`      <- version-python/tools/extract_all.py
//! - `dlphelper gwshop ...`       <- version-python/dlphelper/gwshop.py
//!
//! Secrets rule: the Discord token is resolved via --token / $DISCORD_TOKEN /
//! hidden prompt and only ever handed to the DCE container via env.

use std::io::IsTerminal;
use std::path::{Path, PathBuf};
use std::process::exit;

use anyhow::Context as _;
use clap::{Parser, Subcommand};

mod catalogue;
mod difflib;
mod discord;
mod extract;
mod globalstore;
mod gofile;
mod gwshop;
mod manifest;
mod orchestrate;
mod sorter;
mod state;
mod util;
mod warhammer_data;
mod zero807;

#[derive(Parser)]
#[command(
    name = "dlphelper",
    version,
    about = "Collect, catalogue and sort Warhammer downloads (Rust rewrite)."
)]
struct Cli {
    #[command(subcommand)]
    cmd: Cmd,
}

#[derive(Subcommand)]
enum Cmd {
    /// Collect downloads from Discord channels (gofile + 0807).
    Dlp(DlpArgs),
    /// Classify the collection into system / faction / class.
    Catalogue(CatalogueArgs),
    /// Fuzzysort collection files into the library (plan/apply).
    Sorter(SorterArgs),
    /// Extract archives into per-source directories.
    Extract(ExtractArgs),
    /// Dump the GW webshop catalog via Algolia.
    Gwshop(GwshopArgs),
}

#[derive(clap::Args)]
struct DlpArgs {
    /// Discord channel or thread id
    channel_id: Option<String>,
    /// directory for downloads and state
    out_dir: Option<PathBuf>,
    /// process a subscription list instead of one channel
    /// (entries: channel_id, out_dir, optional name)
    #[arg(long, value_name = "FILE")]
    subs: Option<PathBuf>,
    /// Discord token (else $DISCORD_TOKEN or prompt)
    #[arg(long)]
    token: Option<String>,
    /// skip the Discord export; read URLs from this file (e.g. failed_urls.txt)
    #[arg(long, value_name = "FILE")]
    urls_file: Option<PathBuf>,
    /// password for password-protected gofile links
    #[arg(long)]
    password: Option<String>,
    /// attempts per URL (default 3)
    #[arg(long, default_value_t = 3)]
    retries: u32,
    /// base seconds between retries, multiplied per attempt (default 5)
    #[arg(long, default_value_t = 5)]
    retry_delay: u64,
    /// consecutive failures before a cooldown pause (default 3)
    #[arg(long, default_value_t = 3)]
    cooldown_failures: u32,
    /// length of the cooldown pause (default 60)
    #[arg(long, default_value_t = 60)]
    cooldown_seconds: u64,
    /// process only one host class
    #[arg(long)]
    only: Option<String>,
    /// only URLs posted at/after this point: e.g. 5d, 12h, 2026-10-04
    #[arg(long, value_name = "WHEN")]
    since: Option<String>,
    /// download at most N pending URLs per host class (test runs)
    #[arg(long, value_name = "N")]
    limit: Option<usize>,
    /// randomize which URLs a --limit picks
    #[arg(long)]
    shuffle: bool,
    /// record sha256 of each downloaded file
    #[arg(long)]
    hash: bool,
    /// unzip downloaded archives into per-archive folders
    #[arg(long)]
    extract: bool,
    /// re-export the Discord channel even if export.json exists
    /// (subs mode does this by default)
    #[arg(long)]
    fresh: bool,
    /// subs mode: reuse cached exports instead of re-exporting
    #[arg(long)]
    no_fresh: bool,
    /// export/extract/classify only, download nothing
    #[arg(long)]
    dry_run: bool,
    /// cross-run URL store FILE (default: $XDG_DATA_HOME/dlphelper/urls.json
    /// or ~/.local/share/dlphelper/urls.json)
    #[arg(long, value_name = "FILE")]
    global_state: Option<PathBuf>,
    /// disable the cross-run URL store for this run
    #[arg(long)]
    no_global: bool,
    /// mark every URL in the file as already downloaded, then exit
    /// (manual import of old download lists)
    #[arg(long, value_name = "FILE")]
    assume_done: Option<PathBuf>,
}

#[derive(clap::Args)]
struct CatalogueArgs {
    /// a dlp.py run dir (uses its manifest.jsonl and downloads/)
    run_dir: Option<PathBuf>,
    /// additional directory to scan for archive files (repeatable)
    #[arg(long, value_name = "DIR")]
    scan: Vec<PathBuf>,
    /// explicit manifest.jsonl (default: <run_dir>/manifest.jsonl)
    #[arg(long, value_name = "FILE")]
    manifest: Option<PathBuf>,
    /// manual classification overrides (JSON)
    #[arg(long, value_name = "FILE")]
    overrides: Option<PathBuf>,
    #[arg(long, value_name = "FILE")]
    out_md: Option<PathBuf>,
    #[arg(long, value_name = "FILE")]
    out_jsonl: Option<PathBuf>,
    /// move files into the library structure (default: preview)
    #[arg(long)]
    apply: bool,
    /// library root for --apply (required with --apply)
    #[arg(long, value_name = "DIR")]
    library: Option<PathBuf>,
}

#[derive(clap::Args)]
struct SorterArgs {
    /// a directory to sort or a single file
    source: PathBuf,
    /// library root the files will be linked into
    #[arg(long, value_name = "DIR")]
    library: PathBuf,
    /// catalog FILE (default: version-python/data/gwshop_catalog.json,
    /// falling back to data/gwshop_catalog.json, both relative to the cwd)
    #[arg(long, value_name = "FILE")]
    catalog: Option<PathBuf>,
    /// fuzzy match threshold (default 0.62)
    #[arg(long, default_value_t = sorter::DEFAULT_THRESHOLD)]
    threshold: f64,
    /// create the targets (hardlink, fallback copy)
    #[arg(long)]
    apply: bool,
    /// audit FILE (default: <library>/../_logs/sorter_applied.jsonl)
    #[arg(long, value_name = "FILE")]
    audit: Option<PathBuf>,
    /// extract_manifest.jsonl from extract: sort whole extraction
    /// directories as units
    #[arg(long, value_name = "FILE")]
    provenance: Option<PathBuf>,
    /// print the plan as JSON instead of text
    #[arg(long)]
    json: bool,
    /// never place into this game system (default-dropped: Middle-Earth)
    #[arg(long, value_name = "NAME")]
    drop_system: Vec<String>,
    /// re-enable a default-dropped game system
    #[arg(long, value_name = "NAME")]
    keep_system: Vec<String>,
}

#[derive(clap::Args)]
struct ExtractArgs {
    /// archives or directories containing archives
    sources: Vec<PathBuf>,
    /// directory to extract into
    #[arg(long, required = true, value_name = "DIR")]
    out: PathBuf,
}

#[derive(clap::Args)]
struct GwshopArgs {
    /// output catalog JSON path
    out: PathBuf,
    /// Algolia index name
    #[arg(long, default_value = gwshop::DEFAULT_INDEX)]
    index: String,
    /// also write the raw Algolia hits here
    #[arg(long, value_name = "FILE")]
    raw: Option<PathBuf>,
}

/// argparse parser.error equivalent: message to stderr, exit code 2.
fn parser_error(msg: &str) -> ! {
    eprintln!("error: {msg}");
    exit(2);
}

/// python resolve_token: --token, then $DISCORD_TOKEN, then hidden prompt.
fn resolve_token(cli_value: Option<&str>) -> anyhow::Result<String> {
    if let Some(t) = cli_value {
        return Ok(t.to_string());
    }
    if let Ok(t) = std::env::var("DISCORD_TOKEN") {
        if !t.is_empty() {
            return Ok(t);
        }
    }
    log::info!("No Discord token given ($DISCORD_TOKEN); prompting (input hidden).");
    if !std::io::stdin().is_terminal() {
        anyhow::bail!(
            "No Discord token available and stdin is not a terminal; \
                      use --token or $DISCORD_TOKEN."
        );
    }
    let token = rpassword::prompt_password("Discord token: ").context("reading the token")?;
    if token.is_empty() {
        anyhow::bail!("No token provided, aborting.");
    }
    Ok(token)
}

/// The global store for this run, honoring --no-global / --global-state.
fn make_gstore(args: &DlpArgs) -> globalstore::GlobalStore {
    if args.no_global {
        globalstore::GlobalStore::disabled()
    } else if let Some(p) = &args.global_state {
        globalstore::GlobalStore::load(p)
    } else {
        match globalstore::default_global_path() {
            Some(p) => globalstore::GlobalStore::load(p),
            None => globalstore::GlobalStore::disabled(), // no home dir: like python gstore=None
        }
    }
}

fn default_catalog_path() -> PathBuf {
    let v = PathBuf::from("version-python/data/gwshop_catalog.json");
    if v.exists() {
        return v;
    }
    let alt = PathBuf::from("data/gwshop_catalog.json");
    if alt.exists() {
        return alt;
    }
    v // surfaces a clear error in load_catalog
}

fn cmd_dlp(args: &DlpArgs) -> anyhow::Result<()> {
    if let Some(assume) = &args.assume_done {
        if args.channel_id.is_some() || args.subs.is_some() {
            parser_error("--assume-done cannot be combined with a channel or --subs");
        }
        let mut gstore = make_gstore(args);
        orchestrate::mark_assume_done(assume, &mut gstore)?;
        return Ok(());
    }

    if let Some(subs_path) = &args.subs {
        if args.channel_id.is_some() || args.out_dir.is_some() || args.urls_file.is_some() {
            parser_error("--subs cannot be combined with channel_id/out_dir/--urls-file");
        }
        let subs = orchestrate::load_subscriptions(subs_path)?;
        let token = resolve_token(args.token.as_deref())?;
        let mut gstore = make_gstore(args);
        let mut total_failed = 0u64;
        for (channel_id, out_dir, name) in &subs {
            log::info!(
                "=== subscription: {} ===",
                name.clone().unwrap_or_else(|| channel_id.clone())
            );
            let opts = orchestrate::RunOpts::from(args);
            // subs mode re-exports by default (python force_export=not --no-fresh)
            total_failed += orchestrate::process_channel(
                channel_id,
                Path::new(out_dir),
                &opts,
                Some(&token),
                Some(&mut gstore),
                None,
                !args.no_fresh,
            )?;
        }
        if total_failed > 0 {
            log::warn!("{total_failed} URL(s) failed in total across subscriptions.");
            exit(1);
        }
        return Ok(());
    }

    let (Some(channel_id), Some(out_dir)) = (&args.channel_id, &args.out_dir) else {
        parser_error("channel_id and out_dir are required (or use --subs/--assume-done)");
    };
    // A token is only needed when we will export a channel (--urls-file skips that)
    let token = if args.urls_file.is_some() {
        None
    } else {
        Some(resolve_token(args.token.as_deref())?)
    };
    let opts = orchestrate::RunOpts::from(args);
    let mut gstore = make_gstore(args);
    let failures = orchestrate::process_channel(
        channel_id,
        out_dir,
        &opts,
        token.as_deref(),
        Some(&mut gstore),
        args.urls_file.as_deref(),
        false,
    )?;
    if failures > 0 {
        exit(1);
    }
    Ok(())
}

impl From<&DlpArgs> for orchestrate::RunOpts {
    fn from(a: &DlpArgs) -> Self {
        orchestrate::RunOpts {
            retries: a.retries,
            retry_delay: a.retry_delay,
            cooldown_failures: a.cooldown_failures,
            cooldown_seconds: a.cooldown_seconds,
            hash: a.hash,
            extract: a.extract,
            password: a.password.clone(),
            limit: a.limit,
            shuffle: a.shuffle,
            only: a.only.clone(),
            since: a.since.clone(),
            dry_run: a.dry_run,
            fresh: a.fresh,
        }
    }
}

fn cmd_catalogue(args: &CatalogueArgs) -> anyhow::Result<()> {
    let run_dir = args.run_dir.as_deref();
    let manifests: Vec<PathBuf> = if let Some(m) = &args.manifest {
        vec![m.clone()]
    } else {
        run_dir
            .map(|d| vec![d.join("manifest.jsonl")])
            .unwrap_or_default()
    };
    let mut scan_dirs = args.scan.clone();
    if let Some(rd) = run_dir {
        if args.manifest.is_none() {
            // run dir is scanned as a whole (covers files not yet in the manifest)
            scan_dirs.push(rd.to_path_buf());
        }
    }

    if manifests.is_empty() && scan_dirs.is_empty() {
        parser_error("give a run_dir or at least one --scan directory");
    }

    let out_base: PathBuf = run_dir
        .map(Path::to_path_buf)
        .unwrap_or_else(|| std::env::current_dir().unwrap_or_else(|_| PathBuf::from(".")));
    let out_md = args
        .out_md
        .clone()
        .unwrap_or_else(|| out_base.join("catalogue.md"));
    let out_jsonl = args
        .out_jsonl
        .clone()
        .unwrap_or_else(|| out_base.join("catalogue.jsonl"));

    let mut entries =
        catalogue::build_catalogue(&manifests, &scan_dirs, args.overrides.as_deref())?;
    catalogue::write_reports(&entries, &out_md, &out_jsonl)?;

    let classified = entries
        .iter()
        .filter(|e| e.faction != catalogue::UNCLASSIFIED_FACTION)
        .count();
    log::info!(
        "Catalogued {} files ({} classified) -> {}",
        entries.len(),
        classified,
        out_md.display()
    );

    // compact per-faction summary to the console
    let mut summary: std::collections::BTreeMap<(String, String), (u64, u64)> =
        std::collections::BTreeMap::new();
    for e in &entries {
        let entry = summary
            .entry((e.system.clone(), e.faction.clone()))
            .or_insert((0, 0));
        entry.0 += 1;
        entry.1 += e.size;
    }
    for ((system, faction), (n, size)) in &summary {
        log::info!(
            "  {:<18} {:<32} {:>4} files  {}",
            system,
            faction,
            n,
            catalogue::format_size(*size)
        );
    }

    if args.apply {
        let Some(library) = &args.library else {
            parser_error("--apply needs --library DIR");
        };
        let Some(run_dir) = run_dir else {
            parser_error("--apply needs a run_dir (files are moved out of it)");
        };
        let moved = catalogue::apply_moves(
            &mut entries,
            library,
            &library.join("catalogue_moves.jsonl"),
        )?;
        log::info!(
            "Moved {} files into {} (audit: catalogue_moves.jsonl)",
            moved,
            library.display()
        );
        // rewrite reports with updated paths
        catalogue::write_reports(&entries, &out_md, &out_jsonl)?;
        let _ = run_dir;
    }
    Ok(())
}

fn load_provenance(path: &Path) -> anyhow::Result<sorter::Provenance> {
    use std::io::BufRead;
    let mut prov = sorter::Provenance::new();
    let fh = std::fs::File::open(path).with_context(|| format!("reading {}", path.display()))?;
    for line in std::io::BufReader::new(fh).lines() {
        let rec: serde_json::Value = serde_json::from_str(&line?)?;
        let dest = rec["dest"]
            .as_str()
            .context("provenance row lacks 'dest'")?;
        let unit = rec["unit"]
            .as_str()
            .context("provenance row lacks 'unit'")?;
        prov.insert(dest.to_string(), unit.to_string());
    }
    Ok(prov)
}

fn cmd_sorter(args: &SorterArgs) -> anyhow::Result<()> {
    let dropped = sorter::dropped_systems_default(&args.keep_system, &args.drop_system);
    if !args.keep_system.is_empty() || !args.drop_system.is_empty() {
        log::info!(
            "System filter adjusted: dropping {}",
            if dropped.is_empty() {
                "nothing".to_string()
            } else {
                let mut s = dropped.clone();
                s.sort();
                s.join(", ")
            }
        );
    }

    let source = &args.source;
    let files: Vec<PathBuf> = if source.is_file() {
        vec![source.clone()]
    } else {
        // hidden trees like .broken/ must never reach the plan
        let mut out = Vec::new();
        collect_visible(source, source, &mut out)?;
        out.sort();
        out
    };
    let catalog_path = args.catalog.clone().unwrap_or_else(default_catalog_path);
    let catalog = sorter::load_catalog(&catalog_path, &dropped)?;
    if !dropped.is_empty() {
        let mut s = dropped.clone();
        s.sort();
        log::info!("Dropped systems: {}", s.join(", "));
    }
    // Bookkeeping must never land inside the library itself (it may be a
    // seeded torrent whose file set is frozen): default to a _logs folder
    // beside it. An explicit --audit overrides this.
    let audit = args.audit.clone().unwrap_or_else(|| {
        args.library
            .parent()
            .unwrap_or(Path::new("."))
            .join("_logs")
            .join("sorter_applied.jsonl")
    });
    log::info!("Audit file: {}", audit.display());

    let provenance = match &args.provenance {
        Some(p) => {
            let prov = load_provenance(p)?;
            log::info!("Provenance: {} extraction directories", prov.len());
            Some(prov)
        }
        None => None,
    };

    let mut items = sorter::sort_plan(
        &files,
        &args.library,
        &catalog,
        args.threshold,
        provenance.as_ref(),
        Some(source),
    );

    if args.json {
        let rows: Vec<serde_json::Value> = items
            .iter()
            .map(|i| {
                serde_json::json!({
                    "source": i.path.display().to_string(),
                    "name": i.name,
                    "size": i.size,
                    "system": i.system,
                    "group": i.group,
                    "faction": i.faction,
                    "subfaction": i.subfaction,
                    "class": i.klass,
                    "unit": i.unit,
                    "score": (i.match_score * 1000.0).round() / 1000.0,
                    "kind": i.match_kind,
                    "inner_count": i.inner_names.len(),
                    "target": i.target
                        .as_ref()
                        .map(|t| t.display().to_string())
                        .unwrap_or_else(|| "None".into()),
                })
            })
            .collect();
        println!("{}", serde_json::to_string_pretty(&rows)?);
    } else {
        println!("{}", sorter::format_plan(&items));
    }

    let n_catalog = items.iter().filter(|i| i.match_kind == "catalog").count();
    let n_implicit = items.iter().filter(|i| i.match_kind == "implicit").count();
    let n_none = items.iter().filter(|i| i.match_kind == "none").count();
    log::info!(
        "Plan: {} files -> {} catalog matches [+] / {} implicit [~] / {} unclassifiable [?]",
        items.len(),
        n_catalog,
        n_implicit,
        n_none
    );

    if args.apply {
        let (linked, copied, skipped) = sorter::apply_plan(&mut items, &audit)?;
        log::info!(
            "Applied: {} hardlinked, {} copied, {} skipped (audit: {})",
            linked,
            copied,
            skipped,
            audit.display()
        );
    }
    Ok(())
}

/// Recursively collect files under `root`, skipping any path whose parts
/// (relative to root) start with '.' (python rglob filter).
fn collect_visible(root: &Path, dir: &Path, out: &mut Vec<PathBuf>) -> std::io::Result<()> {
    for entry in std::fs::read_dir(dir)? {
        let entry = entry?;
        let path = entry.path();
        let name = entry.file_name().to_string_lossy().into_owned();
        if name.starts_with('.') {
            continue;
        }
        if path.is_dir() {
            collect_visible(root, &path, out)?;
        } else {
            out.push(path);
        }
    }
    Ok(())
}

fn cmd_extract(args: &ExtractArgs) -> anyhow::Result<()> {
    let ex = extract::Extractor {
        out_root: args.out.clone(),
    };
    ex.run(&args.sources)
}

fn cmd_gwshop(args: &GwshopArgs) -> anyhow::Result<()> {
    gwshop::run_cli(&args.out, &args.index, args.raw.as_deref())
}

fn main() {
    env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info"))
        .format(|buf, record| {
            use std::io::Write as _;
            writeln!(
                buf,
                "{} {:<7} {}",
                chrono::Local::now().format("%H:%M:%S"),
                record.level(),
                record.args()
            )
        })
        .init();

    let cli = Cli::parse();
    let res = match &cli.cmd {
        Cmd::Dlp(args) => cmd_dlp(args),
        Cmd::Catalogue(args) => cmd_catalogue(args),
        Cmd::Sorter(args) => cmd_sorter(args),
        Cmd::Extract(args) => cmd_extract(args),
        Cmd::Gwshop(args) => cmd_gwshop(args),
    };
    if let Err(err) = res {
        log::error!("{err:#}");
        exit(1);
    }
}