Initial version: dlp.py orchestrates Discord export, link extraction and host-aware downloads
- dlphelper/discord.py: DCE docker export (JSON) + base64/plain URL extraction (extraction regex kept compatible with parts/grab.sh) - dlphelper/gofile.py: GoFile client adapted from Lysagxra/GoFileDownloader's API scheme, but ONE guest account per run and per-file failure tracking instead of sys.exit - fixes the rate-limit death after ~6 downloads - dlphelper/zero807.py: curl downloader with the UA+Referer contract 0807.st requires; .part files for resumable downloads - dlphelper/state.py: state.json (resume/skip) + manifest.jsonl (torrent stage later) + failed_urls.txt - tools/new_repo.py: Gitea repo helper reading ~/.git-credentials itself - AGENTS.md with agent rules, README, offline extraction test
This commit is contained in:
@@ -0,0 +1,83 @@
|
||||
"""Downloader for 0807.st direct links via curl.
|
||||
|
||||
0807.st serves the files directly but rejects requests that don't look like a
|
||||
browser (requires a User-Agent and the Referer of the site page), and it
|
||||
rejects multi-connection/range abuse patterns differently than curl - which is
|
||||
why curl is used instead of aria2c here (see chat history: curl worked,
|
||||
aria2c's default request got 403).
|
||||
|
||||
Downloads go to a .part file first and are renamed on success, so an
|
||||
interrupted download is resumable (--continue-at -) and partial files are
|
||||
never mistaken for finished ones.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
USER_AGENT = "Mozilla/5.0"
|
||||
REFERER = "https://0807.st/"
|
||||
|
||||
|
||||
class Zero807Error(Exception):
|
||||
"""Base error for 0807 downloads."""
|
||||
|
||||
|
||||
def target_name(url: str) -> str:
|
||||
"""Derive the filename from the URL path (e.g. /6saCNie.zip -> 6saCNie.zip)."""
|
||||
name = Path(urlparse(url).path).name
|
||||
if not name:
|
||||
raise Zero807Error(f"cannot derive a filename from URL: {url}")
|
||||
return name
|
||||
|
||||
|
||||
def download(
|
||||
url: str,
|
||||
out_dir: Path,
|
||||
timeout_seconds: int = 7200,
|
||||
resume: bool = True,
|
||||
) -> tuple[Path, int]:
|
||||
"""Download one URL with curl; returns (final_path, size_bytes)."""
|
||||
name = target_name(url)
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
dest = out_dir / name
|
||||
tmp = dest.with_name(dest.name + ".part")
|
||||
|
||||
if dest.exists() and dest.stat().st_size > 0:
|
||||
log.info("Skip existing: %s", dest)
|
||||
return dest, dest.stat().st_size
|
||||
|
||||
cmd = [
|
||||
"curl", "-fSL",
|
||||
"--connect-timeout", "20",
|
||||
"--max-time", str(timeout_seconds),
|
||||
"--retry", "2", "--retry-delay", "5",
|
||||
"-A", USER_AGENT,
|
||||
"-e", REFERER,
|
||||
"-o", str(tmp),
|
||||
]
|
||||
if resume:
|
||||
cmd.append("--continue-at")
|
||||
cmd.append("-")
|
||||
cmd.append(url)
|
||||
|
||||
log.info("Downloading %s -> %s", url, dest.name)
|
||||
proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
|
||||
if proc.returncode != 0:
|
||||
tail = [line for line in proc.stderr.strip().splitlines() if line][-3:]
|
||||
raise Zero807Error(f"curl exit code {proc.returncode}: {' | '.join(tail)}")
|
||||
|
||||
size = tmp.stat().st_size if tmp.exists() else 0
|
||||
if size == 0:
|
||||
tmp.unlink(missing_ok=True)
|
||||
raise Zero807Error(f"empty file for {name}")
|
||||
if dest.suffix.lower() == ".zip" and not tmp.read_bytes()[:2] == b"PK":
|
||||
log.warning("%s does not look like a ZIP file (first bytes are not PK)", name)
|
||||
os.replace(tmp, dest)
|
||||
return dest, size
|
||||
Reference in New Issue
Block a user