Ben_Kosytorz
db8a1c1b52
- dlphelper/discord.py: DCE docker export (JSON) + base64/plain URL extraction (extraction regex kept compatible with parts/grab.sh) - dlphelper/gofile.py: GoFile client adapted from Lysagxra/GoFileDownloader's API scheme, but ONE guest account per run and per-file failure tracking instead of sys.exit - fixes the rate-limit death after ~6 downloads - dlphelper/zero807.py: curl downloader with the UA+Referer contract 0807.st requires; .part files for resumable downloads - dlphelper/state.py: state.json (resume/skip) + manifest.jsonl (torrent stage later) + failed_urls.txt - tools/new_repo.py: Gitea repo helper reading ~/.git-credentials itself - AGENTS.md with agent rules, README, offline extraction test
84 lines
2.6 KiB
Python
84 lines
2.6 KiB
Python
"""Downloader for 0807.st direct links via curl.
|
|
|
|
0807.st serves the files directly but rejects requests that don't look like a
|
|
browser (requires a User-Agent and the Referer of the site page), and it
|
|
rejects multi-connection/range abuse patterns differently than curl - which is
|
|
why curl is used instead of aria2c here (see chat history: curl worked,
|
|
aria2c's default request got 403).
|
|
|
|
Downloads go to a .part file first and are renamed on success, so an
|
|
interrupted download is resumable (--continue-at -) and partial files are
|
|
never mistaken for finished ones.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import os
|
|
import subprocess
|
|
from pathlib import Path
|
|
from urllib.parse import urlparse
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
USER_AGENT = "Mozilla/5.0"
|
|
REFERER = "https://0807.st/"
|
|
|
|
|
|
class Zero807Error(Exception):
|
|
"""Base error for 0807 downloads."""
|
|
|
|
|
|
def target_name(url: str) -> str:
|
|
"""Derive the filename from the URL path (e.g. /6saCNie.zip -> 6saCNie.zip)."""
|
|
name = Path(urlparse(url).path).name
|
|
if not name:
|
|
raise Zero807Error(f"cannot derive a filename from URL: {url}")
|
|
return name
|
|
|
|
|
|
def download(
|
|
url: str,
|
|
out_dir: Path,
|
|
timeout_seconds: int = 7200,
|
|
resume: bool = True,
|
|
) -> tuple[Path, int]:
|
|
"""Download one URL with curl; returns (final_path, size_bytes)."""
|
|
name = target_name(url)
|
|
out_dir.mkdir(parents=True, exist_ok=True)
|
|
dest = out_dir / name
|
|
tmp = dest.with_name(dest.name + ".part")
|
|
|
|
if dest.exists() and dest.stat().st_size > 0:
|
|
log.info("Skip existing: %s", dest)
|
|
return dest, dest.stat().st_size
|
|
|
|
cmd = [
|
|
"curl", "-fSL",
|
|
"--connect-timeout", "20",
|
|
"--max-time", str(timeout_seconds),
|
|
"--retry", "2", "--retry-delay", "5",
|
|
"-A", USER_AGENT,
|
|
"-e", REFERER,
|
|
"-o", str(tmp),
|
|
]
|
|
if resume:
|
|
cmd.append("--continue-at")
|
|
cmd.append("-")
|
|
cmd.append(url)
|
|
|
|
log.info("Downloading %s -> %s", url, dest.name)
|
|
proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
|
|
if proc.returncode != 0:
|
|
tail = [line for line in proc.stderr.strip().splitlines() if line][-3:]
|
|
raise Zero807Error(f"curl exit code {proc.returncode}: {' | '.join(tail)}")
|
|
|
|
size = tmp.stat().st_size if tmp.exists() else 0
|
|
if size == 0:
|
|
tmp.unlink(missing_ok=True)
|
|
raise Zero807Error(f"empty file for {name}")
|
|
if dest.suffix.lower() == ".zip" and not tmp.read_bytes()[:2] == b"PK":
|
|
log.warning("%s does not look like a ZIP file (first bytes are not PK)", name)
|
|
os.replace(tmp, dest)
|
|
return dest, size
|