diff --git a/dlp.py b/dlp.py index c5ad859..0776c0d 100644 --- a/dlp.py +++ b/dlp.py @@ -177,6 +177,11 @@ def run_gofile( log.error("PASSWORD REQUIRED %s: %s", f.url, exc) state.mark(f.url, status="failed", error=str(exc), files=[]) break + except dlph_gofile.GoFileNotFound as exc: + # Dead links never come back - no retry + log.error("DEAD LINK %s: %s", f.url, exc) + state.mark(f.url, status="failed", error=str(exc), files=[]) + break except dlph_gofile.GoFileAuthError as exc: # Token may have died mid-run (rate limit); re-auth once per attempt log.warning("Auth error for %s: %s - re-authenticating", f.url, exc) @@ -246,6 +251,11 @@ def run_0807( extract_zips([entry]) consecutive_failures = 0 break + except dlph_zero807.Zero807NotFound as exc: + # Dead links never come back - no retry + log.error("DEAD LINK %s: %s", f.url, exc) + state.mark(f.url, status="failed", error=str(exc), files=[]) + break except dlph_zero807.Zero807Error as exc: log.warning("Attempt %d/%d failed for %s: %s", attempt, args.retries, f.url, exc) @@ -397,13 +407,14 @@ def process_channel( ok = [u for u, r in state.records.items() if r.get("status") == "ok"] bad = state.failures() + dead = sum(1 for r in state.records.values() if "dead link" in (r.get("error") or "")) total_files = sum(len(r.get("files") or []) for r in state.records.values()) total_size = sum( (f_.get("size") or 0) for r in state.records.values() for f_ in (r.get("files") or []) ) - log.info("%s: %d ok (%d files, %s) / %d failed / %d seen", + log.info("%s: %d ok (%d files, %s) / %d failed (%d dead links) / %d seen", out_dir, len(ok), total_files, format_size(total_size), - len(bad), len(state.records)) + len(bad), dead, len(state.records)) if bad: log.info("Failed URLs written to %s - re-run this tool or use " "python3 dlp.py --urls-file %s to retry them.", diff --git a/dlphelper/gofile.py b/dlphelper/gofile.py index aeb3d7e..f9cba05 100644 --- a/dlphelper/gofile.py +++ b/dlphelper/gofile.py @@ -53,6 +53,13 @@ class GoFilePasswordRequired(GoFileError): """The content is password protected and none (or a wrong one) was given.""" +class GoFileNotFound(GoFileError): + """The content/file no longer exists (404 / status notFound). + + Dead links never come back, so the orchestrator must not retry these. + """ + + class GoFileAuthError(GoFileError): """The account token was rejected; the client can re-auth once.""" @@ -153,6 +160,8 @@ class GoFileClient: except requests.RequestException as exc: raise GoFileError(f"content request for {cid} failed: {exc}") from exc + if resp.status_code == 404: + raise GoFileNotFound(f"content {cid} not found (dead link)") if resp.status_code in (401, 403): raise GoFileAuthError(f"content request denied ({resp.status_code}) for {cid}") try: @@ -160,6 +169,8 @@ class GoFileClient: except ValueError as exc: raise GoFileError(f"non-JSON response for {cid}: {resp.text[:200]}") from exc + if data.get("status") == "notFound": + raise GoFileNotFound(f"content {cid} does not exist anymore (dead link)") if data.get("status") != "ok": raise GoFileError(f"content request failed for {cid}: {data.get('status')}") @@ -199,6 +210,8 @@ class GoFileClient: with self.session.get( link, headers=headers, stream=True, timeout=(10, 60), allow_redirects=True ) as resp: + if resp.status_code == 404: + raise GoFileNotFound(f"file {dest.name} not found (dead link)") if resp.status_code != 200: raise GoFileError(f"HTTP {resp.status_code} while downloading {dest.name}") total = int(resp.headers.get("Content-Length") or 0) diff --git a/dlphelper/zero807.py b/dlphelper/zero807.py index bf4d447..de7354d 100644 --- a/dlphelper/zero807.py +++ b/dlphelper/zero807.py @@ -29,6 +29,10 @@ class Zero807Error(Exception): """Base error for 0807 downloads.""" +class Zero807NotFound(Zero807Error): + """The file no longer exists (HTTP 404/410). Not retryable.""" + + def target_name(url: str) -> str: """Derive the filename from the URL path (e.g. /6saCNie.zip -> 6saCNie.zip).""" name = Path(urlparse(url).path).name @@ -71,7 +75,11 @@ def download( proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) if proc.returncode != 0: tail = [line for line in proc.stderr.strip().splitlines() if line][-3:] - raise Zero807Error(f"curl exit code {proc.returncode}: {' | '.join(tail)}") + stderr = " | ".join(tail) + # -f makes curl exit 22 on any HTTP error; dead links fail for good + if "404" in stderr or "410" in stderr: + raise Zero807NotFound(f"dead link (HTTP 404/410): {stderr}") + raise Zero807Error(f"curl exit code {proc.returncode}: {stderr}") size = tmp.stat().st_size if tmp.exists() else 0 if size == 0: