Dead links (gofile notFound/404, 0807 404/410) fail immediately without retry

This commit is contained in:
2026-10-04 18:54:08 +02:00
parent 9d47c29495
commit 413672f114
3 changed files with 35 additions and 3 deletions
+13 -2
View File
@@ -177,6 +177,11 @@ def run_gofile(
log.error("PASSWORD REQUIRED %s: %s", f.url, exc) log.error("PASSWORD REQUIRED %s: %s", f.url, exc)
state.mark(f.url, status="failed", error=str(exc), files=[]) state.mark(f.url, status="failed", error=str(exc), files=[])
break break
except dlph_gofile.GoFileNotFound as exc:
# Dead links never come back - no retry
log.error("DEAD LINK %s: %s", f.url, exc)
state.mark(f.url, status="failed", error=str(exc), files=[])
break
except dlph_gofile.GoFileAuthError as exc: except dlph_gofile.GoFileAuthError as exc:
# Token may have died mid-run (rate limit); re-auth once per attempt # Token may have died mid-run (rate limit); re-auth once per attempt
log.warning("Auth error for %s: %s - re-authenticating", f.url, exc) log.warning("Auth error for %s: %s - re-authenticating", f.url, exc)
@@ -246,6 +251,11 @@ def run_0807(
extract_zips([entry]) extract_zips([entry])
consecutive_failures = 0 consecutive_failures = 0
break break
except dlph_zero807.Zero807NotFound as exc:
# Dead links never come back - no retry
log.error("DEAD LINK %s: %s", f.url, exc)
state.mark(f.url, status="failed", error=str(exc), files=[])
break
except dlph_zero807.Zero807Error as exc: except dlph_zero807.Zero807Error as exc:
log.warning("Attempt %d/%d failed for %s: %s", log.warning("Attempt %d/%d failed for %s: %s",
attempt, args.retries, f.url, exc) attempt, args.retries, f.url, exc)
@@ -397,13 +407,14 @@ def process_channel(
ok = [u for u, r in state.records.items() if r.get("status") == "ok"] ok = [u for u, r in state.records.items() if r.get("status") == "ok"]
bad = state.failures() bad = state.failures()
dead = sum(1 for r in state.records.values() if "dead link" in (r.get("error") or ""))
total_files = sum(len(r.get("files") or []) for r in state.records.values()) total_files = sum(len(r.get("files") or []) for r in state.records.values())
total_size = sum( total_size = sum(
(f_.get("size") or 0) for r in state.records.values() for f_ in (r.get("files") or []) (f_.get("size") or 0) for r in state.records.values() for f_ in (r.get("files") or [])
) )
log.info("%s: %d ok (%d files, %s) / %d failed / %d seen", log.info("%s: %d ok (%d files, %s) / %d failed (%d dead links) / %d seen",
out_dir, len(ok), total_files, format_size(total_size), out_dir, len(ok), total_files, format_size(total_size),
len(bad), len(state.records)) len(bad), dead, len(state.records))
if bad: if bad:
log.info("Failed URLs written to %s - re-run this tool or use " log.info("Failed URLs written to %s - re-run this tool or use "
"python3 dlp.py <id> <out> --urls-file %s to retry them.", "python3 dlp.py <id> <out> --urls-file %s to retry them.",
+13
View File
@@ -53,6 +53,13 @@ class GoFilePasswordRequired(GoFileError):
"""The content is password protected and none (or a wrong one) was given.""" """The content is password protected and none (or a wrong one) was given."""
class GoFileNotFound(GoFileError):
"""The content/file no longer exists (404 / status notFound).
Dead links never come back, so the orchestrator must not retry these.
"""
class GoFileAuthError(GoFileError): class GoFileAuthError(GoFileError):
"""The account token was rejected; the client can re-auth once.""" """The account token was rejected; the client can re-auth once."""
@@ -153,6 +160,8 @@ class GoFileClient:
except requests.RequestException as exc: except requests.RequestException as exc:
raise GoFileError(f"content request for {cid} failed: {exc}") from exc raise GoFileError(f"content request for {cid} failed: {exc}") from exc
if resp.status_code == 404:
raise GoFileNotFound(f"content {cid} not found (dead link)")
if resp.status_code in (401, 403): if resp.status_code in (401, 403):
raise GoFileAuthError(f"content request denied ({resp.status_code}) for {cid}") raise GoFileAuthError(f"content request denied ({resp.status_code}) for {cid}")
try: try:
@@ -160,6 +169,8 @@ class GoFileClient:
except ValueError as exc: except ValueError as exc:
raise GoFileError(f"non-JSON response for {cid}: {resp.text[:200]}") from exc raise GoFileError(f"non-JSON response for {cid}: {resp.text[:200]}") from exc
if data.get("status") == "notFound":
raise GoFileNotFound(f"content {cid} does not exist anymore (dead link)")
if data.get("status") != "ok": if data.get("status") != "ok":
raise GoFileError(f"content request failed for {cid}: {data.get('status')}") raise GoFileError(f"content request failed for {cid}: {data.get('status')}")
@@ -199,6 +210,8 @@ class GoFileClient:
with self.session.get( with self.session.get(
link, headers=headers, stream=True, timeout=(10, 60), allow_redirects=True link, headers=headers, stream=True, timeout=(10, 60), allow_redirects=True
) as resp: ) as resp:
if resp.status_code == 404:
raise GoFileNotFound(f"file {dest.name} not found (dead link)")
if resp.status_code != 200: if resp.status_code != 200:
raise GoFileError(f"HTTP {resp.status_code} while downloading {dest.name}") raise GoFileError(f"HTTP {resp.status_code} while downloading {dest.name}")
total = int(resp.headers.get("Content-Length") or 0) total = int(resp.headers.get("Content-Length") or 0)
+9 -1
View File
@@ -29,6 +29,10 @@ class Zero807Error(Exception):
"""Base error for 0807 downloads.""" """Base error for 0807 downloads."""
class Zero807NotFound(Zero807Error):
"""The file no longer exists (HTTP 404/410). Not retryable."""
def target_name(url: str) -> str: def target_name(url: str) -> str:
"""Derive the filename from the URL path (e.g. /6saCNie.zip -> 6saCNie.zip).""" """Derive the filename from the URL path (e.g. /6saCNie.zip -> 6saCNie.zip)."""
name = Path(urlparse(url).path).name name = Path(urlparse(url).path).name
@@ -71,7 +75,11 @@ def download(
proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
if proc.returncode != 0: if proc.returncode != 0:
tail = [line for line in proc.stderr.strip().splitlines() if line][-3:] tail = [line for line in proc.stderr.strip().splitlines() if line][-3:]
raise Zero807Error(f"curl exit code {proc.returncode}: {' | '.join(tail)}") stderr = " | ".join(tail)
# -f makes curl exit 22 on any HTTP error; dead links fail for good
if "404" in stderr or "410" in stderr:
raise Zero807NotFound(f"dead link (HTTP 404/410): {stderr}")
raise Zero807Error(f"curl exit code {proc.returncode}: {stderr}")
size = tmp.stat().st_size if tmp.exists() else 0 size = tmp.stat().st_size if tmp.exists() else 0
if size == 0: if size == 0: