Dead links (gofile notFound/404, 0807 404/410) fail immediately without retry
This commit is contained in:
@@ -177,6 +177,11 @@ def run_gofile(
|
|||||||
log.error("PASSWORD REQUIRED %s: %s", f.url, exc)
|
log.error("PASSWORD REQUIRED %s: %s", f.url, exc)
|
||||||
state.mark(f.url, status="failed", error=str(exc), files=[])
|
state.mark(f.url, status="failed", error=str(exc), files=[])
|
||||||
break
|
break
|
||||||
|
except dlph_gofile.GoFileNotFound as exc:
|
||||||
|
# Dead links never come back - no retry
|
||||||
|
log.error("DEAD LINK %s: %s", f.url, exc)
|
||||||
|
state.mark(f.url, status="failed", error=str(exc), files=[])
|
||||||
|
break
|
||||||
except dlph_gofile.GoFileAuthError as exc:
|
except dlph_gofile.GoFileAuthError as exc:
|
||||||
# Token may have died mid-run (rate limit); re-auth once per attempt
|
# Token may have died mid-run (rate limit); re-auth once per attempt
|
||||||
log.warning("Auth error for %s: %s - re-authenticating", f.url, exc)
|
log.warning("Auth error for %s: %s - re-authenticating", f.url, exc)
|
||||||
@@ -246,6 +251,11 @@ def run_0807(
|
|||||||
extract_zips([entry])
|
extract_zips([entry])
|
||||||
consecutive_failures = 0
|
consecutive_failures = 0
|
||||||
break
|
break
|
||||||
|
except dlph_zero807.Zero807NotFound as exc:
|
||||||
|
# Dead links never come back - no retry
|
||||||
|
log.error("DEAD LINK %s: %s", f.url, exc)
|
||||||
|
state.mark(f.url, status="failed", error=str(exc), files=[])
|
||||||
|
break
|
||||||
except dlph_zero807.Zero807Error as exc:
|
except dlph_zero807.Zero807Error as exc:
|
||||||
log.warning("Attempt %d/%d failed for %s: %s",
|
log.warning("Attempt %d/%d failed for %s: %s",
|
||||||
attempt, args.retries, f.url, exc)
|
attempt, args.retries, f.url, exc)
|
||||||
@@ -397,13 +407,14 @@ def process_channel(
|
|||||||
|
|
||||||
ok = [u for u, r in state.records.items() if r.get("status") == "ok"]
|
ok = [u for u, r in state.records.items() if r.get("status") == "ok"]
|
||||||
bad = state.failures()
|
bad = state.failures()
|
||||||
|
dead = sum(1 for r in state.records.values() if "dead link" in (r.get("error") or ""))
|
||||||
total_files = sum(len(r.get("files") or []) for r in state.records.values())
|
total_files = sum(len(r.get("files") or []) for r in state.records.values())
|
||||||
total_size = sum(
|
total_size = sum(
|
||||||
(f_.get("size") or 0) for r in state.records.values() for f_ in (r.get("files") or [])
|
(f_.get("size") or 0) for r in state.records.values() for f_ in (r.get("files") or [])
|
||||||
)
|
)
|
||||||
log.info("%s: %d ok (%d files, %s) / %d failed / %d seen",
|
log.info("%s: %d ok (%d files, %s) / %d failed (%d dead links) / %d seen",
|
||||||
out_dir, len(ok), total_files, format_size(total_size),
|
out_dir, len(ok), total_files, format_size(total_size),
|
||||||
len(bad), len(state.records))
|
len(bad), dead, len(state.records))
|
||||||
if bad:
|
if bad:
|
||||||
log.info("Failed URLs written to %s - re-run this tool or use "
|
log.info("Failed URLs written to %s - re-run this tool or use "
|
||||||
"python3 dlp.py <id> <out> --urls-file %s to retry them.",
|
"python3 dlp.py <id> <out> --urls-file %s to retry them.",
|
||||||
|
|||||||
@@ -53,6 +53,13 @@ class GoFilePasswordRequired(GoFileError):
|
|||||||
"""The content is password protected and none (or a wrong one) was given."""
|
"""The content is password protected and none (or a wrong one) was given."""
|
||||||
|
|
||||||
|
|
||||||
|
class GoFileNotFound(GoFileError):
|
||||||
|
"""The content/file no longer exists (404 / status notFound).
|
||||||
|
|
||||||
|
Dead links never come back, so the orchestrator must not retry these.
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
class GoFileAuthError(GoFileError):
|
class GoFileAuthError(GoFileError):
|
||||||
"""The account token was rejected; the client can re-auth once."""
|
"""The account token was rejected; the client can re-auth once."""
|
||||||
|
|
||||||
@@ -153,6 +160,8 @@ class GoFileClient:
|
|||||||
except requests.RequestException as exc:
|
except requests.RequestException as exc:
|
||||||
raise GoFileError(f"content request for {cid} failed: {exc}") from exc
|
raise GoFileError(f"content request for {cid} failed: {exc}") from exc
|
||||||
|
|
||||||
|
if resp.status_code == 404:
|
||||||
|
raise GoFileNotFound(f"content {cid} not found (dead link)")
|
||||||
if resp.status_code in (401, 403):
|
if resp.status_code in (401, 403):
|
||||||
raise GoFileAuthError(f"content request denied ({resp.status_code}) for {cid}")
|
raise GoFileAuthError(f"content request denied ({resp.status_code}) for {cid}")
|
||||||
try:
|
try:
|
||||||
@@ -160,6 +169,8 @@ class GoFileClient:
|
|||||||
except ValueError as exc:
|
except ValueError as exc:
|
||||||
raise GoFileError(f"non-JSON response for {cid}: {resp.text[:200]}") from exc
|
raise GoFileError(f"non-JSON response for {cid}: {resp.text[:200]}") from exc
|
||||||
|
|
||||||
|
if data.get("status") == "notFound":
|
||||||
|
raise GoFileNotFound(f"content {cid} does not exist anymore (dead link)")
|
||||||
if data.get("status") != "ok":
|
if data.get("status") != "ok":
|
||||||
raise GoFileError(f"content request failed for {cid}: {data.get('status')}")
|
raise GoFileError(f"content request failed for {cid}: {data.get('status')}")
|
||||||
|
|
||||||
@@ -199,6 +210,8 @@ class GoFileClient:
|
|||||||
with self.session.get(
|
with self.session.get(
|
||||||
link, headers=headers, stream=True, timeout=(10, 60), allow_redirects=True
|
link, headers=headers, stream=True, timeout=(10, 60), allow_redirects=True
|
||||||
) as resp:
|
) as resp:
|
||||||
|
if resp.status_code == 404:
|
||||||
|
raise GoFileNotFound(f"file {dest.name} not found (dead link)")
|
||||||
if resp.status_code != 200:
|
if resp.status_code != 200:
|
||||||
raise GoFileError(f"HTTP {resp.status_code} while downloading {dest.name}")
|
raise GoFileError(f"HTTP {resp.status_code} while downloading {dest.name}")
|
||||||
total = int(resp.headers.get("Content-Length") or 0)
|
total = int(resp.headers.get("Content-Length") or 0)
|
||||||
|
|||||||
@@ -29,6 +29,10 @@ class Zero807Error(Exception):
|
|||||||
"""Base error for 0807 downloads."""
|
"""Base error for 0807 downloads."""
|
||||||
|
|
||||||
|
|
||||||
|
class Zero807NotFound(Zero807Error):
|
||||||
|
"""The file no longer exists (HTTP 404/410). Not retryable."""
|
||||||
|
|
||||||
|
|
||||||
def target_name(url: str) -> str:
|
def target_name(url: str) -> str:
|
||||||
"""Derive the filename from the URL path (e.g. /6saCNie.zip -> 6saCNie.zip)."""
|
"""Derive the filename from the URL path (e.g. /6saCNie.zip -> 6saCNie.zip)."""
|
||||||
name = Path(urlparse(url).path).name
|
name = Path(urlparse(url).path).name
|
||||||
@@ -71,7 +75,11 @@ def download(
|
|||||||
proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
|
proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
|
||||||
if proc.returncode != 0:
|
if proc.returncode != 0:
|
||||||
tail = [line for line in proc.stderr.strip().splitlines() if line][-3:]
|
tail = [line for line in proc.stderr.strip().splitlines() if line][-3:]
|
||||||
raise Zero807Error(f"curl exit code {proc.returncode}: {' | '.join(tail)}")
|
stderr = " | ".join(tail)
|
||||||
|
# -f makes curl exit 22 on any HTTP error; dead links fail for good
|
||||||
|
if "404" in stderr or "410" in stderr:
|
||||||
|
raise Zero807NotFound(f"dead link (HTTP 404/410): {stderr}")
|
||||||
|
raise Zero807Error(f"curl exit code {proc.returncode}: {stderr}")
|
||||||
|
|
||||||
size = tmp.stat().st_size if tmp.exists() else 0
|
size = tmp.stat().st_size if tmp.exists() else 0
|
||||||
if size == 0:
|
if size == 0:
|
||||||
|
|||||||
Reference in New Issue
Block a user