Initial version: dlp.py orchestrates Discord export, link extraction and host-aware downloads
- dlphelper/discord.py: DCE docker export (JSON) + base64/plain URL extraction (extraction regex kept compatible with parts/grab.sh) - dlphelper/gofile.py: GoFile client adapted from Lysagxra/GoFileDownloader's API scheme, but ONE guest account per run and per-file failure tracking instead of sys.exit - fixes the rate-limit death after ~6 downloads - dlphelper/zero807.py: curl downloader with the UA+Referer contract 0807.st requires; .part files for resumable downloads - dlphelper/state.py: state.json (resume/skip) + manifest.jsonl (torrent stage later) + failed_urls.txt - tools/new_repo.py: Gitea repo helper reading ~/.git-credentials itself - AGENTS.md with agent rules, README, offline extraction test
This commit is contained in:
Executable
+38
@@ -0,0 +1,38 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
THREAD_ID="${THREAD_ID:-1455993943471820842}"
|
||||
OUT_DIR="${OUT_DIR:-$PWD/thread}"
|
||||
HTML="$OUT_DIR/export.html"
|
||||
URLS="$OUT_DIR/urls.txt" # aria2c input file: one URL per line
|
||||
|
||||
# Token from env, otherwise prompt (input hidden)
|
||||
if [[ -z "${DISCORD_TOKEN:-}" ]]; then
|
||||
read -rsp "Discord token: " DISCORD_TOKEN
|
||||
echo
|
||||
fi
|
||||
|
||||
mkdir -p "$OUT_DIR"
|
||||
|
||||
# 1. Export the thread to HTML
|
||||
docker run --rm --user "$(id -u):$(id -g)" \
|
||||
-v "$OUT_DIR:/out" \
|
||||
tyrrrz/discordchatexporter:stable export \
|
||||
-t "$DISCORD_TOKEN" -c "$THREAD_ID" -f HtmlDark -o /out/export.html
|
||||
|
||||
unset DISCORD_TOKEN
|
||||
|
||||
# 2. Find base64 strings starting with "https://", then decode them
|
||||
grep -oP 'aHR0cHM6Ly[89][A-Za-z0-9+/_-]*' "$HTML" \
|
||||
| sort -u \
|
||||
| while read -r s; do
|
||||
s=$(printf '%s' "$s" | tr '_-' '/+' | tr -d '=')
|
||||
while (( ${#s} % 4 )); do s="$s="; done
|
||||
printf '%s' "$s" | base64 -d 2>/dev/null | tr -d '\r\n' || true
|
||||
echo
|
||||
done \
|
||||
| grep -E '^https://[[:graph:]]+$' \
|
||||
| awk '!seen[$0]++' > "$URLS"
|
||||
|
||||
echo "Found $(wc -l < "$URLS") URLs -> $URLS"
|
||||
echo "Download with: aria2c -i \"$URLS\" -d \"$OUT_DIR/downloads\""
|
||||
Reference in New Issue
Block a user