Add --limit (per host class, on pending URLs) and --shuffle for test runs

dlp.py

@@ -27,9 +27,11 @@ import hashlib
 import json
 import logging
 import os
+import random
 import sys
 import time
 import zipfile
+from itertools import chain
 from pathlib import Path
 
 from dlphelper import discord as dlp_discord
@@ -355,7 +357,18 @@ def process_channel(
         for f in found
     }
 
+    # --limit applies per host class to the *pending* URLs, so a test run
+    # downloads exactly N new things per host (done ones don't use up the quota)
+    if args.limit is not None:
+        for key, items in buckets.items():
+            pending = [f for f in items if not is_done(f.url)]
+            if args.shuffle:
+                random.shuffle(pending)
+            buckets[key] = pending[: args.limit]
+
     pending = [f for f in found if not is_done(f.url)]
+    if args.limit is not None:
+        pending = [f for f in chain.from_iterable(buckets.values()) if not is_done(f.url)]
     log.info(
         "%s: %d URLs (%d gofile, %d 0807); %d done, %d to download",
         channel_id, len(found),
@@ -364,8 +377,14 @@ def process_channel(
     )
 
     if args.dry_run:
+        selected = {f.url for f in chain.from_iterable(buckets.values())}
         for f in found:
-            flag = "done" if is_done(f.url) else "download"
+            if f.url in selected:
+                flag = "download"
+            elif is_done(f.url):
+                flag = "done"
+            else:
+                flag = "skip"  # cut by --limit / --only
             print(f"[{f.host:6}] [{flag:8}] {f.url}")
         return 0
 
@@ -432,6 +451,11 @@ def main() -> None:
                         help="length of the cooldown pause (default 60)")
     parser.add_argument("--only", choices=("gofile", "0807"),
                         help="process only one host class")
+    parser.add_argument("--limit", type=int, metavar="N",
+                        help="download at most N pending URLs per host class "
+                             "(test runs)")
+    parser.add_argument("--shuffle", action="store_true",
+                        help="randomize which URLs a --limit picks")
     parser.add_argument("--hash", action="store_true",
                         help="record sha256 of each downloaded file")
     parser.add_argument("--extract", action="store_true",