fix(thumbs): xhamster miniatury z JSON strony wynikow, mypornerleak z src i og:image posta, backfill brakujacych

- _search_base: hak _thumbnail_map (mapa url -> miniatura z calej strony) przed
  szukaniem <img> w oknie tekstu; xhamster mial 54% zrodel bez miniatury
- mypornerleak search: karty maja teraz src zamiast data-src; browse (REST) bez
  featured media -> og:image ze strony posta
- scripts/backfill_thumbs.py: uzupelnia miniatury ze strony sceny, od najnowszych

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

app/connectors/direct_scrapers/_search_base.py

@@ -107,6 +107,15 @@ class BaseSearchScraper(BaseDirectTubeScraper):
         """
         return urllib.parse.quote_plus(query.strip())
 
+    def _thumbnail_map(self, listing_html: str) -> dict[str, str]:
+        """Opcjonalny hak: mapa scene_url -> thumbnail_url z calej strony wynikow.
+
+        Domyslnie pusta i wtedy dziala szukanie `<img>` w oknie wokol linku. Tuby,
+        ktore renderuja karty z JSON-a (xhamster: `window.initials` z `pageURL` i
+        `thumbURL`), maja miniature dla KAZDEJ karty w jednym miejscu, a okno tekstu
+        gubilo polowe (54% zrodel xhamster bez miniatury, zgloszenie 8ec9dc96)."""
+        return {}
+
     def _fetch_scene_metadata(
         self, scene_url: str
     ) -> tuple[RawStudio | None, list[RawPerformer], list[RawTag]] | None:
@@ -169,6 +178,11 @@ class BaseSearchScraper(BaseDirectTubeScraper):
         }
 
         seen: set[str] = set()
+        try:
+            thumb_map = self._thumbnail_map(r.text)
+        except Exception as e:  # noqa: BLE001
+            log.debug("%s thumbnail map failed: %s", self.sitetag, e)
+            thumb_map = {}
         yielded = 0
         for m in self._scene_url_re.finditer(r.text):
             scene_url = self._scene_url_from_match(m).strip()
@@ -213,8 +227,8 @@ class BaseSearchScraper(BaseDirectTubeScraper):
             window_start = max(0, m.start() - self._thumbnail_window)
             window_end = min(len(r.text), m.end() + self._thumbnail_window)
             window_html = r.text[window_start:window_end]
-            thumb_url: str | None = None
+            thumb_url: str | None = thumb_map.get(scene_url)
-            img_m = _IMG_SRC_RE.search(window_html)
+            img_m = None if thumb_url else _IMG_SRC_RE.search(window_html)
             if img_m:
                 thumb_url = img_m.group(1).strip()
                 if thumb_url.startswith("//"):

app/connectors/direct_scrapers/mypornerleak.py

@@ -33,7 +33,9 @@ _BASE = "https://mypornerleak.com"
 _ARTICLE_RE = re.compile(r'<article[^>]*\bclass="([^"]+)"', re.IGNORECASE)
 _LINK_RE = re.compile(r'<a\s+href="([^"]+)"\s+title="([^"]+)"', re.IGNORECASE)
 _DURATION_RE = re.compile(r'class="duration">(?:<[^>]+>)*\s*(\d{1,2}:\d{2}(?::\d{2})?)', re.IGNORECASE)
-_THUMB_RE = re.compile(r'data-src="([^"]+)"', re.IGNORECASE)
+# Do 2026-09 karty mialy lazy `data-src`, teraz zwykle `src` (class video-main-thumb);
+# stary wzorzec zostawial 59% zrodel bez miniatury (zgloszenie 94e1da85).
+_THUMB_RE = re.compile(r'<img[^>]+(?:data-src|src)="(https?://[^"]+\.(?:jpg|jpeg|png|webp)[^"]*)"', re.IGNORECASE)
 _WN_HOST_RE = re.compile(r"https?://w\d+\.mypornerleak\.com", re.IGNORECASE)
 _CLASS_ACTOR_RE = re.compile(r"\bactors-([a-z0-9-]+)")
 _CLASS_CAT_RE = re.compile(r"\bcategory-([a-z0-9-]+)")

app/connectors/direct_scrapers/mypornerleak_browse.py

@@ -28,6 +28,7 @@ from app.connectors.direct_scrapers._browse_base import (
     compute_thumbnail_phash,
 )
 from app.extractors import browser_get
+from app.extractors.thumb_extract import extract_thumbnail_url
 from app.normalize.text import slugify
 
 log = logging.getLogger(__name__)
@@ -85,6 +86,15 @@ class MyPornerLeakBrowseScraper(BaseBrowseScraper):
             emb = p.get("_embedded") or {}
             fm = emb.get("wp:featuredmedia") or []
             thumb = (fm[0].get("source_url") if fm and isinstance(fm[0], dict) else None) or None
+            if not thumb:
+                # Od 2026-09 REST nie ma featured media (featured_media=0, brak img w
+                # tresci); strona posta ma og:image (zgloszenie 94e1da85).
+                try:
+                    pr = browser_get(link, timeout=15.0)
+                    if pr.status_code == 200:
+                        thumb = extract_thumbnail_url(pr.text)
+                except Exception as e:  # noqa: BLE001
+                    log.debug("mypornerleak og:image fetch failed for %s: %s", link, e)
 
             studio: RawStudio | None = None
             tags: list[RawTag] = []

app/connectors/direct_scrapers/xhamster.py

@@ -7,6 +7,13 @@ from __future__ import annotations
 
 import re
 
+# Karty sa renderowane z JSON-a w HTML (`window.initials`): kazda ma `pageURL` i
+# `thumbURL` (526x298 webp). Szukanie `<img>` w oknie tekstu gubilo polowe scen, bo
+# ten sam link wystepuje dwa razy (miniatura i tytul), a img jest tylko przy pierwszym.
+_JSON_CARD_RE = re.compile(
+    r'"pageURL":"(https:\\/\\/xhamster\.com\\/videos\\/[^"]+)","thumbURL":"([^"]+)"'
+)
+
 from app.connectors.direct_scrapers._search_base import BaseSearchScraper
 
 
@@ -17,3 +24,9 @@ class XHamsterScraper(BaseSearchScraper):
         r'href="(?P<url>https://xhamster\.com/videos/(?P<slug>[a-z0-9_\-]+))"',
         re.IGNORECASE,
     )
+
+    def _thumbnail_map(self, listing_html: str) -> dict[str, str]:
+        out: dict[str, str] = {}
+        for page_url, thumb in _JSON_CARD_RE.findall(listing_html):
+            out[page_url.replace("\\/", "/")] = thumb.replace("\\/", "/")
+        return out

scripts/backfill_thumbs.py

@@ -0,0 +1,102 @@
+"""Uzupelnia brakujace miniatury scen z jednego tube'a ze strony sceny (og:image itd.).
+
+Po co: scraper xhamster gubil miniatury polowy scen (szukal <img> w oknie tekstu, a
+karty sa w JSON-ie), a mypornerleak przestal wystawiac obrazek w REST. Ingest jest
+juz naprawiony (2026-09-14), ten skrypt sprzata to, co weszlo bez miniatury: wchodzi
+na strone sceny jak przycisk „Refresh thumbnail" w apce i zapisuje miniature na
+wszystkich zrodlach sceny bez miniatury.
+
+Kolejnosc: od najnowszych scen, bo te sa na gorze list. Limit czasu przez --days.
+
+  docker compose run -d --rm --name goon-thumbs-xh worker python3 -u /srv/scripts/backfill_thumbs.py --origin tube:xhamstercom --days 30 --rps 3
+  docker compose run -d --rm --name goon-thumbs-mpl worker python3 -u /srv/scripts/backfill_thumbs.py --origin tube:mypornerleakcom --days 30 --rps 2
+"""
+from __future__ import annotations
+
+import argparse
+import logging
+import sys
+import time
+
+from sqlalchemy import text
+
+from app.db import session_scope
+from app.extractors._fetch import browser_get
+from app.extractors.thumb_extract import extract_thumbnail_url
+
+log = logging.getLogger("backfill_thumbs")
+
+_SQL_KANDYDACI = text(
+    """
+    SELECT DISTINCT ON (s.id) s.id AS scene_id, ps.page_url, s.created_at
+    FROM scenes s
+    JOIN playback_sources ps ON ps.scene_id = s.id AND ps.dead_at IS NULL AND ps.origin = :origin
+    WHERE ps.thumbnail_url IS NULL
+      AND s.created_at > now() - make_interval(days => :days)
+      AND NOT EXISTS (
+        SELECT 1 FROM playback_sources q
+        WHERE q.scene_id = s.id AND q.dead_at IS NULL AND q.thumbnail_url IS NOT NULL
+      )
+    ORDER BY s.id, ps.created_at
+    """
+)
+
+
+def main() -> int:
+    logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--origin", required=True, help="np. tube:xhamstercom")
+    ap.add_argument("--days", type=int, default=30)
+    ap.add_argument("--limit", type=int, default=0, help="0 = bez limitu")
+    ap.add_argument("--rps", type=float, default=3.0, help="zadan na sekunde")
+    ap.add_argument("--dry-run", action="store_true")
+    args = ap.parse_args()
+
+    with session_scope() as s:
+        rows = s.execute(_SQL_KANDYDACI, {"origin": args.origin, "days": args.days}).all()
+    rows.sort(key=lambda r: r.created_at, reverse=True)
+    if args.limit:
+        rows = rows[: args.limit]
+    print(f"kandydatow: {len(rows)} ({args.origin}, {args.days} dni){' [dry-run]' if args.dry_run else ''}")
+    if args.dry_run:
+        return 0
+
+    ok = miss = err = 0
+    okres = 1.0 / max(args.rps, 0.1)
+    t_last = 0.0
+    for i, row in enumerate(rows, 1):
+        czekaj = okres - (time.monotonic() - t_last)
+        if czekaj > 0:
+            time.sleep(czekaj)
+        t_last = time.monotonic()
+        try:
+            r = browser_get(row.page_url, timeout=15.0)
+            if r.status_code != 200:
+                err += 1
+                continue
+            thumb = extract_thumbnail_url(r.text)
+        except Exception as e:  # noqa: BLE001
+            log.debug("fetch %s: %s", row.page_url, e)
+            err += 1
+            continue
+        if not thumb:
+            miss += 1
+            continue
+        with session_scope() as s:
+            s.execute(
+                text(
+                    "UPDATE playback_sources SET thumbnail_url = :t, updated_at = now() "
+                    "WHERE scene_id = :sid AND dead_at IS NULL AND thumbnail_url IS NULL"
+                ),
+                {"t": thumb, "sid": row.scene_id},
+            )
+            s.commit()
+        ok += 1
+        if i % 200 == 0:
+            print(f"{i}/{len(rows)} ok={ok} brak={miss} blad={err}", flush=True)
+    print(f"KONIEC {args.origin}: ok={ok} brak={miss} blad={err} z {len(rows)}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())