diff --git a/app/connectors/direct_scrapers/_search_base.py b/app/connectors/direct_scrapers/_search_base.py
index b60dc63..06403f8 100644
--- a/app/connectors/direct_scrapers/_search_base.py
+++ b/app/connectors/direct_scrapers/_search_base.py
@@ -107,6 +107,15 @@ class BaseSearchScraper(BaseDirectTubeScraper):
"""
return urllib.parse.quote_plus(query.strip())
+ def _thumbnail_map(self, listing_html: str) -> dict[str, str]:
+ """Opcjonalny hak: mapa scene_url -> thumbnail_url z calej strony wynikow.
+
+ Domyslnie pusta i wtedy dziala szukanie `
` w oknie wokol linku. Tuby,
+ ktore renderuja karty z JSON-a (xhamster: `window.initials` z `pageURL` i
+ `thumbURL`), maja miniature dla KAZDEJ karty w jednym miejscu, a okno tekstu
+ gubilo polowe (54% zrodel xhamster bez miniatury, zgloszenie 8ec9dc96)."""
+ return {}
+
def _fetch_scene_metadata(
self, scene_url: str
) -> tuple[RawStudio | None, list[RawPerformer], list[RawTag]] | None:
@@ -169,6 +178,11 @@ class BaseSearchScraper(BaseDirectTubeScraper):
}
seen: set[str] = set()
+ try:
+ thumb_map = self._thumbnail_map(r.text)
+ except Exception as e: # noqa: BLE001
+ log.debug("%s thumbnail map failed: %s", self.sitetag, e)
+ thumb_map = {}
yielded = 0
for m in self._scene_url_re.finditer(r.text):
scene_url = self._scene_url_from_match(m).strip()
@@ -213,8 +227,8 @@ class BaseSearchScraper(BaseDirectTubeScraper):
window_start = max(0, m.start() - self._thumbnail_window)
window_end = min(len(r.text), m.end() + self._thumbnail_window)
window_html = r.text[window_start:window_end]
- thumb_url: str | None = None
- img_m = _IMG_SRC_RE.search(window_html)
+ thumb_url: str | None = thumb_map.get(scene_url)
+ img_m = None if thumb_url else _IMG_SRC_RE.search(window_html)
if img_m:
thumb_url = img_m.group(1).strip()
if thumb_url.startswith("//"):
diff --git a/app/connectors/direct_scrapers/mypornerleak.py b/app/connectors/direct_scrapers/mypornerleak.py
index 44015b1..dc20442 100644
--- a/app/connectors/direct_scrapers/mypornerleak.py
+++ b/app/connectors/direct_scrapers/mypornerleak.py
@@ -33,7 +33,9 @@ _BASE = "https://mypornerleak.com"
_ARTICLE_RE = re.compile(r']*\bclass="([^"]+)"', re.IGNORECASE)
_LINK_RE = re.compile(r'(?:<[^>]+>)*\s*(\d{1,2}:\d{2}(?::\d{2})?)', re.IGNORECASE)
-_THUMB_RE = re.compile(r'data-src="([^"]+)"', re.IGNORECASE)
+# Do 2026-09 karty mialy lazy `data-src`, teraz zwykle `src` (class video-main-thumb);
+# stary wzorzec zostawial 59% zrodel bez miniatury (zgloszenie 94e1da85).
+_THUMB_RE = re.compile(r'
]+(?:data-src|src)="(https?://[^"]+\.(?:jpg|jpeg|png|webp)[^"]*)"', re.IGNORECASE)
_WN_HOST_RE = re.compile(r"https?://w\d+\.mypornerleak\.com", re.IGNORECASE)
_CLASS_ACTOR_RE = re.compile(r"\bactors-([a-z0-9-]+)")
_CLASS_CAT_RE = re.compile(r"\bcategory-([a-z0-9-]+)")
diff --git a/app/connectors/direct_scrapers/mypornerleak_browse.py b/app/connectors/direct_scrapers/mypornerleak_browse.py
index 5b5f023..be18199 100644
--- a/app/connectors/direct_scrapers/mypornerleak_browse.py
+++ b/app/connectors/direct_scrapers/mypornerleak_browse.py
@@ -28,6 +28,7 @@ from app.connectors.direct_scrapers._browse_base import (
compute_thumbnail_phash,
)
from app.extractors import browser_get
+from app.extractors.thumb_extract import extract_thumbnail_url
from app.normalize.text import slugify
log = logging.getLogger(__name__)
@@ -85,6 +86,15 @@ class MyPornerLeakBrowseScraper(BaseBrowseScraper):
emb = p.get("_embedded") or {}
fm = emb.get("wp:featuredmedia") or []
thumb = (fm[0].get("source_url") if fm and isinstance(fm[0], dict) else None) or None
+ if not thumb:
+ # Od 2026-09 REST nie ma featured media (featured_media=0, brak img w
+ # tresci); strona posta ma og:image (zgloszenie 94e1da85).
+ try:
+ pr = browser_get(link, timeout=15.0)
+ if pr.status_code == 200:
+ thumb = extract_thumbnail_url(pr.text)
+ except Exception as e: # noqa: BLE001
+ log.debug("mypornerleak og:image fetch failed for %s: %s", link, e)
studio: RawStudio | None = None
tags: list[RawTag] = []
diff --git a/app/connectors/direct_scrapers/xhamster.py b/app/connectors/direct_scrapers/xhamster.py
index 311cce8..5fcf5b9 100644
--- a/app/connectors/direct_scrapers/xhamster.py
+++ b/app/connectors/direct_scrapers/xhamster.py
@@ -7,6 +7,13 @@ from __future__ import annotations
import re
+# Karty sa renderowane z JSON-a w HTML (`window.initials`): kazda ma `pageURL` i
+# `thumbURL` (526x298 webp). Szukanie `
` w oknie tekstu gubilo polowe scen, bo
+# ten sam link wystepuje dwa razy (miniatura i tytul), a img jest tylko przy pierwszym.
+_JSON_CARD_RE = re.compile(
+ r'"pageURL":"(https:\\/\\/xhamster\.com\\/videos\\/[^"]+)","thumbURL":"([^"]+)"'
+)
+
from app.connectors.direct_scrapers._search_base import BaseSearchScraper
@@ -17,3 +24,9 @@ class XHamsterScraper(BaseSearchScraper):
r'href="(?Phttps://xhamster\.com/videos/(?P[a-z0-9_\-]+))"',
re.IGNORECASE,
)
+
+ def _thumbnail_map(self, listing_html: str) -> dict[str, str]:
+ out: dict[str, str] = {}
+ for page_url, thumb in _JSON_CARD_RE.findall(listing_html):
+ out[page_url.replace("\\/", "/")] = thumb.replace("\\/", "/")
+ return out
diff --git a/scripts/backfill_thumbs.py b/scripts/backfill_thumbs.py
new file mode 100644
index 0000000..d357222
--- /dev/null
+++ b/scripts/backfill_thumbs.py
@@ -0,0 +1,102 @@
+"""Uzupelnia brakujace miniatury scen z jednego tube'a ze strony sceny (og:image itd.).
+
+Po co: scraper xhamster gubil miniatury polowy scen (szukal
w oknie tekstu, a
+karty sa w JSON-ie), a mypornerleak przestal wystawiac obrazek w REST. Ingest jest
+juz naprawiony (2026-09-14), ten skrypt sprzata to, co weszlo bez miniatury: wchodzi
+na strone sceny jak przycisk „Refresh thumbnail" w apce i zapisuje miniature na
+wszystkich zrodlach sceny bez miniatury.
+
+Kolejnosc: od najnowszych scen, bo te sa na gorze list. Limit czasu przez --days.
+
+ docker compose run -d --rm --name goon-thumbs-xh worker python3 -u /srv/scripts/backfill_thumbs.py --origin tube:xhamstercom --days 30 --rps 3
+ docker compose run -d --rm --name goon-thumbs-mpl worker python3 -u /srv/scripts/backfill_thumbs.py --origin tube:mypornerleakcom --days 30 --rps 2
+"""
+from __future__ import annotations
+
+import argparse
+import logging
+import sys
+import time
+
+from sqlalchemy import text
+
+from app.db import session_scope
+from app.extractors._fetch import browser_get
+from app.extractors.thumb_extract import extract_thumbnail_url
+
+log = logging.getLogger("backfill_thumbs")
+
+_SQL_KANDYDACI = text(
+ """
+ SELECT DISTINCT ON (s.id) s.id AS scene_id, ps.page_url, s.created_at
+ FROM scenes s
+ JOIN playback_sources ps ON ps.scene_id = s.id AND ps.dead_at IS NULL AND ps.origin = :origin
+ WHERE ps.thumbnail_url IS NULL
+ AND s.created_at > now() - make_interval(days => :days)
+ AND NOT EXISTS (
+ SELECT 1 FROM playback_sources q
+ WHERE q.scene_id = s.id AND q.dead_at IS NULL AND q.thumbnail_url IS NOT NULL
+ )
+ ORDER BY s.id, ps.created_at
+ """
+)
+
+
+def main() -> int:
+ logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
+ ap = argparse.ArgumentParser()
+ ap.add_argument("--origin", required=True, help="np. tube:xhamstercom")
+ ap.add_argument("--days", type=int, default=30)
+ ap.add_argument("--limit", type=int, default=0, help="0 = bez limitu")
+ ap.add_argument("--rps", type=float, default=3.0, help="zadan na sekunde")
+ ap.add_argument("--dry-run", action="store_true")
+ args = ap.parse_args()
+
+ with session_scope() as s:
+ rows = s.execute(_SQL_KANDYDACI, {"origin": args.origin, "days": args.days}).all()
+ rows.sort(key=lambda r: r.created_at, reverse=True)
+ if args.limit:
+ rows = rows[: args.limit]
+ print(f"kandydatow: {len(rows)} ({args.origin}, {args.days} dni){' [dry-run]' if args.dry_run else ''}")
+ if args.dry_run:
+ return 0
+
+ ok = miss = err = 0
+ okres = 1.0 / max(args.rps, 0.1)
+ t_last = 0.0
+ for i, row in enumerate(rows, 1):
+ czekaj = okres - (time.monotonic() - t_last)
+ if czekaj > 0:
+ time.sleep(czekaj)
+ t_last = time.monotonic()
+ try:
+ r = browser_get(row.page_url, timeout=15.0)
+ if r.status_code != 200:
+ err += 1
+ continue
+ thumb = extract_thumbnail_url(r.text)
+ except Exception as e: # noqa: BLE001
+ log.debug("fetch %s: %s", row.page_url, e)
+ err += 1
+ continue
+ if not thumb:
+ miss += 1
+ continue
+ with session_scope() as s:
+ s.execute(
+ text(
+ "UPDATE playback_sources SET thumbnail_url = :t, updated_at = now() "
+ "WHERE scene_id = :sid AND dead_at IS NULL AND thumbnail_url IS NULL"
+ ),
+ {"t": thumb, "sid": row.scene_id},
+ )
+ s.commit()
+ ok += 1
+ if i % 200 == 0:
+ print(f"{i}/{len(rows)} ok={ok} brak={miss} blad={err}", flush=True)
+ print(f"KONIEC {args.origin}: ok={ok} brak={miss} blad={err} z {len(rows)}")
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())