diff --git a/app/connectors/direct_scrapers/_search_base.py b/app/connectors/direct_scrapers/_search_base.py index b60dc63..06403f8 100644 --- a/app/connectors/direct_scrapers/_search_base.py +++ b/app/connectors/direct_scrapers/_search_base.py @@ -107,6 +107,15 @@ class BaseSearchScraper(BaseDirectTubeScraper): """ return urllib.parse.quote_plus(query.strip()) + def _thumbnail_map(self, listing_html: str) -> dict[str, str]: + """Opcjonalny hak: mapa scene_url -> thumbnail_url z calej strony wynikow. + + Domyslnie pusta i wtedy dziala szukanie `` w oknie wokol linku. Tuby, + ktore renderuja karty z JSON-a (xhamster: `window.initials` z `pageURL` i + `thumbURL`), maja miniature dla KAZDEJ karty w jednym miejscu, a okno tekstu + gubilo polowe (54% zrodel xhamster bez miniatury, zgloszenie 8ec9dc96).""" + return {} + def _fetch_scene_metadata( self, scene_url: str ) -> tuple[RawStudio | None, list[RawPerformer], list[RawTag]] | None: @@ -169,6 +178,11 @@ class BaseSearchScraper(BaseDirectTubeScraper): } seen: set[str] = set() + try: + thumb_map = self._thumbnail_map(r.text) + except Exception as e: # noqa: BLE001 + log.debug("%s thumbnail map failed: %s", self.sitetag, e) + thumb_map = {} yielded = 0 for m in self._scene_url_re.finditer(r.text): scene_url = self._scene_url_from_match(m).strip() @@ -213,8 +227,8 @@ class BaseSearchScraper(BaseDirectTubeScraper): window_start = max(0, m.start() - self._thumbnail_window) window_end = min(len(r.text), m.end() + self._thumbnail_window) window_html = r.text[window_start:window_end] - thumb_url: str | None = None - img_m = _IMG_SRC_RE.search(window_html) + thumb_url: str | None = thumb_map.get(scene_url) + img_m = None if thumb_url else _IMG_SRC_RE.search(window_html) if img_m: thumb_url = img_m.group(1).strip() if thumb_url.startswith("//"): diff --git a/app/connectors/direct_scrapers/mypornerleak.py b/app/connectors/direct_scrapers/mypornerleak.py index 44015b1..dc20442 100644 --- a/app/connectors/direct_scrapers/mypornerleak.py +++ b/app/connectors/direct_scrapers/mypornerleak.py @@ -33,7 +33,9 @@ _BASE = "https://mypornerleak.com" _ARTICLE_RE = re.compile(r']*\bclass="([^"]+)"', re.IGNORECASE) _LINK_RE = re.compile(r'(?:<[^>]+>)*\s*(\d{1,2}:\d{2}(?::\d{2})?)', re.IGNORECASE) -_THUMB_RE = re.compile(r'data-src="([^"]+)"', re.IGNORECASE) +# Do 2026-09 karty mialy lazy `data-src`, teraz zwykle `src` (class video-main-thumb); +# stary wzorzec zostawial 59% zrodel bez miniatury (zgloszenie 94e1da85). +_THUMB_RE = re.compile(r']+(?:data-src|src)="(https?://[^"]+\.(?:jpg|jpeg|png|webp)[^"]*)"', re.IGNORECASE) _WN_HOST_RE = re.compile(r"https?://w\d+\.mypornerleak\.com", re.IGNORECASE) _CLASS_ACTOR_RE = re.compile(r"\bactors-([a-z0-9-]+)") _CLASS_CAT_RE = re.compile(r"\bcategory-([a-z0-9-]+)") diff --git a/app/connectors/direct_scrapers/mypornerleak_browse.py b/app/connectors/direct_scrapers/mypornerleak_browse.py index 5b5f023..be18199 100644 --- a/app/connectors/direct_scrapers/mypornerleak_browse.py +++ b/app/connectors/direct_scrapers/mypornerleak_browse.py @@ -28,6 +28,7 @@ from app.connectors.direct_scrapers._browse_base import ( compute_thumbnail_phash, ) from app.extractors import browser_get +from app.extractors.thumb_extract import extract_thumbnail_url from app.normalize.text import slugify log = logging.getLogger(__name__) @@ -85,6 +86,15 @@ class MyPornerLeakBrowseScraper(BaseBrowseScraper): emb = p.get("_embedded") or {} fm = emb.get("wp:featuredmedia") or [] thumb = (fm[0].get("source_url") if fm and isinstance(fm[0], dict) else None) or None + if not thumb: + # Od 2026-09 REST nie ma featured media (featured_media=0, brak img w + # tresci); strona posta ma og:image (zgloszenie 94e1da85). + try: + pr = browser_get(link, timeout=15.0) + if pr.status_code == 200: + thumb = extract_thumbnail_url(pr.text) + except Exception as e: # noqa: BLE001 + log.debug("mypornerleak og:image fetch failed for %s: %s", link, e) studio: RawStudio | None = None tags: list[RawTag] = [] diff --git a/app/connectors/direct_scrapers/xhamster.py b/app/connectors/direct_scrapers/xhamster.py index 311cce8..5fcf5b9 100644 --- a/app/connectors/direct_scrapers/xhamster.py +++ b/app/connectors/direct_scrapers/xhamster.py @@ -7,6 +7,13 @@ from __future__ import annotations import re +# Karty sa renderowane z JSON-a w HTML (`window.initials`): kazda ma `pageURL` i +# `thumbURL` (526x298 webp). Szukanie `` w oknie tekstu gubilo polowe scen, bo +# ten sam link wystepuje dwa razy (miniatura i tytul), a img jest tylko przy pierwszym. +_JSON_CARD_RE = re.compile( + r'"pageURL":"(https:\\/\\/xhamster\.com\\/videos\\/[^"]+)","thumbURL":"([^"]+)"' +) + from app.connectors.direct_scrapers._search_base import BaseSearchScraper @@ -17,3 +24,9 @@ class XHamsterScraper(BaseSearchScraper): r'href="(?Phttps://xhamster\.com/videos/(?P[a-z0-9_\-]+))"', re.IGNORECASE, ) + + def _thumbnail_map(self, listing_html: str) -> dict[str, str]: + out: dict[str, str] = {} + for page_url, thumb in _JSON_CARD_RE.findall(listing_html): + out[page_url.replace("\\/", "/")] = thumb.replace("\\/", "/") + return out diff --git a/scripts/backfill_thumbs.py b/scripts/backfill_thumbs.py new file mode 100644 index 0000000..d357222 --- /dev/null +++ b/scripts/backfill_thumbs.py @@ -0,0 +1,102 @@ +"""Uzupelnia brakujace miniatury scen z jednego tube'a ze strony sceny (og:image itd.). + +Po co: scraper xhamster gubil miniatury polowy scen (szukal w oknie tekstu, a +karty sa w JSON-ie), a mypornerleak przestal wystawiac obrazek w REST. Ingest jest +juz naprawiony (2026-09-14), ten skrypt sprzata to, co weszlo bez miniatury: wchodzi +na strone sceny jak przycisk „Refresh thumbnail" w apce i zapisuje miniature na +wszystkich zrodlach sceny bez miniatury. + +Kolejnosc: od najnowszych scen, bo te sa na gorze list. Limit czasu przez --days. + + docker compose run -d --rm --name goon-thumbs-xh worker python3 -u /srv/scripts/backfill_thumbs.py --origin tube:xhamstercom --days 30 --rps 3 + docker compose run -d --rm --name goon-thumbs-mpl worker python3 -u /srv/scripts/backfill_thumbs.py --origin tube:mypornerleakcom --days 30 --rps 2 +""" +from __future__ import annotations + +import argparse +import logging +import sys +import time + +from sqlalchemy import text + +from app.db import session_scope +from app.extractors._fetch import browser_get +from app.extractors.thumb_extract import extract_thumbnail_url + +log = logging.getLogger("backfill_thumbs") + +_SQL_KANDYDACI = text( + """ + SELECT DISTINCT ON (s.id) s.id AS scene_id, ps.page_url, s.created_at + FROM scenes s + JOIN playback_sources ps ON ps.scene_id = s.id AND ps.dead_at IS NULL AND ps.origin = :origin + WHERE ps.thumbnail_url IS NULL + AND s.created_at > now() - make_interval(days => :days) + AND NOT EXISTS ( + SELECT 1 FROM playback_sources q + WHERE q.scene_id = s.id AND q.dead_at IS NULL AND q.thumbnail_url IS NOT NULL + ) + ORDER BY s.id, ps.created_at + """ +) + + +def main() -> int: + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") + ap = argparse.ArgumentParser() + ap.add_argument("--origin", required=True, help="np. tube:xhamstercom") + ap.add_argument("--days", type=int, default=30) + ap.add_argument("--limit", type=int, default=0, help="0 = bez limitu") + ap.add_argument("--rps", type=float, default=3.0, help="zadan na sekunde") + ap.add_argument("--dry-run", action="store_true") + args = ap.parse_args() + + with session_scope() as s: + rows = s.execute(_SQL_KANDYDACI, {"origin": args.origin, "days": args.days}).all() + rows.sort(key=lambda r: r.created_at, reverse=True) + if args.limit: + rows = rows[: args.limit] + print(f"kandydatow: {len(rows)} ({args.origin}, {args.days} dni){' [dry-run]' if args.dry_run else ''}") + if args.dry_run: + return 0 + + ok = miss = err = 0 + okres = 1.0 / max(args.rps, 0.1) + t_last = 0.0 + for i, row in enumerate(rows, 1): + czekaj = okres - (time.monotonic() - t_last) + if czekaj > 0: + time.sleep(czekaj) + t_last = time.monotonic() + try: + r = browser_get(row.page_url, timeout=15.0) + if r.status_code != 200: + err += 1 + continue + thumb = extract_thumbnail_url(r.text) + except Exception as e: # noqa: BLE001 + log.debug("fetch %s: %s", row.page_url, e) + err += 1 + continue + if not thumb: + miss += 1 + continue + with session_scope() as s: + s.execute( + text( + "UPDATE playback_sources SET thumbnail_url = :t, updated_at = now() " + "WHERE scene_id = :sid AND dead_at IS NULL AND thumbnail_url IS NULL" + ), + {"t": thumb, "sid": row.scene_id}, + ) + s.commit() + ok += 1 + if i % 200 == 0: + print(f"{i}/{len(rows)} ok={ok} brak={miss} blad={err}", flush=True) + print(f"KONIEC {args.origin}: ok={ok} brak={miss} blad={err} z {len(rows)}") + return 0 + + +if __name__ == "__main__": + sys.exit(main())