diff --git a/app/connectors/direct_scrapers/hqporner.py b/app/connectors/direct_scrapers/hqporner.py index 7f4337a..e73f816 100644 --- a/app/connectors/direct_scrapers/hqporner.py +++ b/app/connectors/direct_scrapers/hqporner.py @@ -19,6 +19,7 @@ from collections.abc import Iterator from app.connectors.base import RawPerformer, RawPlaybackSource, RawScene from app.connectors.direct_scrapers._search_base import BaseSearchScraper from app.connectors.direct_scrapers.base import BaseDirectTubeScraper +from app.connectors.direct_scrapers.hqporner_browse import listing_covers from app.extractors import browser_get log = logging.getLogger(__name__) @@ -66,6 +67,7 @@ class HQPornerScraper(BaseDirectTubeScraper): seen_urls: set[str] = set() yielded = 0 + covers = listing_covers(r.text) for m in _SCENE_HREF_RE.finditer(r.text): scene_id = m.group(1) slug_part = m.group(2) @@ -95,6 +97,7 @@ class HQPornerScraper(BaseDirectTubeScraper): RawPlaybackSource( origin="tube:hqpornercom", page_url=scene_url, + thumbnail_url=covers.get(scene_id), ) ], # Wymuszamy hint performera = query, search per performer name znaczy diff --git a/app/connectors/direct_scrapers/hqporner_browse.py b/app/connectors/direct_scrapers/hqporner_browse.py index 118b605..06271af 100644 --- a/app/connectors/direct_scrapers/hqporner_browse.py +++ b/app/connectors/direct_scrapers/hqporner_browse.py @@ -12,7 +12,7 @@ Detail (`/hdporn/-.html`) ma komplet metadanych: - długość → meta description "Video duration is 46min 58sec", - performerki → `Display Name` (display, nie slug), - tagi → `/category/`, - - miniaturka → `fastporndelivery.hqporner.com/imgs/.../_main.jpg`. + - miniaturki NIE ma: okladka jest tylko w kafelku listingu (patrz _COVER_RE). `external_id` MUSI być identyczny jak w `HQPornerScraper` (search): `hqpornercom:`. Inaczej ta sama scena z obu ścieżek zrobiłaby się dwoma wpisami. @@ -52,7 +52,21 @@ _ACTRESS_RE = re.compile( re.IGNORECASE | re.DOTALL, ) _CATEGORY_RE = re.compile(r'href=[\'"]/category/([a-z0-9\-_]+)[\'"]', re.IGNORECASE) -_THUMB_RE = re.compile(r"fastporndelivery\.hqporner\.com/imgs/[^\"'\s]+_main\.jpg", re.IGNORECASE) +# Okladka filmu jest TYLKO w kafelku listingu (``). Strona +# filmu wlasnej okladki nie ma, a pierwszy `_main.jpg` na niej to kafelek innego, +# polecanego filmu. Braniem go ~90% zrodel hqporner dostalo cudze miniatury (62 tys. +# zrodel, 5,4 tys. roznych miniatur; zgloszenie 0982b000, 2026-09-24). +_COVER_RE = re.compile( + r'id=["\']cover_(\d+)["\']\s+src=["\'](//fastporndelivery\.hqporner\.com/imgs/[^"\']+_main\.jpg)', + re.IGNORECASE, +) + + +def listing_covers(listing_html: str) -> dict[str, str]: + """Okladki z kafelkow listingu (glowny, wyszukiwarka, kategorie): {id filmu: url}.""" + return {vid: "https:" + src for vid, src in _COVER_RE.findall(listing_html)} + + _TAG_STRIP = re.compile(r"<[^>]+>") # hqporner wrzuca do /category/ także jakość ("1080p-porn", "4k-porn"). To nie jest tag @@ -71,6 +85,10 @@ class HQPornerBrowseScraper(BaseBrowseScraper): return _BASE + "/" if page <= 1 else f"{_BASE}/?p={page}" def _extract_scene_urls(self, listing_html: str) -> list[str]: + # Okladki zapamietujemy tu, bo _parse_detail dostaje juz tylko strone filmu. + if not hasattr(self, "_covers"): + self._covers: dict[str, str] = {} + self._covers.update(listing_covers(listing_html)) seen: set[str] = set() out: list[str] = [] for m in _SCENE_HREF_RE.finditer(listing_html): @@ -122,10 +140,10 @@ class HQPornerBrowseScraper(BaseBrowseScraper): ) ) - thumbnail_url: str | None = None - thm = _THUMB_RE.search(detail_html) - if thm: - thumbnail_url = "https://" + thm.group(0) + # Tylko okladka z kafelka listingu. Brak kafelka = brak miniatury, nigdy obrazek ze + # strony filmu (patrz _COVER_RE). + vid = _SCENE_HREF_RE.search(scene_url) + thumbnail_url = getattr(self, "_covers", {}).get(vid.group(1)) if vid else None fingerprints: list[RawFingerprint] = [] if thumbnail_url: diff --git a/scripts/hqporner_fix_thumbs.py b/scripts/hqporner_fix_thumbs.py new file mode 100644 index 0000000..903b847 --- /dev/null +++ b/scripts/hqporner_fix_thumbs.py @@ -0,0 +1,104 @@ +"""Naprawa miniatur hqporner: okladki z kafelkow listingu zamiast cudzych z detali. + +Scraper bral pierwszy `_main.jpg` ze strony filmu, a to kafelek polecanego filmu +(app/connectors/direct_scrapers/hqporner_browse.py, _COVER_RE). Skrypt przechodzi +listing `https://hqporner.com/?p=N`, zbiera {id filmu: okladka} i poprawia +playback_sources.thumbnail_url zrodel tube:hqpornercom. + +Uzycie: python3 -u scripts/hqporner_fix_thumbs.py [--commit] [--max-pages N] [--start N] +Bez --commit tylko liczy. Postep co 50 stron; wznawianie przez --start. +""" +from __future__ import annotations + +import argparse +import re +import time + +from sqlalchemy import text + +from app.connectors.direct_scrapers.hqporner_browse import listing_covers +from app.db import session_scope +from app.extractors import browser_get + +_BASE = "https://hqporner.com" +_ID_RE = re.compile(r"/hdporn/(\d+)-") + + +def _load_sources() -> dict[str, list[tuple[str, str | None]]]: + """{id filmu: [(id zrodla, obecna miniatura)]} dla zrodel hqporner.""" + out: dict[str, list[tuple[str, str | None]]] = {} + with session_scope() as s: + for sid, page, thumb in s.execute( + text("SELECT id, page_url, thumbnail_url FROM playback_sources WHERE origin = 'tube:hqpornercom'") + ): + m = _ID_RE.search(page or "") + if m: + out.setdefault(m.group(1), []).append((str(sid), thumb)) + return out + + +def _apply(fixes: list[tuple[str, str]], commit: bool) -> None: + if not fixes or not commit: + return + with session_scope() as s: + s.execute( + text("UPDATE playback_sources SET thumbnail_url = :t, updated_at = now() WHERE id = CAST(:i AS uuid)"), + [{"i": i, "t": t} for i, t in fixes], + ) + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--commit", action="store_true") + ap.add_argument("--max-pages", type=int, default=5000) + ap.add_argument("--start", type=int, default=1) + ap.add_argument("--pause", type=float, default=0.3) + args = ap.parse_args() + + sources = _load_sources() + print(f"zrodel hqporner: {sum(len(v) for v in sources.values())}, filmow: {len(sources)}", flush=True) + seen_ids: set[str] = set() + fixed = same = empty_streak = 0 + pending: list[tuple[str, str]] = [] + t0 = time.time() + for page in range(args.start, args.start + args.max_pages): + url = _BASE + "/" if page <= 1 else f"{_BASE}/?p={page}" + covers: dict[str, str] = {} + for attempt in range(3): + try: + covers = listing_covers(browser_get(url, timeout=40).text) + break + except Exception as e: # noqa: BLE001 + print(f" strona {page} proba {attempt + 1}: {e}", flush=True) + time.sleep(5) + new = {k: v for k, v in covers.items() if k not in seen_ids} + if not new: + empty_streak += 1 + if empty_streak >= 3: + print(f"koniec listingu na stronie {page}", flush=True) + break + else: + empty_streak = 0 + seen_ids.update(new) + for vid, cover in new.items(): + for sid, thumb in sources.get(vid, []): + if thumb == cover: + same += 1 + else: + pending.append((sid, cover)) + fixed += 1 + if page % 50 == 0: + _apply(pending, args.commit) + pending.clear() + print(f"strona {page}: filmow w listingu {len(seen_ids)}, poprawionych {fixed}, " + f"juz dobrych {same}, {time.time() - t0:.0f} s", flush=True) + time.sleep(args.pause) + _apply(pending, args.commit) + ours = set(sources) + print(f"KONIEC: filmow w listingu {len(seen_ids)}, naszych filmow {len(ours)}, " + f"znalezionych {len(ours & seen_ids)}, poprawionych zrodel {fixed}, juz dobrych {same}, " + f"{'ZAPISANE' if args.commit else 'bez zapisu'}, {time.time() - t0:.0f} s", flush=True) + + +if __name__ == "__main__": + main()