"""Naprawa miniatur hqporner: okladki z kafelkow listingu zamiast cudzych z detali. Scraper bral pierwszy `_main.jpg` ze strony filmu, a to kafelek polecanego filmu (app/connectors/direct_scrapers/hqporner_browse.py, _COVER_RE). Skrypt przechodzi listing `https://hqporner.com/?p=N`, zbiera {id filmu: okladka} i poprawia playback_sources.thumbnail_url zrodel tube:hqpornercom. Uzycie: python3 -u scripts/hqporner_fix_thumbs.py [--commit] [--max-pages N] [--start N] Bez --commit tylko liczy. Postep co 50 stron; wznawianie przez --start. """ from __future__ import annotations import argparse import re import time from sqlalchemy import text from app.connectors.direct_scrapers.hqporner_browse import listing_covers from app.db import session_scope from app.extractors import browser_get _BASE = "https://hqporner.com" _ID_RE = re.compile(r"/hdporn/(\d+)-") def _load_sources() -> dict[str, list[tuple[str, str | None]]]: """{id filmu: [(id zrodla, obecna miniatura)]} dla zrodel hqporner.""" out: dict[str, list[tuple[str, str | None]]] = {} with session_scope() as s: for sid, page, thumb in s.execute( text("SELECT id, page_url, thumbnail_url FROM playback_sources WHERE origin = 'tube:hqpornercom'") ): m = _ID_RE.search(page or "") if m: out.setdefault(m.group(1), []).append((str(sid), thumb)) return out def _apply(fixes: list[tuple[str, str]], commit: bool) -> None: if not fixes or not commit: return with session_scope() as s: s.execute( text("UPDATE playback_sources SET thumbnail_url = :t, updated_at = now() WHERE id = CAST(:i AS uuid)"), [{"i": i, "t": t} for i, t in fixes], ) def main() -> None: ap = argparse.ArgumentParser() ap.add_argument("--commit", action="store_true") ap.add_argument("--max-pages", type=int, default=5000) ap.add_argument("--start", type=int, default=1) ap.add_argument("--pause", type=float, default=0.3) args = ap.parse_args() sources = _load_sources() print(f"zrodel hqporner: {sum(len(v) for v in sources.values())}, filmow: {len(sources)}", flush=True) seen_ids: set[str] = set() fixed = same = empty_streak = 0 pending: list[tuple[str, str]] = [] t0 = time.time() for page in range(args.start, args.start + args.max_pages): url = _BASE + "/" if page <= 1 else f"{_BASE}/?p={page}" covers: dict[str, str] = {} for attempt in range(3): try: covers = listing_covers(browser_get(url, timeout=40).text) break except Exception as e: # noqa: BLE001 print(f" strona {page} proba {attempt + 1}: {e}", flush=True) time.sleep(5) new = {k: v for k, v in covers.items() if k not in seen_ids} if not new: empty_streak += 1 if empty_streak >= 3: print(f"koniec listingu na stronie {page}", flush=True) break else: empty_streak = 0 seen_ids.update(new) for vid, cover in new.items(): for sid, thumb in sources.get(vid, []): if thumb == cover: same += 1 else: pending.append((sid, cover)) fixed += 1 if page % 50 == 0: _apply(pending, args.commit) pending.clear() print(f"strona {page}: filmow w listingu {len(seen_ids)}, poprawionych {fixed}, " f"juz dobrych {same}, {time.time() - t0:.0f} s", flush=True) time.sleep(args.pause) _apply(pending, args.commit) ours = set(sources) print(f"KONIEC: filmow w listingu {len(seen_ids)}, naszych filmow {len(ours)}, " f"znalezionych {len(ours & seen_ids)}, poprawionych zrodel {fixed}, juz dobrych {same}, " f"{'ZAPISANE' if args.commit else 'bez zapisu'}, {time.time() - t0:.0f} s", flush=True) if __name__ == "__main__": main()