"""Uzupelnia brakujace miniatury scen z jednego tube'a ze strony sceny (og:image itd.). Po co: scraper xhamster gubil miniatury polowy scen (szukal w oknie tekstu, a karty sa w JSON-ie), a mypornerleak przestal wystawiac obrazek w REST. Ingest jest juz naprawiony (2026-09-14), ten skrypt sprzata to, co weszlo bez miniatury: wchodzi na strone sceny jak przycisk „Refresh thumbnail" w apce i zapisuje miniature na wszystkich zrodlach sceny bez miniatury. Kolejnosc: od najnowszych scen, bo te sa na gorze list. Limit czasu przez --days. docker compose run -d --rm --name goon-thumbs-xh worker python3 -u /srv/scripts/backfill_thumbs.py --origin tube:xhamstercom --days 30 --rps 3 docker compose run -d --rm --name goon-thumbs-mpl worker python3 -u /srv/scripts/backfill_thumbs.py --origin tube:mypornerleakcom --days 30 --rps 2 """ from __future__ import annotations import argparse import logging import sys import time from sqlalchemy import text from app.db import session_scope from app.extractors._fetch import browser_get from app.extractors.thumb_extract import extract_thumbnail_url log = logging.getLogger("backfill_thumbs") _SQL_KANDYDACI = text( """ SELECT DISTINCT ON (s.id) s.id AS scene_id, ps.page_url, s.created_at FROM scenes s JOIN playback_sources ps ON ps.scene_id = s.id AND ps.dead_at IS NULL AND ps.origin = :origin WHERE ps.thumbnail_url IS NULL AND s.created_at > now() - make_interval(days => :days) AND NOT EXISTS ( SELECT 1 FROM playback_sources q WHERE q.scene_id = s.id AND q.dead_at IS NULL AND q.thumbnail_url IS NOT NULL ) ORDER BY s.id, ps.created_at """ ) def main() -> int: logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") ap = argparse.ArgumentParser() ap.add_argument("--origin", required=True, help="np. tube:xhamstercom") ap.add_argument("--days", type=int, default=30) ap.add_argument("--limit", type=int, default=0, help="0 = bez limitu") ap.add_argument("--rps", type=float, default=3.0, help="zadan na sekunde") ap.add_argument("--dry-run", action="store_true") args = ap.parse_args() with session_scope() as s: rows = s.execute(_SQL_KANDYDACI, {"origin": args.origin, "days": args.days}).all() rows.sort(key=lambda r: r.created_at, reverse=True) if args.limit: rows = rows[: args.limit] print(f"kandydatow: {len(rows)} ({args.origin}, {args.days} dni){' [dry-run]' if args.dry_run else ''}") if args.dry_run: return 0 ok = miss = err = 0 okres = 1.0 / max(args.rps, 0.1) t_last = 0.0 for i, row in enumerate(rows, 1): czekaj = okres - (time.monotonic() - t_last) if czekaj > 0: time.sleep(czekaj) t_last = time.monotonic() try: r = browser_get(row.page_url, timeout=15.0) if r.status_code != 200: err += 1 continue thumb = extract_thumbnail_url(r.text) except Exception as e: # noqa: BLE001 log.debug("fetch %s: %s", row.page_url, e) err += 1 continue if not thumb: miss += 1 continue with session_scope() as s: s.execute( text( "UPDATE playback_sources SET thumbnail_url = :t, updated_at = now() " "WHERE scene_id = :sid AND dead_at IS NULL AND thumbnail_url IS NULL" ), {"t": thumb, "sid": row.scene_id}, ) s.commit() ok += 1 if i % 200 == 0: print(f"{i}/{len(rows)} ok={ok} brak={miss} blad={err}", flush=True) print(f"KONIEC {args.origin}: ok={ok} brak={miss} blad={err} z {len(rows)}") return 0 if __name__ == "__main__": sys.exit(main())