goon/scripts/porntrex_duration_walk.py

"""porntrex: dolicza długość scenom bez niej z listingu `/latest-updates/N/`.

Strona szczegółów porntrex nie podaje własnej długości (tylko długości filmów z paska
obok), więc backfill ze strony (`duration_backfill.py`) tu nie działa. Kafelek listingu
ją pokazuje (`class="durations"`), 85 kafelków na stronę, katalog ma ok. 8300 stron
(strona 15000 daje 404). Przebieg idzie po listingu od najnowszych i dla każdego kafelka
ustawia `scenes.duration_sec`, jeśli jest NULL. Nic nie nadpisuje.

Adresy: ścieżka search zapisuje `https://www.porntrex.com/video/<id>/<slug>`, browse
`https://porntrex.com/video/<id>/<slug>` (bez www), dlatego dopasowanie po obu wariantach.

Uruchomienie (długi przebieg, nie przez `exec`, patrz pamięć o restartach workera):
    docker compose run -d --rm worker python3 scripts/porntrex_duration_walk.py --start 1
    docker compose run --rm worker python3 scripts/porntrex_duration_walk.py --start 1 --end 20 --dry-run
Postęp w `--state-file` (domyślnie /tmp/porntrex_duration_walk.json), wznowienie od
zapisanej strony przy braku `--start`.
"""
from __future__ import annotations

import argparse
import json
import logging
import sys
import time

from sqlalchemy import text

from app.connectors.direct_scrapers.porntrex_browse import _A_RE, _DUR_RE, _parse_duration
from app.db import session_scope
from app.extractors import browser_get

log = logging.getLogger("porntrex_duration_walk")
_LISTING = "https://www.porntrex.com/latest-updates/{page}/"
_ORIGIN = "tube:porntrexcom"


def parse_cards(html: str) -> dict[str, int]:
    """Kafelki listingu → {url bez www i bez końcowego '/': sekundy}."""
    out: dict[str, int] = {}
    anchors = list(_A_RE.finditer(html))
    for idx, m in enumerate(anchors):
        url = m.group("url").replace("://www.", "://").rstrip("/")
        if url in out:
            continue
        end = anchors[idx + 1].start() if idx + 1 < len(anchors) else m.end() + 700
        dm = _DUR_RE.search(html[m.start():end])
        sec = _parse_duration(dm.group(1) if dm else None)
        if sec:
            out[url] = sec
    return out


def variants(url: str) -> list[str]:
    bare = url.replace("://www.", "://").rstrip("/")
    www = bare.replace("https://", "https://www.", 1)
    return [bare, bare + "/", www, www + "/"]


def apply(cards: dict[str, int], *, dry_run: bool) -> dict[str, int]:
    """Zwraca liczniki: matched (źródła w bazie), filled (ustawione), had (już miały)."""
    counters = {"matched": 0, "filled": 0, "had": 0}
    if not cards:
        return counters
    lookup: dict[str, int] = {}
    for url, sec in cards.items():
        for v in variants(url):
            lookup[v] = sec
    with session_scope() as s:
        rows = s.execute(
            text(
                """
                SELECT ps.page_url, ps.id AS ps_id, sc.id AS scene_id, sc.duration_sec
                  FROM playback_sources ps
                  JOIN scenes sc ON sc.id = ps.scene_id
                 WHERE ps.origin = :origin AND ps.page_url = ANY(:urls)
                """
            ),
            {"origin": _ORIGIN, "urls": list(lookup)},
        ).fetchall()
        for page_url, ps_id, scene_id, have in rows:
            counters["matched"] += 1
            sec = lookup.get(page_url)
            if not sec:
                continue
            if have:
                counters["had"] += 1
                continue
            counters["filled"] += 1
            if dry_run:
                continue
            s.execute(
                text("UPDATE scenes SET duration_sec = :d WHERE id = :id AND duration_sec IS NULL"),
                {"d": sec, "id": scene_id},
            )
            s.execute(
                text(
                    "UPDATE playback_sources SET duration_sec = :d "
                    " WHERE id = :id AND duration_sec IS NULL"
                ),
                {"d": sec, "id": ps_id},
            )
    return counters


def _fetch(page: int) -> tuple[int, str]:
    res = browser_get(_LISTING.format(page=page), timeout=45)
    status = int(getattr(res, "status_code", 200) or 200)
    html = res.text if hasattr(res, "text") else str(res)
    return status, html


def main() -> int:
    ap = argparse.ArgumentParser()
    ap.add_argument("--start", type=int, default=None, help="pierwsza strona (domyślnie ze state-file albo 1)")
    ap.add_argument("--end", type=int, default=20000, help="ostatnia strona (przebieg i tak staje na 404)")
    ap.add_argument("--pace", type=float, default=1.0, help="odstęp między stronami w sekundach")
    ap.add_argument("--dry-run", action="store_true")
    ap.add_argument("--state-file", default="/tmp/porntrex_duration_walk.json")
    args = ap.parse_args()
    logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")

    start = args.start
    if start is None:
        try:
            with open(args.state_file, encoding="utf-8") as f:
                start = int(json.load(f).get("next_page", 1))
        except (OSError, ValueError):
            start = 1

    total = {"pages": 0, "cards": 0, "matched": 0, "filled": 0, "had": 0}
    empty_streak = 0
    t0 = time.time()
    for page in range(start, args.end + 1):
        try:
            status, html = _fetch(page)
        except Exception as e:
            log.warning("strona %d: błąd pobrania %s, czekam 30 s", page, e)
            time.sleep(30)
            try:
                status, html = _fetch(page)
            except Exception as e2:
                log.error("strona %d: drugi błąd %s, przerywam", page, e2)
                break
        cards = parse_cards(html) if status < 400 else {}
        if not cards:
            empty_streak += 1
            log.info("strona %d: http %s, brak kafelków (%d z rzędu)", page, status, empty_streak)
            if empty_streak >= 3:
                log.info("koniec listingu na stronie %d", page)
                break
            time.sleep(args.pace)
            continue
        empty_streak = 0
        c = apply(cards, dry_run=args.dry_run)
        total["pages"] += 1
        total["cards"] += len(cards)
        for k in ("matched", "filled", "had"):
            total[k] += c[k]
        log.info(
            "strona %d: kafelków %d, w bazie %d, doliczone %d, miały %d%s",
            page, len(cards), c["matched"], c["filled"], c["had"], " (dry-run)" if args.dry_run else "",
        )
        if not args.dry_run:
            try:
                with open(args.state_file, "w", encoding="utf-8") as f:
                    json.dump({"next_page": page + 1, "total": total, "ts": time.time()}, f)
            except OSError:
                pass
        time.sleep(args.pace)
    log.info("razem: %s w %.0f s", total, time.time() - t0)
    print(json.dumps(total))
    return 0


if __name__ == "__main__":
    sys.exit(main())