goon/app/connectors/direct_scrapers/fullvideosporn.py

"""fullvideosporn.com (= sextu.com pod nową marką) — browse scraper z BRAMKĄ NA OBSADĘ.

To NIE jest klon fullmovies.xxx: inny silnik (TXXX vs KVS), inna taksonomia, 0/140
pokrycia tytułów z naszym korpusem `tube:fullmoviesxxx`.

**BRAMKA (warunek konieczny)**: 65-75% katalogu NIE MA żadnego performera. Sceny bez
obsady mają dodatkowo mocno spunowane tytuły ("Fabulous Porn Movie Gothic Newest
Exclusive Version"), więc nie zmatchują canonical i weszłyby jako puste orphany.
Ingestujemy WYŁĄCZNIE sceny z ≥1 performerem — wtedy sygnał jest dobry: 88-89%
nazwisk z próbki ma u nas performera z refem tpdb/stashdb. Zawężenie zostawia
~25-35% katalogu (~150-220k scen), czyli i tak dużo.

Pozostałe pułapki, wszystkie obsłużone niżej:
  - **tytuł bierzemy z `vit:"…"` (JS playera), NIE z `og:title`/`h1`** — te bywają
    AI-owym spinem SEO ("Redhead MILF Fucks BBC in Courtroom Parody") zamiast realnego
    tytułu sceny ("Alexis Fawx In Rise Anal In The Courtroom").
  - **obsadę czytamy TYLKO z `<h3 class="item">Porn-stars: …</h3>`** — na całej stronie
    jest ~22 linków `videos.php?q=`, w sekcji obsady 1-2 realne (pułapka xxxfiles).
  - sekcje `Porn Site:` / `Porn Categories:` / `Porn-stars:` to ODDZIELNE `<h3>`, więc
    parsujemy je per-blok, inaczej kategorie wyciekają do studia i odwrotnie.
  - listing wymaga bramki cookie (429 → challenge → retry), stąd własny `crawl_page`
    zamiast domyślnego `browser_get` z bazy.

Playback: extractor `fullvideosporn` (TXXX → get_file → 302 → znvcdn, portable cross-IP).
"""
from __future__ import annotations

import html
import logging
import re
from datetime import UTC, datetime, timedelta

from app.connectors.base import (
    RawPerformer,
    RawPlaybackSource,
    RawScene,
    RawStudio,
    RawTag,
)
from app.connectors.direct_scrapers._browse_base import BaseBrowseScraper
from app.extractors.tubes.fullvideosporn import _new_session, gated_get
from app.normalize.text import slugify

log = logging.getLogger(__name__)

_BASE = "https://fullvideosporn.com"

_SCENE_URL_RE = re.compile(r'href="(/en/video/(\d+)/[a-z0-9\-_]+/)"', re.IGNORECASE)
_VIT_RE = re.compile(r'vit:\s*"([^"]+)"')
_OGTITLE_RE = re.compile(r'property="og:title" content="([^"]+)"', re.IGNORECASE)
_DUR_RE = re.compile(r'og:video:duration" content="([0-9]+)"', re.IGNORECASE)
_THUMB_RE = re.compile(r'property="og:image" content="([^"]+)"', re.IGNORECASE)
# Każda sekcja to osobny <h3 class="item">Label: <a>..</a><a>..</a></h3>
_SECTION_RE = re.compile(r'<h3 class="item">(.*?)</h3>', re.IGNORECASE | re.DOTALL)
_ANCHOR_RE = re.compile(r">([^<>]{2,60})</a>")
_SUBMITTED_RE = re.compile(
    r"Submitted:\s*<em>\s*(\d+)\s+(second|minute|hour|day|week|month|year)s?\s+ago",
    re.IGNORECASE,
)
_UNIT_DAYS = {
    "second": 0, "minute": 0, "hour": 0,
    "day": 1, "week": 7, "month": 30, "year": 365,
}


def _submitted_to_date(detail_html: str):
    """`Submitted: <em>3 days ago</em>` → data. Strona nie podaje daty wprost, a
    to jest data uploadu na tube (nie premiery studia) — traktujemy jak inne tuby."""
    m = _SUBMITTED_RE.search(detail_html)
    if not m:
        return None
    n, unit = int(m.group(1)), m.group(2).lower()
    days = n * _UNIT_DAYS.get(unit, 0)
    return (datetime.now(UTC) - timedelta(days=days)).date()


class FullVideosPornScraper(BaseBrowseScraper):
    sitetag = "fullvideosporn"

    def _listing_url(self, page: int) -> str:
        return f"{_BASE}/en/videos.php?p={page}&s=l"

    def _extract_scene_urls(self, listing_html: str) -> list[str]:
        seen: set[str] = set()
        out: list[str] = []
        for m in _SCENE_URL_RE.finditer(listing_html):
            url = _BASE + m.group(1)
            if url not in seen:
                seen.add(url)
                out.append(url)
        return out

    def crawl_page(self, page: int) -> list[RawScene] | None:
        """Własna implementacja, bo listing i detale wymagają bramki cookie (429 →
        challenge → retry) i wspólnej sesji, czego `browser_get` z bazy nie robi."""
        session = _new_session()
        r = gated_get(session, self._listing_url(page), timeout=self._timeout)
        if r is None or r.status_code != 200:
            log.warning(
                "fullvideosporn listing page=%d status=%s", page, getattr(r, "status_code", None)
            )
            return None
        urls = self._extract_scene_urls(r.text)
        if not urls:
            return []

        out: list[RawScene] = []
        gated = 0
        for scene_url in urls:
            d = gated_get(session, scene_url, timeout=self._timeout)
            if d is None or d.status_code != 200:
                continue
            try:
                raw = self._parse_detail(scene_url, d.text)
            except Exception as e:
                log.warning("fullvideosporn detail parse failed %s: %s", scene_url, e)
                continue
            if raw is None:
                gated += 1
                continue
            out.append(raw)
        if gated:
            log.info(
                "fullvideosporn page=%d: %d/%d scen pominietych (brak obsady)",
                page, gated, len(urls),
            )
        return out

    def _parse_detail(self, scene_url: str, detail_html: str) -> RawScene | None:
        # Sekcje metadanych — każda w swoim <h3 class="item">.
        performers: list[RawPerformer] = []
        tags: list[RawTag] = []
        studio: RawStudio | None = None
        seen_p: set[str] = set()
        seen_t: set[str] = set()

        for sec in _SECTION_RE.findall(detail_html):
            label = re.sub(r"<[^>]+>", " ", sec).strip().lower()
            names = [html.unescape(x).strip() for x in _ANCHOR_RE.findall(sec)]
            if label.startswith("porn-stars"):
                for name in names:
                    sl = slugify(name)
                    if sl and sl not in seen_p:
                        seen_p.add(sl)
                        performers.append(
                            RawPerformer(external_id=f"{self.sitetag}:performer:{sl}", name=name)
                        )
            elif label.startswith("porn site"):
                if names and studio is None:
                    sname = names[0]
                    studio = RawStudio(
                        external_id=f"{self.sitetag}:studio:{slugify(sname)}",
                        name=sname,
                        slug=slugify(sname),
                    )
            elif label.startswith("porn categories"):
                for name in names:
                    sl = slugify(name)
                    if sl and sl not in seen_t:
                        seen_t.add(sl)
                        tags.append(
                            RawTag(external_id=f"{self.sitetag}:tag:{sl}", name=name, slug=sl)
                        )

        # BRAMKA: bez obsady nie ingestujemy (patrz docstring — 2/3 katalogu to
        # spunowane tytuły bez performera, które weszłyby jako puste orphany).
        if not performers:
            return None

        # Tytuł z playera; og:title/h1 bywa AI-owym spinem SEO.
        tm = _VIT_RE.search(detail_html)
        title = html.unescape(tm.group(1)).strip() if tm else ""
        if not title:
            om = _OGTITLE_RE.search(detail_html)
            title = html.unescape(om.group(1)).strip() if om else ""
        if not title:
            return None

        dm = _DUR_RE.search(detail_html)
        duration_sec = int(dm.group(1)) if dm else None
        thm = _THUMB_RE.search(detail_html)
        thumbnail_url = thm.group(1) if thm else None

        # Tag == nazwisko performera (np. „Octavia Red" bywa i kategorią) → wytnij.
        tags = [t for t in tags if t.slug not in seen_p]

        return RawScene(
            external_id=f"{self.sitetag}:{scene_url}",
            title=title,
            release_date=_submitted_to_date(detail_html),
            duration_sec=duration_sec,
            url=scene_url,
            studio=studio,
            performers=performers,
            tags=tags,
            playback_sources=[
                RawPlaybackSource(
                    origin=f"tube:{self.sitetag}",
                    page_url=scene_url,
                    duration_sec=duration_sec,
                    thumbnail_url=thumbnail_url,
                )
            ],
        )