goon/app/connectors/xtapes.py

"""xtapes.la, konektor filmow (kategoria "Full Movies"). Dodany 2026-10-15.

WordPress z otwartym REST API, wiec liste bierzemy z `/wp-json/wp/v2/posts` z filtrem
kategorii, a nie z HTML-a. Ocena przed dodaniem (2026-10-15): 1846 filmow, 3 do 5 nowych
dziennie, czesto w dniu premiery; z 30 najnowszych 10 nie mielismy wcale, reszte z
yourdailypornmovies/paradisehill/mangoporn (xtapes dokleja im kolejny link).

Metadane z REST (`_embed=wp:term`):
- tytul "TYTUL (RRRR) - Full Movie" -> tytul i rok,
- studio to kategoria w nawiasie kwadratowym ("[EvilAngel]", "[Brazzers Movies]"),
- pozostale kategorie to gatunki (bez "Full Movies"),
- obsada to tagi (`post_tag`),
- opis z `content` (czesto pusty).
Ze strony filmu: odtwarzacze (iframe), dlugosc (`itemprop="duration"`), okladka (og:image).

Odtwarzacze: sextapes.to (P.A.C.K.E.R. + JWPlayer, HLS typu StreamWish) i maxcloud.fit.
maxcloud to posrednik (lista serwerow w base64, za nia kolejny serwis z blokada devtools),
wiec go pomijamy. Link sextapes jest zwiazany z IP rozwiazujacego (z VPS 403 z innego IP),
dlatego rozwiazuje go telefon (packerHoster.ts), a backend go nie ekstrahuje (playback.py).

Dostep: VPS dostaje od Cloudflare 403 na xtapes.la (REST i strony), przez Bright Data 200.
Proxy tylko do HTML/JSON, nie do wideo. sextapes.to z VPS odpowiada bez proxy.
"""
from __future__ import annotations

import html as html_mod
import json
import logging
import re
from collections.abc import Iterator
from datetime import datetime
from urllib.parse import urlparse

from app.config import get_settings
from app.connectors.base import (
    BaseMovieConnector,
    RawMovie,
    RawPerformer,
    RawPlaybackSource,
    RawStudio,
    RawTag,
)
from app.extractors import browser_get
from app.models.source import SourceKind
from app.normalize.text import slugify

log = logging.getLogger(__name__)

_BASE = "https://xtapes.la"
#: Id kategorii "Full Movies" (sprawdzone 2026-10-15; slugi kategorii sa zaciemnione liczbami).
_MOVIES_CATEGORY = 1766
_PER_PAGE = 30

_TITLE_RE = re.compile(
    r"^(?P<title>.+?)\s*\((?P<year>(?:19|20)\d{2})\)\s*(?:[-\u2013\u2014]\s*Full\s+Movie)?\s*$",
    re.IGNORECASE,
)
_FULL_MOVIE_TAIL_RE = re.compile(r"\s*[-\u2013\u2014]\s*Full\s+Movie\s*$", re.IGNORECASE)
_STUDIO_CAT_RE = re.compile(r"^\[(?P<name>.+)\]$")
_IFRAME_RE = re.compile(r"""<iframe[^>]+src=["']((?:https?:)?//[^"']+)["']""", re.IGNORECASE)
_DURATION_RE = re.compile(r'itemprop="duration"\s+content="P(?:\d+D)?T?(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?"')
_OG_IMAGE_RE = re.compile(r"""property=["']og:image["']\s+content=["']([^"']+)""", re.IGNORECASE)
_TAGS_RE = re.compile(r"<[^>]+>")

#: Odtwarzacze, ktorych nie zapisujemy: posrednik bez wlasnego wideo i reklamy.
_SKIP_HOSTS = ("maxcloud", "realsrv", "magsrv", "exosrv", "juicyads", "google", "doubleclick")


def _host_label(url: str) -> str | None:
    host = (urlparse(url).hostname or "").lower().removeprefix("www.")
    if not host:
        return None
    parts = host.split(".")
    return parts[-2] if len(parts) >= 2 else parts[0]


def parse_title(raw: str) -> tuple[str, int | None]:
    """'Tushy Raw V87 (2026) - Full Movie' -> ('Tushy Raw V87', 2026)."""
    t = html_mod.unescape(raw).strip()
    m = _TITLE_RE.match(t)
    if m:
        return m.group("title").strip(), int(m.group("year"))
    return _FULL_MOVIE_TAIL_RE.sub("", t).strip(), None


def parse_duration(page: str) -> int | None:
    m = _DURATION_RE.search(page)
    if not m or not any(m.groups()):
        return None
    h, mi, s = (int(x) if x else 0 for x in m.groups())
    total = h * 3600 + mi * 60 + s
    return total or None


class XtapesConnector(BaseMovieConnector):
    kind = SourceKind.scraper
    name = "xtapes"
    base_url = _BASE

    _MAX_PAGES_DELTA = 3
    _MAX_PAGES_FULL = 20

    def __init__(self, *, timeout: float = 30.0) -> None:
        self._timeout = timeout
        self._proxy = get_settings().brightdata_proxy_url

    def close(self) -> None:
        pass

    def _get(self, url: str):
        """xtapes.la od razu przez proxy (Cloudflare daje VPS-owi 403), bez proxy tylko gdy
        go nie skonfigurowano (lokalnie z domowego IP strona odpowiada wprost)."""
        r = browser_get(url, timeout=self._timeout, follow_redirects=True, proxy=self._proxy)
        if r.status_code >= 400:
            raise RuntimeError(f"{r.status_code} for {url}")
        return r

    def fetch_movies(
        self, *, since: datetime | None = None, limit: int | None = None
    ) -> Iterator[RawMovie]:
        seen = 0
        max_pages = self._MAX_PAGES_DELTA if since is not None else self._MAX_PAGES_FULL
        for page in range(1, max_pages + 1):
            url = (
                f"{_BASE}/wp-json/wp/v2/posts?categories={_MOVIES_CATEGORY}"
                f"&per_page={_PER_PAGE}&page={page}&_embed=wp:term"
                "&_fields=id,date,slug,link,title,content,_links,_embedded"
            )
            if since is not None:
                # Tylko nowsze od ostatniego udanego przebiegu: strona filmu idzie przez proxy
                # (~9 s), a 3 pelne strony REST to 90 filmow, ponad limit 6 min na konektor.
                url += f"&after={since.strftime('%Y-%m-%dT%H:%M:%S')}"
            try:
                posts = json.loads(self._get(url).text)
            except Exception as e:
                # Za ostatnia strona WP oddaje 400 (rest_post_invalid_page_number).
                log.info("%s REST page=%d: %s", self.name, page, e)
                return
            if not isinstance(posts, list) or not posts:
                return
            for post in posts:
                try:
                    movie = self._movie_from_post(post)
                except Exception as e:
                    log.warning("%s post %s failed: %s", self.name, post.get("link"), e)
                    continue
                if movie is None:
                    continue
                yield movie
                seen += 1
                if limit is not None and seen >= limit:
                    return

    def _movie_from_post(self, post: dict) -> RawMovie | None:
        link = (post.get("link") or "").strip()
        title, year = parse_title((post.get("title") or {}).get("rendered", ""))
        if not link or not title:
            return None

        studio: RawStudio | None = None
        tags: list[RawTag] = []
        performers: list[RawPerformer] = []
        for group in (post.get("_embedded") or {}).get("wp:term") or []:
            for term in group or []:
                name = html_mod.unescape(term.get("name") or "").strip()
                tax = term.get("taxonomy")
                if not name:
                    continue
                if tax == "category":
                    sm = _STUDIO_CAT_RE.match(name)
                    if sm:
                        sname = sm.group("name").strip()
                        if studio is None and slugify(sname):
                            studio = RawStudio(
                                external_id=f"{self.name}:studio:{slugify(sname)}",
                                name=sname,
                                slug=slugify(sname),
                            )
                    elif name.lower() != "full movies" and slugify(name):
                        tags.append(
                            RawTag(external_id=f"{self.name}:tag:{slugify(name)}", name=name, slug=slugify(name))
                        )
                elif tax == "post_tag" and slugify(name):
                    performers.append(
                        RawPerformer(external_id=f"{self.name}:performer:{slugify(name)}", name=name)
                    )

        page = self._get(link).text

        # Playback najpierw: bez odtwarzacza film jest bezuzyteczny, nie tworzymy sieroty.
        playback: list[RawPlaybackSource] = []
        seen_host: set[str] = set()
        for m in _IFRAME_RE.finditer(page):
            embed = m.group(1)
            if embed.startswith("//"):
                embed = "https:" + embed
            host = _host_label(embed)
            if not host or host in seen_host or any(s in embed for s in _SKIP_HOSTS):
                continue
            seen_host.add(host)
            playback.append(
                RawPlaybackSource(origin=f"{self.name}:{host}", page_url=embed, embed_url=embed)
            )
        if not playback:
            return None

        desc = _TAGS_RE.sub(" ", (post.get("content") or {}).get("rendered") or "")
        desc = re.sub(r"\s+", " ", html_mod.unescape(desc)).strip() or None
        om = _OG_IMAGE_RE.search(page)
        poster = html_mod.unescape(om.group(1)).strip() if om else None

        return RawMovie(
            external_id=post.get("slug") or urlparse(link).path.strip("/").split("/")[-1],
            title=title,
            description=desc,
            release_year=year,
            duration_sec=parse_duration(page),
            url=link,
            poster_url=poster,
            studio=studio,
            performers=performers,
            tags=tags,
            playback_sources=playback,
            raw={"source": self.name, "url": link, "wp_id": post.get("id")},
        )