goon/app/connectors/direct_scrapers/vjav.py

"""vjav.com — JAV browse scraper (TXXX network). Sitemap id-walk + JSON metadata API.

Osobna pula JAV (tytuły azjatyckie + `title_jp`, studia typu JAVHD), origin `tube:vjav`.
Świadomie orphan vertical (sekcja JAV) — NIE deduplikuje się z zachodnim katalogiem.

vjav to SPA (ktk_player + blob) — listing renderuje się JS-owo, więc HTML listingu
nie ma linków do scen. Zamiast tego:
  - **browse**: `sitemap.xml` -> `sitemap_vids_N.xml` (id-e wszystkich wideo). Newest
    = najwyższe id (auto-increment). Cache posortowanej listy id malejąco, page N =
    kolejny wycinek PER_PAGE.
  - **metadata**: bogaty JSON API `GET /api/json/video/1/<floor>/<id>/<id>.json`
    (floor = id//1000*1000) — title (+title_jp), duration, post_date, channel.title
    (studio), models{} (performerki), categories{}+tags{}.
  - **stream**: osobny ekstraktor `app/extractors/tubes/vjav.py` (videofile.php ->
    get_file HLS). RE 2026-07-10.

Override `crawl_page()` (API/sitemap flow, wzór jak EpornerApiScraper). HTML-owe
_listing_url/_extract/_parse to stuby (ABC ich wymaga, nieużywane).
"""
from __future__ import annotations

import json
import logging
import re
from datetime import date, datetime

from app.connectors.base import (
    RawPerformer,
    RawPlaybackSource,
    RawScene,
    RawStudio,
    RawTag,
)
from app.config import get_settings
from app.connectors.direct_scrapers._browse_base import BaseBrowseScraper
from app.extractors import browser_get
from app.normalize.text import slugify

log = logging.getLogger(__name__)

_BASE = "https://vjav.com"
_PER_PAGE = 40
_SITEMAP_INDEX = f"{_BASE}/sitemap.xml"
_SITEMAP_RE = re.compile(r"https://vjav\.com/sitemap_vids_\d+\.xml")
_VID_RE = re.compile(r"/videos/(\d+)/")


def _parse_duration(value: str | None) -> int | None:
    """`1:16:43` / `14:17` -> sekundy."""
    if not value:
        return None
    try:
        parts = [int(p) for p in value.strip().split(":")]
    except ValueError:
        return None
    sec = 0
    for p in parts:
        sec = sec * 60 + p
    return sec or None


def _parse_post_date(value: str | None) -> date | None:
    if not value:
        return None
    try:
        return datetime.strptime(value.strip(), "%Y-%m-%d %H:%M:%S").date()
    except ValueError:
        return None


class VjavScraper(BaseBrowseScraper):
    sitetag = "vjav"

    def __init__(self, *args, **kwargs) -> None:
        super().__init__(*args, **kwargs)
        # Cache id-ów wideo (posortowane malejąco = newest first), per instancja/run.
        self._sorted_ids: list[int] | None = None
        # vjav odciął IP VPS-a: 2026-08-03 KAŻDA ścieżka (sitemap, strona główna,
        # /latest-updates/) oddawała 429 przy trzech próbach z rzędu, a przez proxy
        # 200 od pierwszej. Źródło stało 12 dni. Bright Data jest tu więc warunkiem
        # działania, nie optymalizacją — bez klucza scraper się wyłącza.
        self._proxy = get_settings().brightdata_proxy_url
        if not self._proxy:
            log.warning("vjav: BRAK BRIGHTDATA_PROXY_URL — vjav blokuje IP VPS-a (429)")

    # ---- browse: sitemap id-walk ------------------------------------------------
    def _load_ids(self) -> None:
        """`self._sorted_ids = []` znaczy „katalog pusty", a to woła `crawl_page` do
        zwrócenia [], czyli „koniec katalogu". Dlatego błąd pobrania NIE MOŻE tu
        ustawiać []: przy 429 scraper raportował 0 scen i 0 błędów, więc przez 12 dni
        wyglądał na zdrowy. Zostawiamy None = „nie wiadomo", co crawl_page tłumaczy na
        transient-fail. Status sprawdzamy jawnie, bo `browser_get` nie rzuca na 4xx —
        i to właśnie przez to treść błędu 429 była parsowana jako XML."""
        try:
            res = browser_get(_SITEMAP_INDEX, timeout=self._timeout, proxy=self._proxy)
            if getattr(res, "status_code", 200) >= 400:
                log.warning("vjav: sitemap index HTTP %s — przerywam", res.status_code)
                self._sorted_ids = None
                return
            idx = res.text if hasattr(res, "text") else res
        except Exception as e:
            log.warning("vjav: sitemap index fetch fail: %s", e)
            self._sorted_ids = None
            return
        maps = list(dict.fromkeys(_SITEMAP_RE.findall(idx)))
        if not maps:
            log.warning("vjav: sitemap index bez sitemap wideo (%d B) — przerywam", len(idx))
            self._sorted_ids = None
            return
        ids: set[int] = set()
        for sm in maps:
            try:
                res = browser_get(sm, timeout=self._timeout, proxy=self._proxy)
                if getattr(res, "status_code", 200) >= 400:
                    log.info("vjav: sitemap %s HTTP %s", sm, res.status_code)
                    continue
                body = res.text if hasattr(res, "text") else res
            except Exception as e:
                log.info("vjav: sitemap %s fetch fail: %s", sm, e)
                continue
            for m in _VID_RE.finditer(body):
                ids.add(int(m.group(1)))
        self._sorted_ids = sorted(ids, reverse=True)
        log.info("vjav: załadowano %d id z %d sitemap", len(self._sorted_ids), len(maps))

    def _meta_url(self, vid: int) -> str:
        floor = (vid // 1000) * 1000
        return f"{_BASE}/api/json/video/1/{floor}/{vid}/{vid}.json"

    # ---- metadata JSON -> RawScene ----------------------------------------------
    def _parse_meta(self, vid: int, v: dict) -> RawScene | None:
        title = (v.get("title") or "").strip()
        slug = (v.get("dir") or "").strip()
        if not title or not slug:
            return None
        page_url = f"{_BASE}/videos/{vid}/{slug}/"
        duration_sec = _parse_duration(v.get("duration"))
        release_date = _parse_post_date(v.get("post_date"))
        thumb = (v.get("thumbsrc") or v.get("thumb") or "").strip() or None

        studio: RawStudio | None = None
        ch = v.get("channel")
        if isinstance(ch, dict) and (ch.get("title") or "").strip():
            st = ch["title"].strip()
            sg = slugify(st)
            studio = RawStudio(external_id=f"vjav:studio:{sg}", name=st, slug=sg)

        performers: list[RawPerformer] = []
        models = v.get("models")
        if isinstance(models, dict):
            for mid, md in models.items():
                nm = (md.get("title") or "").strip() if isinstance(md, dict) else ""
                if nm:
                    performers.append(RawPerformer(external_id=f"vjav:model:{mid}", name=nm))

        tags: list[RawTag] = []
        seen_slugs: set[str] = set()
        for coll in ("categories", "tags"):
            c = v.get(coll)
            if not isinstance(c, dict):
                continue
            for td in c.values():
                nm = (td.get("title") or "").strip() if isinstance(td, dict) else ""
                if not nm:
                    continue
                sg = slugify(nm)
                if not sg or sg in seen_slugs or len(sg) > 60:
                    continue
                seen_slugs.add(sg)
                tags.append(RawTag(external_id=f"vjav:tag:{sg}", name=nm, slug=sg))

        return RawScene(
            external_id=f"{self.sitetag}:{page_url}",
            title=title,
            release_date=release_date,
            duration_sec=duration_sec,
            url=page_url,
            studio=studio,
            performers=performers,
            tags=tags,
            playback_sources=[
                RawPlaybackSource(
                    origin=f"tube:{self.sitetag}",
                    page_url=page_url,
                    duration_sec=duration_sec,
                    thumbnail_url=thumb,
                )
            ],
            raw={"source": "vjav_api", "id": vid},
        )

    def crawl_page(self, page: int) -> list[RawScene] | None:
        if self._sorted_ids is None:
            self._load_ids()
        if not self._sorted_ids:
            # [] gdy sitemap pusty (exhausted); None tylko przy fetch-failu indexu.
            return [] if self._sorted_ids == [] else None
        start = (page - 1) * _PER_PAGE
        chunk = self._sorted_ids[start : start + _PER_PAGE]
        if not chunk:
            return []
        out: list[RawScene] = []
        for vid in chunk:
            try:
                # Proxy także tutaj — API metadanych siedzi na tym samym hoście, więc
                # obejmuje je ten sam 429 co sitemap.
                res = browser_get(
                    self._meta_url(vid),
                    timeout=self._timeout,
                    headers={"Referer": _BASE + "/"},
                    proxy=self._proxy,
                )
                body = res.text if hasattr(res, "text") else res
                v = json.loads(body).get("video")
            except Exception as e:
                log.info("vjav: meta fetch/parse fail id=%s: %s", vid, e)
                continue
            if not isinstance(v, dict):
                continue
            try:
                raw = self._parse_meta(vid, v)
            except Exception as e:
                log.warning("vjav: meta parse fail id=%s: %s", vid, e)
                continue
            if raw is not None:
                out.append(raw)
        return out

    # ---- HTML stuby (nieużywane, ABC wymaga) ------------------------------------
    def _listing_url(self, page: int) -> str:  # pragma: no cover
        raise NotImplementedError("VjavScraper używa crawl_page (sitemap+API)")

    def _extract_scene_urls(self, listing_html: str) -> list[str]:  # pragma: no cover
        raise NotImplementedError

    def _parse_detail(self, scene_url: str, detail_html: str) -> RawScene | None:  # pragma: no cover
        raise NotImplementedError