"""vjav.com — JAV browse scraper (TXXX network). Sitemap id-walk + JSON metadata API. Osobna pula JAV (tytuły azjatyckie + `title_jp`, studia typu JAVHD), origin `tube:vjav`. Świadomie orphan vertical (sekcja JAV) — NIE deduplikuje się z zachodnim katalogiem. vjav to SPA (ktk_player + blob) — listing renderuje się JS-owo, więc HTML listingu nie ma linków do scen. Zamiast tego: - **browse**: `sitemap.xml` -> `sitemap_vids_N.xml` (id-e wszystkich wideo). Newest = najwyższe id (auto-increment). Cache posortowanej listy id malejąco, page N = kolejny wycinek PER_PAGE. - **metadata**: bogaty JSON API `GET /api/json/video/1///.json` (floor = id//1000*1000) — title (+title_jp), duration, post_date, channel.title (studio), models{} (performerki), categories{}+tags{}. - **stream**: osobny ekstraktor `app/extractors/tubes/vjav.py` (videofile.php -> get_file HLS). RE 2026-07-10. Override `crawl_page()` (API/sitemap flow, wzór jak EpornerApiScraper). HTML-owe _listing_url/_extract/_parse to stuby (ABC ich wymaga, nieużywane). """ from __future__ import annotations import json import logging import re from datetime import date, datetime from app.connectors.base import ( RawPerformer, RawPlaybackSource, RawScene, RawStudio, RawTag, ) from app.config import get_settings from app.connectors.direct_scrapers._browse_base import BaseBrowseScraper from app.extractors import browser_get from app.normalize.text import slugify log = logging.getLogger(__name__) _BASE = "https://vjav.com" _PER_PAGE = 40 _SITEMAP_INDEX = f"{_BASE}/sitemap.xml" _SITEMAP_RE = re.compile(r"https://vjav\.com/sitemap_vids_\d+\.xml") _VID_RE = re.compile(r"/videos/(\d+)/") def _parse_duration(value: str | None) -> int | None: """`1:16:43` / `14:17` -> sekundy.""" if not value: return None try: parts = [int(p) for p in value.strip().split(":")] except ValueError: return None sec = 0 for p in parts: sec = sec * 60 + p return sec or None def _parse_post_date(value: str | None) -> date | None: if not value: return None try: return datetime.strptime(value.strip(), "%Y-%m-%d %H:%M:%S").date() except ValueError: return None class VjavScraper(BaseBrowseScraper): sitetag = "vjav" def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) # Cache id-ów wideo (posortowane malejąco = newest first), per instancja/run. self._sorted_ids: list[int] | None = None # vjav odciął IP VPS-a: 2026-08-03 KAŻDA ścieżka (sitemap, strona główna, # /latest-updates/) oddawała 429 przy trzech próbach z rzędu, a przez proxy # 200 od pierwszej. Źródło stało 12 dni. Bright Data jest tu więc warunkiem # działania, nie optymalizacją — bez klucza scraper się wyłącza. self._proxy = get_settings().brightdata_proxy_url if not self._proxy: log.warning("vjav: BRAK BRIGHTDATA_PROXY_URL — vjav blokuje IP VPS-a (429)") # ---- browse: sitemap id-walk ------------------------------------------------ def _load_ids(self) -> None: """`self._sorted_ids = []` znaczy „katalog pusty", a to woła `crawl_page` do zwrócenia [], czyli „koniec katalogu". Dlatego błąd pobrania NIE MOŻE tu ustawiać []: przy 429 scraper raportował 0 scen i 0 błędów, więc przez 12 dni wyglądał na zdrowy. Zostawiamy None = „nie wiadomo", co crawl_page tłumaczy na transient-fail. Status sprawdzamy jawnie, bo `browser_get` nie rzuca na 4xx — i to właśnie przez to treść błędu 429 była parsowana jako XML.""" try: res = browser_get(_SITEMAP_INDEX, timeout=self._timeout, proxy=self._proxy) if getattr(res, "status_code", 200) >= 400: log.warning("vjav: sitemap index HTTP %s — przerywam", res.status_code) self._sorted_ids = None return idx = res.text if hasattr(res, "text") else res except Exception as e: log.warning("vjav: sitemap index fetch fail: %s", e) self._sorted_ids = None return maps = list(dict.fromkeys(_SITEMAP_RE.findall(idx))) if not maps: log.warning("vjav: sitemap index bez sitemap wideo (%d B) — przerywam", len(idx)) self._sorted_ids = None return ids: set[int] = set() for sm in maps: try: res = browser_get(sm, timeout=self._timeout, proxy=self._proxy) if getattr(res, "status_code", 200) >= 400: log.info("vjav: sitemap %s HTTP %s", sm, res.status_code) continue body = res.text if hasattr(res, "text") else res except Exception as e: log.info("vjav: sitemap %s fetch fail: %s", sm, e) continue for m in _VID_RE.finditer(body): ids.add(int(m.group(1))) self._sorted_ids = sorted(ids, reverse=True) log.info("vjav: załadowano %d id z %d sitemap", len(self._sorted_ids), len(maps)) def _meta_url(self, vid: int) -> str: floor = (vid // 1000) * 1000 return f"{_BASE}/api/json/video/1/{floor}/{vid}/{vid}.json" # ---- metadata JSON -> RawScene ---------------------------------------------- def _parse_meta(self, vid: int, v: dict) -> RawScene | None: title = (v.get("title") or "").strip() slug = (v.get("dir") or "").strip() if not title or not slug: return None page_url = f"{_BASE}/videos/{vid}/{slug}/" duration_sec = _parse_duration(v.get("duration")) release_date = _parse_post_date(v.get("post_date")) thumb = (v.get("thumbsrc") or v.get("thumb") or "").strip() or None studio: RawStudio | None = None ch = v.get("channel") if isinstance(ch, dict) and (ch.get("title") or "").strip(): st = ch["title"].strip() sg = slugify(st) studio = RawStudio(external_id=f"vjav:studio:{sg}", name=st, slug=sg) performers: list[RawPerformer] = [] models = v.get("models") if isinstance(models, dict): for mid, md in models.items(): nm = (md.get("title") or "").strip() if isinstance(md, dict) else "" if nm: performers.append(RawPerformer(external_id=f"vjav:model:{mid}", name=nm)) tags: list[RawTag] = [] seen_slugs: set[str] = set() for coll in ("categories", "tags"): c = v.get(coll) if not isinstance(c, dict): continue for td in c.values(): nm = (td.get("title") or "").strip() if isinstance(td, dict) else "" if not nm: continue sg = slugify(nm) if not sg or sg in seen_slugs or len(sg) > 60: continue seen_slugs.add(sg) tags.append(RawTag(external_id=f"vjav:tag:{sg}", name=nm, slug=sg)) return RawScene( external_id=f"{self.sitetag}:{page_url}", title=title, release_date=release_date, duration_sec=duration_sec, url=page_url, studio=studio, performers=performers, tags=tags, playback_sources=[ RawPlaybackSource( origin=f"tube:{self.sitetag}", page_url=page_url, duration_sec=duration_sec, thumbnail_url=thumb, ) ], raw={"source": "vjav_api", "id": vid}, ) def crawl_page(self, page: int) -> list[RawScene] | None: if self._sorted_ids is None: self._load_ids() if not self._sorted_ids: # [] gdy sitemap pusty (exhausted); None tylko przy fetch-failu indexu. return [] if self._sorted_ids == [] else None start = (page - 1) * _PER_PAGE chunk = self._sorted_ids[start : start + _PER_PAGE] if not chunk: return [] out: list[RawScene] = [] for vid in chunk: try: # Proxy także tutaj — API metadanych siedzi na tym samym hoście, więc # obejmuje je ten sam 429 co sitemap. res = browser_get( self._meta_url(vid), timeout=self._timeout, headers={"Referer": _BASE + "/"}, proxy=self._proxy, ) body = res.text if hasattr(res, "text") else res v = json.loads(body).get("video") except Exception as e: log.info("vjav: meta fetch/parse fail id=%s: %s", vid, e) continue if not isinstance(v, dict): continue try: raw = self._parse_meta(vid, v) except Exception as e: log.warning("vjav: meta parse fail id=%s: %s", vid, e) continue if raw is not None: out.append(raw) return out # ---- HTML stuby (nieużywane, ABC wymaga) ------------------------------------ def _listing_url(self, page: int) -> str: # pragma: no cover raise NotImplementedError("VjavScraper używa crawl_page (sitemap+API)") def _extract_scene_urls(self, listing_html: str) -> list[str]: # pragma: no cover raise NotImplementedError def _parse_detail(self, scene_url: str, detail_html: str) -> RawScene | None: # pragma: no cover raise NotImplementedError