feat(jav): vjav.com scraper + extractor (TXXX-network JAV source)

Second JAV vertical source (origin tube:vjav, gated to JAV tab via JAV_ORIGINS).

Browse: SPA has no server-rendered listing, so id-walk over sitemap_vids
(newest = highest video id) feeding the rich JSON metadata API
(/api/json/video/1/<floor>/<id>/<id>.json): title, duration, post_date,
channel->studio, models->performers, categories+tags.

Stream: videofile.php returns video_url in two obfuscation layers, Cyrillic
homoglyphs (M/C/A/E) over a custom base64 alphabet (comma->slash, tilde->pad,
dash->plus). Decoded get_file is absolute (old shared-txxx videos) or relative
(new vjav-infra, prepend host); adding f=video.m3u8 yields a portable, time-bound
ahcdn HLS (referer=none whitelisted, not IP-bound). Returned as m3u8 +
mobile_direct_ok so playback routes it through /proxy/hls: manifest passthrough,
segments direct from the phone (verified 206 cross-IP, 0 VPS video bandwidth).

Backend-only, no mobile change (JAV tab + jav param already shipped).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

app/connectors/direct_scrapers/__init__.py

@@ -136,6 +136,7 @@ from app.connectors.direct_scrapers.fullmovies import FullmoviesScraper  # noqa:
 from app.connectors.direct_scrapers.hdporngg import HDPornGGScraper  # noqa: E402
 from app.connectors.direct_scrapers.hqfap import HQFapScraper  # noqa: E402
 from app.connectors.direct_scrapers.javflix import JavflixScraper  # noqa: E402
+from app.connectors.direct_scrapers.vjav import VjavScraper  # noqa: E402
 from app.connectors.direct_scrapers.neporn import NepornScraper  # noqa: E402
 from app.connectors.direct_scrapers.superporn import SuperpornScraper  # noqa: E402
 from app.connectors.direct_scrapers.eporner_api import EpornerApiScraper  # noqa: E402
@@ -222,6 +223,10 @@ ALL_BROWSE_SCRAPERS: list[type[BaseBrowseScraper]] = [
     # JavflixScraper — JAV vertical (osobna sekcja). origin tube:javflix jest w
     # JAV_ORIGINS → list_scenes wyklucza je z głównego feedu (tylko zakładka JAV).
     JavflixScraper,
+    # VjavScraper — JAV vertical (osobna sekcja, origin tube:vjav w JAV_ORIGINS).
+    # TXXX network: sitemap id-walk (newest=max id) + JSON metadata API. Stream
+    # videofile.php → get_file HLS (portable, /proxy/hls passthrough). RE 2026-07-10.
+    VjavScraper,
     # NepornScraper — dołączony 2026-06-10 (user request). KVS engine (jak freshporno/
     # porn00), /latest-updates/N/. JSON-LD (title+desc+uploadDate+thumb) + video:duration
     # meta + /models/ performerzy + /categories/ tagi. Brak studio (tytuł bywa

app/connectors/direct_scrapers/vjav.py

@@ -0,0 +1,205 @@
+"""vjav.com — JAV browse scraper (TXXX network). Sitemap id-walk + JSON metadata API.
+
+Osobna pula JAV (tytuły azjatyckie + `title_jp`, studia typu JAVHD), origin `tube:vjav`.
+Świadomie orphan vertical (sekcja JAV) — NIE deduplikuje się z zachodnim katalogiem.
+
+vjav to SPA (ktk_player + blob) — listing renderuje się JS-owo, więc HTML listingu
+nie ma linków do scen. Zamiast tego:
+  - **browse**: `sitemap.xml` -> `sitemap_vids_N.xml` (id-e wszystkich wideo). Newest
+    = najwyższe id (auto-increment). Cache posortowanej listy id malejąco, page N =
+    kolejny wycinek PER_PAGE.
+  - **metadata**: bogaty JSON API `GET /api/json/video/1/<floor>/<id>/<id>.json`
+    (floor = id//1000*1000) — title (+title_jp), duration, post_date, channel.title
+    (studio), models{} (performerki), categories{}+tags{}.
+  - **stream**: osobny ekstraktor `app/extractors/tubes/vjav.py` (videofile.php ->
+    get_file HLS). RE 2026-07-10.
+
+Override `crawl_page()` (API/sitemap flow, wzór jak EpornerApiScraper). HTML-owe
+_listing_url/_extract/_parse to stuby (ABC ich wymaga, nieużywane).
+"""
+from __future__ import annotations
+
+import json
+import logging
+import re
+from datetime import date, datetime
+
+from app.connectors.base import (
+    RawPerformer,
+    RawPlaybackSource,
+    RawScene,
+    RawStudio,
+    RawTag,
+)
+from app.connectors.direct_scrapers._browse_base import BaseBrowseScraper
+from app.extractors import browser_get
+from app.normalize.text import slugify
+
+log = logging.getLogger(__name__)
+
+_BASE = "https://vjav.com"
+_PER_PAGE = 40
+_SITEMAP_INDEX = f"{_BASE}/sitemap.xml"
+_SITEMAP_RE = re.compile(r"https://vjav\.com/sitemap_vids_\d+\.xml")
+_VID_RE = re.compile(r"/videos/(\d+)/")
+
+
+def _parse_duration(value: str | None) -> int | None:
+    """`1:16:43` / `14:17` -> sekundy."""
+    if not value:
+        return None
+    try:
+        parts = [int(p) for p in value.strip().split(":")]
+    except ValueError:
+        return None
+    sec = 0
+    for p in parts:
+        sec = sec * 60 + p
+    return sec or None
+
+
+def _parse_post_date(value: str | None) -> date | None:
+    if not value:
+        return None
+    try:
+        return datetime.strptime(value.strip(), "%Y-%m-%d %H:%M:%S").date()
+    except ValueError:
+        return None
+
+
+class VjavScraper(BaseBrowseScraper):
+    sitetag = "vjav"
+
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        # Cache id-ów wideo (posortowane malejąco = newest first), per instancja/run.
+        self._sorted_ids: list[int] | None = None
+
+    # ---- browse: sitemap id-walk ------------------------------------------------
+    def _load_ids(self) -> None:
+        try:
+            res = browser_get(_SITEMAP_INDEX, timeout=self._timeout)
+            idx = res.text if hasattr(res, "text") else res
+        except Exception as e:
+            log.warning("vjav: sitemap index fetch fail: %s", e)
+            self._sorted_ids = []
+            return
+        maps = list(dict.fromkeys(_SITEMAP_RE.findall(idx)))
+        ids: set[int] = set()
+        for sm in maps:
+            try:
+                res = browser_get(sm, timeout=self._timeout)
+                body = res.text if hasattr(res, "text") else res
+            except Exception as e:
+                log.info("vjav: sitemap %s fetch fail: %s", sm, e)
+                continue
+            for m in _VID_RE.finditer(body):
+                ids.add(int(m.group(1)))
+        self._sorted_ids = sorted(ids, reverse=True)
+        log.info("vjav: załadowano %d id z %d sitemap", len(self._sorted_ids), len(maps))
+
+    def _meta_url(self, vid: int) -> str:
+        floor = (vid // 1000) * 1000
+        return f"{_BASE}/api/json/video/1/{floor}/{vid}/{vid}.json"
+
+    # ---- metadata JSON -> RawScene ----------------------------------------------
+    def _parse_meta(self, vid: int, v: dict) -> RawScene | None:
+        title = (v.get("title") or "").strip()
+        slug = (v.get("dir") or "").strip()
+        if not title or not slug:
+            return None
+        page_url = f"{_BASE}/videos/{vid}/{slug}/"
+        duration_sec = _parse_duration(v.get("duration"))
+        release_date = _parse_post_date(v.get("post_date"))
+        thumb = (v.get("thumbsrc") or v.get("thumb") or "").strip() or None
+
+        studio: RawStudio | None = None
+        ch = v.get("channel")
+        if isinstance(ch, dict) and (ch.get("title") or "").strip():
+            st = ch["title"].strip()
+            sg = slugify(st)
+            studio = RawStudio(external_id=f"vjav:studio:{sg}", name=st, slug=sg)
+
+        performers: list[RawPerformer] = []
+        models = v.get("models")
+        if isinstance(models, dict):
+            for mid, md in models.items():
+                nm = (md.get("title") or "").strip() if isinstance(md, dict) else ""
+                if nm:
+                    performers.append(RawPerformer(external_id=f"vjav:model:{mid}", name=nm))
+
+        tags: list[RawTag] = []
+        seen_slugs: set[str] = set()
+        for coll in ("categories", "tags"):
+            c = v.get(coll)
+            if not isinstance(c, dict):
+                continue
+            for td in c.values():
+                nm = (td.get("title") or "").strip() if isinstance(td, dict) else ""
+                if not nm:
+                    continue
+                sg = slugify(nm)
+                if not sg or sg in seen_slugs or len(sg) > 60:
+                    continue
+                seen_slugs.add(sg)
+                tags.append(RawTag(external_id=f"vjav:tag:{sg}", name=nm, slug=sg))
+
+        return RawScene(
+            external_id=f"{self.sitetag}:{page_url}",
+            title=title,
+            release_date=release_date,
+            duration_sec=duration_sec,
+            url=page_url,
+            studio=studio,
+            performers=performers,
+            tags=tags,
+            playback_sources=[
+                RawPlaybackSource(
+                    origin=f"tube:{self.sitetag}",
+                    page_url=page_url,
+                    duration_sec=duration_sec,
+                    thumbnail_url=thumb,
+                )
+            ],
+            raw={"source": "vjav_api", "id": vid},
+        )
+
+    def crawl_page(self, page: int) -> list[RawScene] | None:
+        if self._sorted_ids is None:
+            self._load_ids()
+        if not self._sorted_ids:
+            # [] gdy sitemap pusty (exhausted); None tylko przy fetch-failu indexu.
+            return [] if self._sorted_ids == [] else None
+        start = (page - 1) * _PER_PAGE
+        chunk = self._sorted_ids[start : start + _PER_PAGE]
+        if not chunk:
+            return []
+        out: list[RawScene] = []
+        for vid in chunk:
+            try:
+                res = browser_get(self._meta_url(vid), timeout=self._timeout, headers={"Referer": _BASE + "/"})
+                body = res.text if hasattr(res, "text") else res
+                v = json.loads(body).get("video")
+            except Exception as e:
+                log.info("vjav: meta fetch/parse fail id=%s: %s", vid, e)
+                continue
+            if not isinstance(v, dict):
+                continue
+            try:
+                raw = self._parse_meta(vid, v)
+            except Exception as e:
+                log.warning("vjav: meta parse fail id=%s: %s", vid, e)
+                continue
+            if raw is not None:
+                out.append(raw)
+        return out
+
+    # ---- HTML stuby (nieużywane, ABC wymaga) ------------------------------------
+    def _listing_url(self, page: int) -> str:  # pragma: no cover
+        raise NotImplementedError("VjavScraper używa crawl_page (sitemap+API)")
+
+    def _extract_scene_urls(self, listing_html: str) -> list[str]:  # pragma: no cover
+        raise NotImplementedError
+
+    def _parse_detail(self, scene_url: str, detail_html: str) -> RawScene | None:  # pragma: no cover
+        raise NotImplementedError

app/extractors/__init__.py

@@ -40,6 +40,7 @@ from app.extractors.tubes import (
     porn00,
     porntrex,
     sxyprn,
+    vjav,
     watchporn,
     xhamster,
     yespornvip,
@@ -176,6 +177,8 @@ _REGISTRY: dict[str, Callable[[str], list[StreamSource] | None]] = {
     # patternem → type='hoster', telefon resolwuje (voe backend, dood/filemoon phone-side).
     # Wrapper javflix.extract odsiewa placeholder players.mp4.
     "javflix": javflix.extract,
+    # vjav (JAV, TXXX network) — videofile.php -> get_file HLS (patrz tubes/vjav.py).
+    "vjav": vjav.extract,
     # neporn — KVS function/0 + license (jak freshporno). Server-side _kvs resolve →
     # data001.neporn.com/remote_control.php portable (cross-IP 206, 2026-06-10).
     "neporncom": neporn.extract,

app/extractors/tubes/vjav.py

@@ -0,0 +1,134 @@
+"""vjav.com — TXXX-network JAV tube. HLS stream extractor.
+
+vjav to SPA na silniku TXXX (ktk_player + MSE/blob src). Stream NIE jest w HTML
+strony (player renderuje się JS-owo). Zamiast tego AJAX endpoint zwraca zaciemniony
+URL pliku:
+
+  GET /api/videofile.php?video_id=<id>&lifetime=8640000
+    -> [{"format":"_hq.mp4","video_url":"<base64+cyrylica>","is_default":1, ...}]
+
+`video_url` to base64 w DWÓCH warstwach zaciemnienia: (1) wielkie łacińskie litery
+podmienione na cyrylicę-homoglif (М->M, С->C, А->A, Е->E), (2) custom alfabet base64
+gdzie `,`->`/`, `~`->`=`, `-`->`+`. Po odkręceniu obu + b64decode dostajemy get_file:
+
+  # stare wideo (shared txxx pool) — URL absolutny:
+  https://videotxxx.com/ext/get_file/9/<hash>/<floor>/<mid>/<mid>_hq.mp4/?d=..&br=..&ti=..
+  # nowe wideo (własna infra vjav, server N) — URL RELATYWNY (prepend https://vjav.com):
+  /get_file/3/<hash>/<floor>/<mid>/<mid>_hq.mp4/?d=..&br=..&ti=..
+
+Dopisanie `&f=video.m3u8` -> 302 chain -> finalny HLS na *.ahcdn.com
+(`key=..,end=..,limit=3` — time-bound, NIE IP-bound, `referer=none` na whiteliscie
+-> portable, gra z residential IP telefonu bez sesji vjav).
+
+Zwracamy get_file (`?f=video.m3u8`) jako type='m3u8' + `mobile_direct_ok` -> playback.py
+owija w `/proxy/hls/<token>/play.m3u8` (passthrough manifestu ~1KB przez VPS, segmenty
+direct z telefonu; patrz stream_proxy.proxy_hls_manifest). Media id w get_file (318577)
+!= page id (5197) — dlatego MUSIMY przejść przez videofile.php. RE 2026-07-10.
+"""
+from __future__ import annotations
+
+import base64
+import json
+import logging
+import re
+
+from app.extractors import browser_get
+from app.extractors._models import StreamSource
+
+log = logging.getLogger(__name__)
+
+_BASE = "https://vjav.com"
+_VIDEO_ID_RE = re.compile(r"/videos/(\d+)/")
+
+# KVS video_url: base64 z cyrylica-homoglifami zamiast łacińskich liter (wielkie +
+# część małych). Odkręcamy je z powrotem na łacinę przed b64decode.
+_HOMOGLYPHS = str.maketrans(
+    {
+        "А": "A", "В": "B", "С": "C", "Е": "E", "Н": "H", "К": "K", "М": "M",
+        "О": "O", "Р": "P", "Т": "T", "Х": "X", "У": "Y",
+        "а": "a", "с": "c", "е": "e", "о": "o", "р": "p", "х": "x", "у": "y",
+    }
+)
+
+
+def _decode_video_url(obfuscated: str) -> str | None:
+    # (1) cyrylica-homoglif -> łacina, (2) custom alfabet base64 -> standardowy.
+    clean = (
+        obfuscated.translate(_HOMOGLYPHS)
+        .replace(",", "/")
+        .replace("~", "=")
+        .replace("-", "+")
+    )
+    clean += "=" * (-len(clean) % 4)  # padding do wielokrotności 4
+    try:
+        return base64.b64decode(clean).decode("utf-8", "ignore")
+    except Exception:
+        return None
+
+
+def _quality_label(fmt: str | None) -> str | None:
+    """`_hq.mp4` -> 'HD', `_sd.mp4` -> 'SD'. Trailer (`_tr`) filtrowany osobno."""
+    if not fmt:
+        return None
+    tok = fmt.strip("_").replace(".mp4", "").lower()
+    return {"hq": "HD", "sd": "SD", "lq": "LOW", "hd": "HD"}.get(tok, tok.upper() or None)
+
+
+def extract(page_url: str, *, timeout: float = 60.0) -> list[StreamSource] | None:
+    m = _VIDEO_ID_RE.search(page_url)
+    if not m:
+        log.info("vjav: brak video id w %s", page_url)
+        return None
+    vid = m.group(1)
+    api = f"{_BASE}/api/videofile.php?video_id={vid}&lifetime=8640000"
+    try:
+        res = browser_get(
+            api,
+            timeout=timeout,
+            headers={"Referer": page_url, "X-Requested-With": "XMLHttpRequest"},
+        )
+        body = res.text if hasattr(res, "text") else res
+        data = json.loads(body)
+    except Exception as e:
+        log.info("vjav: videofile.php fetch/parse fail %s: %s", api, e)
+        return None
+    if not isinstance(data, list):
+        return None
+
+    seen: set[str] = set()
+    out: list[StreamSource] = []
+    for entry in data:
+        if not isinstance(entry, dict):
+            continue
+        fmt = (entry.get("format") or "").strip()
+        # `_tr.mp4` = trailer preview, NIE pełne wideo — pomijamy.
+        if fmt.strip("_").replace(".mp4", "").lower() == "tr":
+            continue
+        vu = entry.get("video_url")
+        if not vu:
+            continue
+        get_file = _decode_video_url(vu)
+        if not get_file or "/get_file/" not in get_file:
+            continue
+        # Nowe wideo daje URL relatywny (/get_file/...) — prepend host vjav.
+        if get_file.startswith("/"):
+            get_file = _BASE + get_file
+        m3u8 = get_file + ("&" if "?" in get_file else "?") + "f=video.m3u8"
+        if m3u8 in seen:
+            continue
+        seen.add(m3u8)
+        out.append(
+            StreamSource(
+                link=m3u8,
+                type="m3u8",
+                quality=_quality_label(fmt),
+                referer=page_url,
+                # Finalny HLS (txxx.ahcdn.com) time-bound + referer=none whitelisted ->
+                # portable. mobile_direct_ok -> playback.py owija w /proxy/hls passthrough.
+                raw={"mobile_direct_ok": True},
+            )
+        )
+    if not out:
+        log.info("vjav: brak dekodowalnego video_url dla %s", page_url)
+        return None
+    return out