From b4b9b7690a9be2286732e0ce62291239cb02e434 Mon Sep 17 00:00:00 2001 From: jtrzupek Date: Fri, 10 Jul 2026 13:13:35 +0200 Subject: [PATCH] feat(jav): vjav.com scraper + extractor (TXXX-network JAV source) Second JAV vertical source (origin tube:vjav, gated to JAV tab via JAV_ORIGINS). Browse: SPA has no server-rendered listing, so id-walk over sitemap_vids (newest = highest video id) feeding the rich JSON metadata API (/api/json/video/1///.json): title, duration, post_date, channel->studio, models->performers, categories+tags. Stream: videofile.php returns video_url in two obfuscation layers, Cyrillic homoglyphs (M/C/A/E) over a custom base64 alphabet (comma->slash, tilde->pad, dash->plus). Decoded get_file is absolute (old shared-txxx videos) or relative (new vjav-infra, prepend host); adding f=video.m3u8 yields a portable, time-bound ahcdn HLS (referer=none whitelisted, not IP-bound). Returned as m3u8 + mobile_direct_ok so playback routes it through /proxy/hls: manifest passthrough, segments direct from the phone (verified 206 cross-IP, 0 VPS video bandwidth). Backend-only, no mobile change (JAV tab + jav param already shipped). Co-Authored-By: Claude Opus 4.8 --- app/connectors/direct_scrapers/__init__.py | 5 + app/connectors/direct_scrapers/vjav.py | 205 +++++++++++++++++++++ app/extractors/__init__.py | 3 + app/extractors/tubes/vjav.py | 134 ++++++++++++++ 4 files changed, 347 insertions(+) create mode 100644 app/connectors/direct_scrapers/vjav.py create mode 100644 app/extractors/tubes/vjav.py diff --git a/app/connectors/direct_scrapers/__init__.py b/app/connectors/direct_scrapers/__init__.py index 0c76da1..ca31c6f 100644 --- a/app/connectors/direct_scrapers/__init__.py +++ b/app/connectors/direct_scrapers/__init__.py @@ -136,6 +136,7 @@ from app.connectors.direct_scrapers.fullmovies import FullmoviesScraper # noqa: from app.connectors.direct_scrapers.hdporngg import HDPornGGScraper # noqa: E402 from app.connectors.direct_scrapers.hqfap import HQFapScraper # noqa: E402 from app.connectors.direct_scrapers.javflix import JavflixScraper # noqa: E402 +from app.connectors.direct_scrapers.vjav import VjavScraper # noqa: E402 from app.connectors.direct_scrapers.neporn import NepornScraper # noqa: E402 from app.connectors.direct_scrapers.superporn import SuperpornScraper # noqa: E402 from app.connectors.direct_scrapers.eporner_api import EpornerApiScraper # noqa: E402 @@ -222,6 +223,10 @@ ALL_BROWSE_SCRAPERS: list[type[BaseBrowseScraper]] = [ # JavflixScraper — JAV vertical (osobna sekcja). origin tube:javflix jest w # JAV_ORIGINS → list_scenes wyklucza je z głównego feedu (tylko zakładka JAV). JavflixScraper, + # VjavScraper — JAV vertical (osobna sekcja, origin tube:vjav w JAV_ORIGINS). + # TXXX network: sitemap id-walk (newest=max id) + JSON metadata API. Stream + # videofile.php → get_file HLS (portable, /proxy/hls passthrough). RE 2026-07-10. + VjavScraper, # NepornScraper — dołączony 2026-06-10 (user request). KVS engine (jak freshporno/ # porn00), /latest-updates/N/. JSON-LD (title+desc+uploadDate+thumb) + video:duration # meta + /models/ performerzy + /categories/ tagi. Brak studio (tytuł bywa diff --git a/app/connectors/direct_scrapers/vjav.py b/app/connectors/direct_scrapers/vjav.py new file mode 100644 index 0000000..82f5f34 --- /dev/null +++ b/app/connectors/direct_scrapers/vjav.py @@ -0,0 +1,205 @@ +"""vjav.com — JAV browse scraper (TXXX network). Sitemap id-walk + JSON metadata API. + +Osobna pula JAV (tytuły azjatyckie + `title_jp`, studia typu JAVHD), origin `tube:vjav`. +Świadomie orphan vertical (sekcja JAV) — NIE deduplikuje się z zachodnim katalogiem. + +vjav to SPA (ktk_player + blob) — listing renderuje się JS-owo, więc HTML listingu +nie ma linków do scen. Zamiast tego: + - **browse**: `sitemap.xml` -> `sitemap_vids_N.xml` (id-e wszystkich wideo). Newest + = najwyższe id (auto-increment). Cache posortowanej listy id malejąco, page N = + kolejny wycinek PER_PAGE. + - **metadata**: bogaty JSON API `GET /api/json/video/1///.json` + (floor = id//1000*1000) — title (+title_jp), duration, post_date, channel.title + (studio), models{} (performerki), categories{}+tags{}. + - **stream**: osobny ekstraktor `app/extractors/tubes/vjav.py` (videofile.php -> + get_file HLS). RE 2026-07-10. + +Override `crawl_page()` (API/sitemap flow, wzór jak EpornerApiScraper). HTML-owe +_listing_url/_extract/_parse to stuby (ABC ich wymaga, nieużywane). +""" +from __future__ import annotations + +import json +import logging +import re +from datetime import date, datetime + +from app.connectors.base import ( + RawPerformer, + RawPlaybackSource, + RawScene, + RawStudio, + RawTag, +) +from app.connectors.direct_scrapers._browse_base import BaseBrowseScraper +from app.extractors import browser_get +from app.normalize.text import slugify + +log = logging.getLogger(__name__) + +_BASE = "https://vjav.com" +_PER_PAGE = 40 +_SITEMAP_INDEX = f"{_BASE}/sitemap.xml" +_SITEMAP_RE = re.compile(r"https://vjav\.com/sitemap_vids_\d+\.xml") +_VID_RE = re.compile(r"/videos/(\d+)/") + + +def _parse_duration(value: str | None) -> int | None: + """`1:16:43` / `14:17` -> sekundy.""" + if not value: + return None + try: + parts = [int(p) for p in value.strip().split(":")] + except ValueError: + return None + sec = 0 + for p in parts: + sec = sec * 60 + p + return sec or None + + +def _parse_post_date(value: str | None) -> date | None: + if not value: + return None + try: + return datetime.strptime(value.strip(), "%Y-%m-%d %H:%M:%S").date() + except ValueError: + return None + + +class VjavScraper(BaseBrowseScraper): + sitetag = "vjav" + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + # Cache id-ów wideo (posortowane malejąco = newest first), per instancja/run. + self._sorted_ids: list[int] | None = None + + # ---- browse: sitemap id-walk ------------------------------------------------ + def _load_ids(self) -> None: + try: + res = browser_get(_SITEMAP_INDEX, timeout=self._timeout) + idx = res.text if hasattr(res, "text") else res + except Exception as e: + log.warning("vjav: sitemap index fetch fail: %s", e) + self._sorted_ids = [] + return + maps = list(dict.fromkeys(_SITEMAP_RE.findall(idx))) + ids: set[int] = set() + for sm in maps: + try: + res = browser_get(sm, timeout=self._timeout) + body = res.text if hasattr(res, "text") else res + except Exception as e: + log.info("vjav: sitemap %s fetch fail: %s", sm, e) + continue + for m in _VID_RE.finditer(body): + ids.add(int(m.group(1))) + self._sorted_ids = sorted(ids, reverse=True) + log.info("vjav: załadowano %d id z %d sitemap", len(self._sorted_ids), len(maps)) + + def _meta_url(self, vid: int) -> str: + floor = (vid // 1000) * 1000 + return f"{_BASE}/api/json/video/1/{floor}/{vid}/{vid}.json" + + # ---- metadata JSON -> RawScene ---------------------------------------------- + def _parse_meta(self, vid: int, v: dict) -> RawScene | None: + title = (v.get("title") or "").strip() + slug = (v.get("dir") or "").strip() + if not title or not slug: + return None + page_url = f"{_BASE}/videos/{vid}/{slug}/" + duration_sec = _parse_duration(v.get("duration")) + release_date = _parse_post_date(v.get("post_date")) + thumb = (v.get("thumbsrc") or v.get("thumb") or "").strip() or None + + studio: RawStudio | None = None + ch = v.get("channel") + if isinstance(ch, dict) and (ch.get("title") or "").strip(): + st = ch["title"].strip() + sg = slugify(st) + studio = RawStudio(external_id=f"vjav:studio:{sg}", name=st, slug=sg) + + performers: list[RawPerformer] = [] + models = v.get("models") + if isinstance(models, dict): + for mid, md in models.items(): + nm = (md.get("title") or "").strip() if isinstance(md, dict) else "" + if nm: + performers.append(RawPerformer(external_id=f"vjav:model:{mid}", name=nm)) + + tags: list[RawTag] = [] + seen_slugs: set[str] = set() + for coll in ("categories", "tags"): + c = v.get(coll) + if not isinstance(c, dict): + continue + for td in c.values(): + nm = (td.get("title") or "").strip() if isinstance(td, dict) else "" + if not nm: + continue + sg = slugify(nm) + if not sg or sg in seen_slugs or len(sg) > 60: + continue + seen_slugs.add(sg) + tags.append(RawTag(external_id=f"vjav:tag:{sg}", name=nm, slug=sg)) + + return RawScene( + external_id=f"{self.sitetag}:{page_url}", + title=title, + release_date=release_date, + duration_sec=duration_sec, + url=page_url, + studio=studio, + performers=performers, + tags=tags, + playback_sources=[ + RawPlaybackSource( + origin=f"tube:{self.sitetag}", + page_url=page_url, + duration_sec=duration_sec, + thumbnail_url=thumb, + ) + ], + raw={"source": "vjav_api", "id": vid}, + ) + + def crawl_page(self, page: int) -> list[RawScene] | None: + if self._sorted_ids is None: + self._load_ids() + if not self._sorted_ids: + # [] gdy sitemap pusty (exhausted); None tylko przy fetch-failu indexu. + return [] if self._sorted_ids == [] else None + start = (page - 1) * _PER_PAGE + chunk = self._sorted_ids[start : start + _PER_PAGE] + if not chunk: + return [] + out: list[RawScene] = [] + for vid in chunk: + try: + res = browser_get(self._meta_url(vid), timeout=self._timeout, headers={"Referer": _BASE + "/"}) + body = res.text if hasattr(res, "text") else res + v = json.loads(body).get("video") + except Exception as e: + log.info("vjav: meta fetch/parse fail id=%s: %s", vid, e) + continue + if not isinstance(v, dict): + continue + try: + raw = self._parse_meta(vid, v) + except Exception as e: + log.warning("vjav: meta parse fail id=%s: %s", vid, e) + continue + if raw is not None: + out.append(raw) + return out + + # ---- HTML stuby (nieużywane, ABC wymaga) ------------------------------------ + def _listing_url(self, page: int) -> str: # pragma: no cover + raise NotImplementedError("VjavScraper używa crawl_page (sitemap+API)") + + def _extract_scene_urls(self, listing_html: str) -> list[str]: # pragma: no cover + raise NotImplementedError + + def _parse_detail(self, scene_url: str, detail_html: str) -> RawScene | None: # pragma: no cover + raise NotImplementedError diff --git a/app/extractors/__init__.py b/app/extractors/__init__.py index 84403eb..14d8e63 100644 --- a/app/extractors/__init__.py +++ b/app/extractors/__init__.py @@ -40,6 +40,7 @@ from app.extractors.tubes import ( porn00, porntrex, sxyprn, + vjav, watchporn, xhamster, yespornvip, @@ -176,6 +177,8 @@ _REGISTRY: dict[str, Callable[[str], list[StreamSource] | None]] = { # patternem → type='hoster', telefon resolwuje (voe backend, dood/filemoon phone-side). # Wrapper javflix.extract odsiewa placeholder players.mp4. "javflix": javflix.extract, + # vjav (JAV, TXXX network) — videofile.php -> get_file HLS (patrz tubes/vjav.py). + "vjav": vjav.extract, # neporn — KVS function/0 + license (jak freshporno). Server-side _kvs resolve → # data001.neporn.com/remote_control.php portable (cross-IP 206, 2026-06-10). "neporncom": neporn.extract, diff --git a/app/extractors/tubes/vjav.py b/app/extractors/tubes/vjav.py new file mode 100644 index 0000000..9163d4e --- /dev/null +++ b/app/extractors/tubes/vjav.py @@ -0,0 +1,134 @@ +"""vjav.com — TXXX-network JAV tube. HLS stream extractor. + +vjav to SPA na silniku TXXX (ktk_player + MSE/blob src). Stream NIE jest w HTML +strony (player renderuje się JS-owo). Zamiast tego AJAX endpoint zwraca zaciemniony +URL pliku: + + GET /api/videofile.php?video_id=&lifetime=8640000 + -> [{"format":"_hq.mp4","video_url":"","is_default":1, ...}] + +`video_url` to base64 w DWÓCH warstwach zaciemnienia: (1) wielkie łacińskie litery +podmienione na cyrylicę-homoglif (М->M, С->C, А->A, Е->E), (2) custom alfabet base64 +gdzie `,`->`/`, `~`->`=`, `-`->`+`. Po odkręceniu obu + b64decode dostajemy get_file: + + # stare wideo (shared txxx pool) — URL absolutny: + https://videotxxx.com/ext/get_file/9////_hq.mp4/?d=..&br=..&ti=.. + # nowe wideo (własna infra vjav, server N) — URL RELATYWNY (prepend https://vjav.com): + /get_file/3////_hq.mp4/?d=..&br=..&ti=.. + +Dopisanie `&f=video.m3u8` -> 302 chain -> finalny HLS na *.ahcdn.com +(`key=..,end=..,limit=3` — time-bound, NIE IP-bound, `referer=none` na whiteliscie +-> portable, gra z residential IP telefonu bez sesji vjav). + +Zwracamy get_file (`?f=video.m3u8`) jako type='m3u8' + `mobile_direct_ok` -> playback.py +owija w `/proxy/hls//play.m3u8` (passthrough manifestu ~1KB przez VPS, segmenty +direct z telefonu; patrz stream_proxy.proxy_hls_manifest). Media id w get_file (318577) +!= page id (5197) — dlatego MUSIMY przejść przez videofile.php. RE 2026-07-10. +""" +from __future__ import annotations + +import base64 +import json +import logging +import re + +from app.extractors import browser_get +from app.extractors._models import StreamSource + +log = logging.getLogger(__name__) + +_BASE = "https://vjav.com" +_VIDEO_ID_RE = re.compile(r"/videos/(\d+)/") + +# KVS video_url: base64 z cyrylica-homoglifami zamiast łacińskich liter (wielkie + +# część małych). Odkręcamy je z powrotem na łacinę przed b64decode. +_HOMOGLYPHS = str.maketrans( + { + "А": "A", "В": "B", "С": "C", "Е": "E", "Н": "H", "К": "K", "М": "M", + "О": "O", "Р": "P", "Т": "T", "Х": "X", "У": "Y", + "а": "a", "с": "c", "е": "e", "о": "o", "р": "p", "х": "x", "у": "y", + } +) + + +def _decode_video_url(obfuscated: str) -> str | None: + # (1) cyrylica-homoglif -> łacina, (2) custom alfabet base64 -> standardowy. + clean = ( + obfuscated.translate(_HOMOGLYPHS) + .replace(",", "/") + .replace("~", "=") + .replace("-", "+") + ) + clean += "=" * (-len(clean) % 4) # padding do wielokrotności 4 + try: + return base64.b64decode(clean).decode("utf-8", "ignore") + except Exception: + return None + + +def _quality_label(fmt: str | None) -> str | None: + """`_hq.mp4` -> 'HD', `_sd.mp4` -> 'SD'. Trailer (`_tr`) filtrowany osobno.""" + if not fmt: + return None + tok = fmt.strip("_").replace(".mp4", "").lower() + return {"hq": "HD", "sd": "SD", "lq": "LOW", "hd": "HD"}.get(tok, tok.upper() or None) + + +def extract(page_url: str, *, timeout: float = 60.0) -> list[StreamSource] | None: + m = _VIDEO_ID_RE.search(page_url) + if not m: + log.info("vjav: brak video id w %s", page_url) + return None + vid = m.group(1) + api = f"{_BASE}/api/videofile.php?video_id={vid}&lifetime=8640000" + try: + res = browser_get( + api, + timeout=timeout, + headers={"Referer": page_url, "X-Requested-With": "XMLHttpRequest"}, + ) + body = res.text if hasattr(res, "text") else res + data = json.loads(body) + except Exception as e: + log.info("vjav: videofile.php fetch/parse fail %s: %s", api, e) + return None + if not isinstance(data, list): + return None + + seen: set[str] = set() + out: list[StreamSource] = [] + for entry in data: + if not isinstance(entry, dict): + continue + fmt = (entry.get("format") or "").strip() + # `_tr.mp4` = trailer preview, NIE pełne wideo — pomijamy. + if fmt.strip("_").replace(".mp4", "").lower() == "tr": + continue + vu = entry.get("video_url") + if not vu: + continue + get_file = _decode_video_url(vu) + if not get_file or "/get_file/" not in get_file: + continue + # Nowe wideo daje URL relatywny (/get_file/...) — prepend host vjav. + if get_file.startswith("/"): + get_file = _BASE + get_file + m3u8 = get_file + ("&" if "?" in get_file else "?") + "f=video.m3u8" + if m3u8 in seen: + continue + seen.add(m3u8) + out.append( + StreamSource( + link=m3u8, + type="m3u8", + quality=_quality_label(fmt), + referer=page_url, + # Finalny HLS (txxx.ahcdn.com) time-bound + referer=none whitelisted -> + # portable. mobile_direct_ok -> playback.py owija w /proxy/hls passthrough. + raw={"mobile_direct_ok": True}, + ) + ) + if not out: + log.info("vjav: brak dekodowalnego video_url dla %s", page_url) + return None + return out