feat(jav): supjav.com scraper + extractor (embed-aggregator, 4th JAV source)
Fourth JAV vertical source (origin tube:supjav, gated to JAV tab via JAV_ORIGINS). Browse: CF-blocks datacenter IPs, so listing goes through the Bright Data ISP proxy with retry (the proxy rotates IPs and CF only lets some through). Homepage-latest only (~66 newest); deeper /page/N/ stays CF-403 even via proxy. Metadata parsed from the listing cards (title, thumbnail, JAV code, year/month) rather than per-post details, which are flaky and slow through the proxy. Stream: supjav hides the real hoster behind a per-server data-link (hex). base.js loads lk1.supremejav.com/supjav.php?l=<data-link>, which reverses the hex string and fetches ?c=<reversed>, 302-ing to the hoster (TV->turbovid, FST->fc2stream, ST->streamtape, VOE->voe). The extractor reproduces that: fetch detail via proxy, reverse each data-link, resolve through lk1 (reachable direct from the server), return the hosters as type=hoster so the phone resolves them (dood/filemoon native, the rest via the WebView fallback on the residential IP). lk1 needs no proxy; only the detail fetch does. Backend-only, no mobile change. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
goon-foss
parent
5b31459a26
commit
8c3ad2b09a
4 changed files with 246 additions and 0 deletions
|
|
@ -137,6 +137,7 @@ from app.connectors.direct_scrapers.hdporngg import HDPornGGScraper # noqa: E40
|
|||
from app.connectors.direct_scrapers.hqfap import HQFapScraper # noqa: E402
|
||||
from app.connectors.direct_scrapers.javflix import JavflixScraper # noqa: E402
|
||||
from app.connectors.direct_scrapers.vjav import VjavScraper # noqa: E402
|
||||
from app.connectors.direct_scrapers.supjav import SupjavScraper # noqa: E402
|
||||
from app.connectors.direct_scrapers.neporn import NepornScraper # noqa: E402
|
||||
from app.connectors.direct_scrapers.superporn import SuperpornScraper # noqa: E402
|
||||
from app.connectors.direct_scrapers.eporner_api import EpornerApiScraper # noqa: E402
|
||||
|
|
@ -227,6 +228,10 @@ ALL_BROWSE_SCRAPERS: list[type[BaseBrowseScraper]] = [
|
|||
# TXXX network: sitemap id-walk (newest=max id) + JSON metadata API. Stream
|
||||
# videofile.php → get_file HLS (portable, /proxy/hls passthrough). RE 2026-07-10.
|
||||
VjavScraper,
|
||||
# SupjavScraper — JAV vertical (osobna sekcja, origin tube:supjav w JAV_ORIGINS).
|
||||
# CF-blokuje VPS → browse homepage przez Bright Data proxy. Embed-aggregator:
|
||||
# data-link reverse-hex → lk1.supremejav.com → hoster (extractor supjav). RE 2026-07-10.
|
||||
SupjavScraper,
|
||||
# NepornScraper — dołączony 2026-06-10 (user request). KVS engine (jak freshporno/
|
||||
# porn00), /latest-updates/N/. JSON-LD (title+desc+uploadDate+thumb) + video:duration
|
||||
# meta + /models/ performerzy + /categories/ tagi. Brak studio (tytuł bywa
|
||||
|
|
|
|||
126
app/connectors/direct_scrapers/supjav.py
Normal file
126
app/connectors/direct_scrapers/supjav.py
Normal file
|
|
@ -0,0 +1,126 @@
|
|||
"""supjav.com — JAV browse scraper (przez Bright Data proxy, CF-blokuje VPS IP).
|
||||
|
||||
Osobna pula JAV (sekcja JAV), origin `tube:supjav` w JAV_ORIGINS. supjav agreguje
|
||||
embedy (serwery TV/FST/ST/VOE), gdzie realny hoster jest schowany za `data-link`
|
||||
(reverse-hex → lk1.supremejav.com); rozwiązuje go ekstraktor `app/extractors/tubes/supjav.py`.
|
||||
|
||||
Browse (RE 2026-07-10):
|
||||
- listing: homepage `/` (najnowsze ~66, WordPress). Głębsza paginacja `/page/N/` leci
|
||||
CF 403 nawet przez proxy → browse ogranicza się do strony 1 (scheduler odświeża).
|
||||
- karty: `<div class="post"><a href="/<id>.html" title="<title>" rel="bookmark"
|
||||
class="img"><img src=".../images/YYYY/MM/<CODE>.jpg!320x216.jpg" class="thumb">`.
|
||||
- Metadane bierzemy Z KARTY (tytuł/thumb/URL/kod JAV/data z roku-miesiąca ścieżki
|
||||
thumb), NIE z detalu — detal przez proxy jest flaky (CF przepuszcza ~40% IP), a
|
||||
per-post fetch byłby wolny i lossy. Studio/tagi (poza kodem) dorabia enrich później.
|
||||
|
||||
supjav blokuje VPS IP twardym CF → listing idzie przez Bright Data ISP proxy z retry
|
||||
(`fetch_supjav_html`, rotacja IP aż trafi przepuszczający). Bez proxy scraper no-op.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import html as _html
|
||||
import logging
|
||||
import re
|
||||
from datetime import date
|
||||
|
||||
from app.config import get_settings
|
||||
from app.connectors.base import RawPlaybackSource, RawScene, RawTag
|
||||
from app.connectors.direct_scrapers._browse_base import BaseBrowseScraper
|
||||
from app.extractors.tubes.supjav import fetch_supjav_html
|
||||
from app.normalize.text import slugify
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
_BASE = "https://supjav.com"
|
||||
# Karta: link-obraz z href + title + thumb. title="([^"]*)" jest bezpieczne (wartość
|
||||
# atrybutu nie zawiera surowego "), więc tytuły z `>` (np. `<<3-Day Offer>>`) przechodzą.
|
||||
# Link-obraz karty: <a href="URL" ... title="TITLE"><img src|data-original="THUMB">.
|
||||
# supjav miesza kolejność atrybutów (href/rel/class="img"/title w dwóch wariantach), więc
|
||||
# NIE wymagamy pozycji class="img" — wymóg `>\s*<img` i tak izoluje link-obraz od
|
||||
# link-tekstu (ten drugi po `>` ma tekst, nie <img>). Thumb eager `src=` lub lazy
|
||||
# `data-original=` (karty poniżej folda: src="data:..." placeholder + realny data-original).
|
||||
_CARD_RE = re.compile(
|
||||
r'href="(https://supjav\.com/\d+\.html)"[^>]*?\btitle="([^"]*)"[^>]*?>\s*'
|
||||
r'<img[^>]*?\b(?:src|data-original|data-src|data-lazy-src)="(https://img\.supjav\.com/[^"]+)"',
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_THUMB_DATE_RE = re.compile(r"/images/(\d{4})/(\d{2})/")
|
||||
# Kod JAV: FC2PPV ma cyfrę w prefiksie (FC2PPV 4931572), studia mają myślnik (MGOLD-053,
|
||||
# DLDSS-510, SNOS-258). Prefiks liter+cyfr, separator - lub spacja, numer.
|
||||
_CODE_RE = re.compile(r"^([A-Z][A-Z0-9]{1,7}[-\s]\d{2,7})", re.IGNORECASE)
|
||||
|
||||
|
||||
class SupjavScraper(BaseBrowseScraper):
|
||||
sitetag = "supjav"
|
||||
_timeout = 70.0
|
||||
|
||||
def __init__(self, *args, **kwargs) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
# CF-blok VPS IP → listing przez Bright Data proxy (jak superporn).
|
||||
self._proxy = get_settings().brightdata_proxy_url
|
||||
if not self._proxy:
|
||||
log.warning("supjav: BRIGHTDATA_PROXY_URL unset — scraper disabled")
|
||||
|
||||
def _listing_url(self, page: int) -> str:
|
||||
return f"{_BASE}/" if page <= 1 else f"{_BASE}/page/{page}/"
|
||||
|
||||
def crawl_page(self, page: int) -> list[RawScene] | None:
|
||||
if not self._proxy:
|
||||
return [] # bez proxy gwarantowany CF 403 (exhausted)
|
||||
html = fetch_supjav_html(self._listing_url(page), proxy=self._proxy, timeout=self._timeout)
|
||||
if not html or len(html) < 15000:
|
||||
# sam challenge / fetch fail. page 1 = transient (None, retry potem);
|
||||
# page > 1 = najpewniej CF-403 głębszej paginacji = koniec (exhausted).
|
||||
return None if page <= 1 else []
|
||||
seen: set[str] = set()
|
||||
out: list[RawScene] = []
|
||||
for m in _CARD_RE.finditer(html):
|
||||
url, title_raw, thumb_raw = m.group(1), m.group(2), m.group(3)
|
||||
if url in seen:
|
||||
continue
|
||||
seen.add(url)
|
||||
raw = self._card_to_scene(url, title_raw, thumb_raw)
|
||||
if raw is not None:
|
||||
out.append(raw)
|
||||
return out
|
||||
|
||||
def _card_to_scene(self, url: str, title_raw: str, thumb_raw: str) -> RawScene | None:
|
||||
title = _html.unescape(title_raw).strip()
|
||||
if not title:
|
||||
return None
|
||||
thumb = thumb_raw.split("!")[0] or None # utnij `!320x216.jpg` → pełna rozdz.
|
||||
release_date: date | None = None
|
||||
dm = _THUMB_DATE_RE.search(thumb_raw)
|
||||
if dm:
|
||||
try:
|
||||
release_date = date(int(dm.group(1)), int(dm.group(2)), 1)
|
||||
except ValueError:
|
||||
release_date = None
|
||||
|
||||
tags: list[RawTag] = []
|
||||
cm = _CODE_RE.match(title)
|
||||
if cm:
|
||||
code = re.sub(r"[ ]", "-", cm.group(1).upper())
|
||||
tags.append(RawTag(external_id=f"javcode:{code}", name=code, slug=slugify(code)))
|
||||
|
||||
return RawScene(
|
||||
external_id=f"{self.sitetag}:{url}",
|
||||
title=title,
|
||||
release_date=release_date,
|
||||
url=url,
|
||||
tags=tags,
|
||||
playback_sources=[
|
||||
RawPlaybackSource(
|
||||
origin=f"tube:{self.sitetag}",
|
||||
page_url=url,
|
||||
thumbnail_url=thumb,
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
# HTML detail nieużywany (browse z kart), ale ABC wymaga.
|
||||
def _extract_scene_urls(self, listing_html: str) -> list[str]: # pragma: no cover
|
||||
return []
|
||||
|
||||
def _parse_detail(self, scene_url: str, detail_html: str) -> RawScene | None: # pragma: no cover
|
||||
return None
|
||||
|
|
@ -39,6 +39,7 @@ from app.extractors.tubes import (
|
|||
paradisehill,
|
||||
porn00,
|
||||
porntrex,
|
||||
supjav,
|
||||
sxyprn,
|
||||
vjav,
|
||||
watchporn,
|
||||
|
|
@ -179,6 +180,8 @@ _REGISTRY: dict[str, Callable[[str], list[StreamSource] | None]] = {
|
|||
"javflix": javflix.extract,
|
||||
# vjav (JAV, TXXX network) — videofile.php -> get_file HLS (patrz tubes/vjav.py).
|
||||
"vjav": vjav.extract,
|
||||
# supjav (JAV, embed-aggregator) — data-link reverse-hex -> lk1 -> hoster (type=hoster).
|
||||
"supjav": supjav.extract,
|
||||
# neporn — KVS function/0 + license (jak freshporno). Server-side _kvs resolve →
|
||||
# data001.neporn.com/remote_control.php portable (cross-IP 206, 2026-06-10).
|
||||
"neporncom": neporn.extract,
|
||||
|
|
|
|||
112
app/extractors/tubes/supjav.py
Normal file
112
app/extractors/tubes/supjav.py
Normal file
|
|
@ -0,0 +1,112 @@
|
|||
"""supjav.com — JAV embed-aggregator. Resolve data-link → hoster (phone-side).
|
||||
|
||||
supjav chowa realny hoster za `data-link` (hex) na przyciskach serwerów (TV/FST/ST/VOE).
|
||||
base.js: klik serwera → iframe `src = lk1.supremejav.com/supjav.php?l=<data-link>&bg=`.
|
||||
supjav.php robi `OLID = data-link.reverse()` (odwrócenie stringa hex) i ładuje
|
||||
`supjav.php?c=<OLID>`, które 302-uje na realny hoster (RE 2026-07-10):
|
||||
TV → turbovidhls.com, FST → fc2stream.tv, ST → streamtape.com, VOE → voe.sx
|
||||
|
||||
Flow ekstraktora (on-demand, play time):
|
||||
1. fetch detail `/<id>.html` PRZEZ proxy (Bright Data — supjav CF-blokuje VPS IP),
|
||||
2. parse `data-link` z `.btn-server`,
|
||||
3. reverse hex → GET `lk1.supremejav.com/supjav.php?c=<rev>` (direct z VPS, follow 302)
|
||||
→ finalny hoster URL,
|
||||
4. zwróć type='hoster' → telefon resolwuje (dood/filemoon natywnie, reszta WebView
|
||||
fallback z residential IP). lk1 osiągalny z VPS bez proxy; tylko detail wymaga proxy.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
|
||||
import curl_cffi.requests as _rq
|
||||
|
||||
from app.config import get_settings
|
||||
from app.extractors import browser_get
|
||||
from app.extractors._fetch import _DEFAULT_IMPERSONATE
|
||||
from app.extractors._models import StreamSource
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
_LK = "https://lk1.supremejav.com/supjav.php"
|
||||
_DATA_LINK_RE = re.compile(r'class="btn-server[^"]*"\s+data-link="([0-9a-f]{16,})"', re.IGNORECASE)
|
||||
# reklama/tracker domeny które lk1 czasem zwraca zamiast hostera — odrzucamy.
|
||||
_AD_RE = re.compile(r"(snaptrckr|trackwilltrk|mayzaent|eix304|doppiocdn|/ad\?)", re.IGNORECASE)
|
||||
|
||||
|
||||
def fetch_supjav_html(url: str, *, proxy: str | None, timeout: float, tries: int = 6) -> str:
|
||||
"""Fetch supjav przez proxy z retry. Bright Data rotuje IP per-request, a CF
|
||||
przepuszcza tylko część IP (reszta dostaje ~6KB challenge). Retry aż trafimy IP
|
||||
który przechodzi (realna strona ma `data-link=`/`<h1>` i jest duża). Zwraca '' gdy
|
||||
wszystkie próby padły."""
|
||||
last = ""
|
||||
for _ in range(max(1, tries)):
|
||||
try:
|
||||
res = browser_get(url, timeout=timeout, proxy=proxy)
|
||||
html = res.text if hasattr(res, "text") else res
|
||||
except Exception:
|
||||
html = ""
|
||||
last = html or last
|
||||
if html and len(html) > 15000 and ("data-link=" in html or "<h1>" in html):
|
||||
return html
|
||||
return last
|
||||
|
||||
|
||||
def _resolve_hoster(data_link: str, timeout: float) -> str | None:
|
||||
"""reverse hex → lk1 supjav.php?c= → follow 302 → finalny hoster URL."""
|
||||
olid = data_link[::-1]
|
||||
try:
|
||||
s = _rq.Session(impersonate=_DEFAULT_IMPERSONATE)
|
||||
r = s.get(
|
||||
f"{_LK}?c={olid}",
|
||||
headers={"Referer": "https://lk1.supremejav.com/"},
|
||||
allow_redirects=True,
|
||||
timeout=timeout,
|
||||
)
|
||||
final = str(r.url)
|
||||
r.close()
|
||||
except Exception as e:
|
||||
log.info("supjav: lk1 resolve failed (%s): %s", data_link[:12], e)
|
||||
return None
|
||||
# supjav.php bez realnego hostera zostaje na lk1 (albo leci na ad) → odrzuć.
|
||||
if "supremejav.com" in final or _AD_RE.search(final):
|
||||
return None
|
||||
# lk1 dokleja fragment `#supjav.com@<code>` (metadata) — hostery go ignorują, tniemy.
|
||||
return final.split("#", 1)[0]
|
||||
|
||||
|
||||
def extract(page_url: str, *, timeout: float = 60.0) -> list[StreamSource] | None:
|
||||
proxy = get_settings().brightdata_proxy_url
|
||||
if not proxy:
|
||||
log.info("supjav: brak proxy — nie mogę pobrać CF-blokowanego detalu %s", page_url)
|
||||
return None
|
||||
html = fetch_supjav_html(page_url, proxy=proxy, timeout=timeout)
|
||||
if not html:
|
||||
log.info("supjav: detail fetch failed (CF) %s", page_url)
|
||||
return None
|
||||
|
||||
data_links = list(dict.fromkeys(_DATA_LINK_RE.findall(html)))
|
||||
if not data_links:
|
||||
log.info("supjav: brak data-link na %s", page_url)
|
||||
return None
|
||||
|
||||
seen: set[str] = set()
|
||||
out: list[StreamSource] = []
|
||||
for dl in data_links:
|
||||
hoster = _resolve_hoster(dl, timeout=min(timeout, 40.0))
|
||||
if not hoster or hoster in seen:
|
||||
continue
|
||||
seen.add(hoster)
|
||||
host = hoster.split("/")[2] if "://" in hoster else hoster
|
||||
out.append(
|
||||
StreamSource(
|
||||
link=hoster,
|
||||
type="hoster",
|
||||
quality=host,
|
||||
referer="https://supjav.com/",
|
||||
)
|
||||
)
|
||||
if not out:
|
||||
log.info("supjav: żaden data-link nie rozwiązał się na hoster (%s)", page_url)
|
||||
return None
|
||||
return out
|
||||
Loading…
Add table
Reference in a new issue