feat(jav): supjav.com scraper + extractor (embed-aggregator, 4th JAV source)

Fourth JAV vertical source (origin tube:supjav, gated to JAV tab via JAV_ORIGINS).

Browse: CF-blocks datacenter IPs, so listing goes through the Bright Data ISP proxy
with retry (the proxy rotates IPs and CF only lets some through). Homepage-latest only
(~66 newest); deeper /page/N/ stays CF-403 even via proxy. Metadata parsed from the
listing cards (title, thumbnail, JAV code, year/month) rather than per-post details,
which are flaky and slow through the proxy.

Stream: supjav hides the real hoster behind a per-server data-link (hex). base.js loads
lk1.supremejav.com/supjav.php?l=<data-link>, which reverses the hex string and fetches
?c=<reversed>, 302-ing to the hoster (TV->turbovid, FST->fc2stream, ST->streamtape,
VOE->voe). The extractor reproduces that: fetch detail via proxy, reverse each data-link,
resolve through lk1 (reachable direct from the server), return the hosters as type=hoster
so the phone resolves them (dood/filemoon native, the rest via the WebView fallback on the
residential IP). lk1 needs no proxy; only the detail fetch does.

Backend-only, no mobile change.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

app/connectors/direct_scrapers/__init__.py

@@ -137,6 +137,7 @@ from app.connectors.direct_scrapers.hdporngg import HDPornGGScraper  # noqa: E40
 from app.connectors.direct_scrapers.hqfap import HQFapScraper  # noqa: E402
 from app.connectors.direct_scrapers.javflix import JavflixScraper  # noqa: E402
 from app.connectors.direct_scrapers.vjav import VjavScraper  # noqa: E402
+from app.connectors.direct_scrapers.supjav import SupjavScraper  # noqa: E402
 from app.connectors.direct_scrapers.neporn import NepornScraper  # noqa: E402
 from app.connectors.direct_scrapers.superporn import SuperpornScraper  # noqa: E402
 from app.connectors.direct_scrapers.eporner_api import EpornerApiScraper  # noqa: E402
@@ -227,6 +228,10 @@ ALL_BROWSE_SCRAPERS: list[type[BaseBrowseScraper]] = [
     # TXXX network: sitemap id-walk (newest=max id) + JSON metadata API. Stream
     # videofile.php → get_file HLS (portable, /proxy/hls passthrough). RE 2026-07-10.
     VjavScraper,
+    # SupjavScraper — JAV vertical (osobna sekcja, origin tube:supjav w JAV_ORIGINS).
+    # CF-blokuje VPS → browse homepage przez Bright Data proxy. Embed-aggregator:
+    # data-link reverse-hex → lk1.supremejav.com → hoster (extractor supjav). RE 2026-07-10.
+    SupjavScraper,
     # NepornScraper — dołączony 2026-06-10 (user request). KVS engine (jak freshporno/
     # porn00), /latest-updates/N/. JSON-LD (title+desc+uploadDate+thumb) + video:duration
     # meta + /models/ performerzy + /categories/ tagi. Brak studio (tytuł bywa

app/connectors/direct_scrapers/supjav.py

@@ -0,0 +1,126 @@
+"""supjav.com — JAV browse scraper (przez Bright Data proxy, CF-blokuje VPS IP).
+
+Osobna pula JAV (sekcja JAV), origin `tube:supjav` w JAV_ORIGINS. supjav agreguje
+embedy (serwery TV/FST/ST/VOE), gdzie realny hoster jest schowany za `data-link`
+(reverse-hex → lk1.supremejav.com); rozwiązuje go ekstraktor `app/extractors/tubes/supjav.py`.
+
+Browse (RE 2026-07-10):
+  - listing: homepage `/` (najnowsze ~66, WordPress). Głębsza paginacja `/page/N/` leci
+    CF 403 nawet przez proxy → browse ogranicza się do strony 1 (scheduler odświeża).
+  - karty: `<div class="post"><a href="/<id>.html" title="<title>" rel="bookmark"
+    class="img"><img src=".../images/YYYY/MM/<CODE>.jpg!320x216.jpg" class="thumb">`.
+  - Metadane bierzemy Z KARTY (tytuł/thumb/URL/kod JAV/data z roku-miesiąca ścieżki
+    thumb), NIE z detalu — detal przez proxy jest flaky (CF przepuszcza ~40% IP), a
+    per-post fetch byłby wolny i lossy. Studio/tagi (poza kodem) dorabia enrich później.
+
+supjav blokuje VPS IP twardym CF → listing idzie przez Bright Data ISP proxy z retry
+(`fetch_supjav_html`, rotacja IP aż trafi przepuszczający). Bez proxy scraper no-op.
+"""
+from __future__ import annotations
+
+import html as _html
+import logging
+import re
+from datetime import date
+
+from app.config import get_settings
+from app.connectors.base import RawPlaybackSource, RawScene, RawTag
+from app.connectors.direct_scrapers._browse_base import BaseBrowseScraper
+from app.extractors.tubes.supjav import fetch_supjav_html
+from app.normalize.text import slugify
+
+log = logging.getLogger(__name__)
+
+_BASE = "https://supjav.com"
+# Karta: link-obraz z href + title + thumb. title="([^"]*)" jest bezpieczne (wartość
+# atrybutu nie zawiera surowego "), więc tytuły z `>` (np. `<<3-Day Offer>>`) przechodzą.
+# Link-obraz karty: <a href="URL" ... title="TITLE"><img src|data-original="THUMB">.
+# supjav miesza kolejność atrybutów (href/rel/class="img"/title w dwóch wariantach), więc
+# NIE wymagamy pozycji class="img" — wymóg `>\s*<img` i tak izoluje link-obraz od
+# link-tekstu (ten drugi po `>` ma tekst, nie <img>). Thumb eager `src=` lub lazy
+# `data-original=` (karty poniżej folda: src="data:..." placeholder + realny data-original).
+_CARD_RE = re.compile(
+    r'href="(https://supjav\.com/\d+\.html)"[^>]*?\btitle="([^"]*)"[^>]*?>\s*'
+    r'<img[^>]*?\b(?:src|data-original|data-src|data-lazy-src)="(https://img\.supjav\.com/[^"]+)"',
+    re.IGNORECASE,
+)
+_THUMB_DATE_RE = re.compile(r"/images/(\d{4})/(\d{2})/")
+# Kod JAV: FC2PPV ma cyfrę w prefiksie (FC2PPV 4931572), studia mają myślnik (MGOLD-053,
+# DLDSS-510, SNOS-258). Prefiks liter+cyfr, separator - lub spacja, numer.
+_CODE_RE = re.compile(r"^([A-Z][A-Z0-9]{1,7}[-\s]\d{2,7})", re.IGNORECASE)
+
+
+class SupjavScraper(BaseBrowseScraper):
+    sitetag = "supjav"
+    _timeout = 70.0
+
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        # CF-blok VPS IP → listing przez Bright Data proxy (jak superporn).
+        self._proxy = get_settings().brightdata_proxy_url
+        if not self._proxy:
+            log.warning("supjav: BRIGHTDATA_PROXY_URL unset — scraper disabled")
+
+    def _listing_url(self, page: int) -> str:
+        return f"{_BASE}/" if page <= 1 else f"{_BASE}/page/{page}/"
+
+    def crawl_page(self, page: int) -> list[RawScene] | None:
+        if not self._proxy:
+            return []  # bez proxy gwarantowany CF 403 (exhausted)
+        html = fetch_supjav_html(self._listing_url(page), proxy=self._proxy, timeout=self._timeout)
+        if not html or len(html) < 15000:
+            # sam challenge / fetch fail. page 1 = transient (None, retry potem);
+            # page > 1 = najpewniej CF-403 głębszej paginacji = koniec (exhausted).
+            return None if page <= 1 else []
+        seen: set[str] = set()
+        out: list[RawScene] = []
+        for m in _CARD_RE.finditer(html):
+            url, title_raw, thumb_raw = m.group(1), m.group(2), m.group(3)
+            if url in seen:
+                continue
+            seen.add(url)
+            raw = self._card_to_scene(url, title_raw, thumb_raw)
+            if raw is not None:
+                out.append(raw)
+        return out
+
+    def _card_to_scene(self, url: str, title_raw: str, thumb_raw: str) -> RawScene | None:
+        title = _html.unescape(title_raw).strip()
+        if not title:
+            return None
+        thumb = thumb_raw.split("!")[0] or None  # utnij `!320x216.jpg` → pełna rozdz.
+        release_date: date | None = None
+        dm = _THUMB_DATE_RE.search(thumb_raw)
+        if dm:
+            try:
+                release_date = date(int(dm.group(1)), int(dm.group(2)), 1)
+            except ValueError:
+                release_date = None
+
+        tags: list[RawTag] = []
+        cm = _CODE_RE.match(title)
+        if cm:
+            code = re.sub(r"[ ]", "-", cm.group(1).upper())
+            tags.append(RawTag(external_id=f"javcode:{code}", name=code, slug=slugify(code)))
+
+        return RawScene(
+            external_id=f"{self.sitetag}:{url}",
+            title=title,
+            release_date=release_date,
+            url=url,
+            tags=tags,
+            playback_sources=[
+                RawPlaybackSource(
+                    origin=f"tube:{self.sitetag}",
+                    page_url=url,
+                    thumbnail_url=thumb,
+                )
+            ],
+        )
+
+    # HTML detail nieużywany (browse z kart), ale ABC wymaga.
+    def _extract_scene_urls(self, listing_html: str) -> list[str]:  # pragma: no cover
+        return []
+
+    def _parse_detail(self, scene_url: str, detail_html: str) -> RawScene | None:  # pragma: no cover
+        return None

app/extractors/__init__.py

@@ -39,6 +39,7 @@ from app.extractors.tubes import (
     paradisehill,
     porn00,
     porntrex,
+    supjav,
     sxyprn,
     vjav,
     watchporn,
@@ -179,6 +180,8 @@ _REGISTRY: dict[str, Callable[[str], list[StreamSource] | None]] = {
     "javflix": javflix.extract,
     # vjav (JAV, TXXX network) — videofile.php -> get_file HLS (patrz tubes/vjav.py).
     "vjav": vjav.extract,
+    # supjav (JAV, embed-aggregator) — data-link reverse-hex -> lk1 -> hoster (type=hoster).
+    "supjav": supjav.extract,
     # neporn — KVS function/0 + license (jak freshporno). Server-side _kvs resolve →
     # data001.neporn.com/remote_control.php portable (cross-IP 206, 2026-06-10).
     "neporncom": neporn.extract,

app/extractors/tubes/supjav.py

@@ -0,0 +1,112 @@
+"""supjav.com — JAV embed-aggregator. Resolve data-link → hoster (phone-side).
+
+supjav chowa realny hoster za `data-link` (hex) na przyciskach serwerów (TV/FST/ST/VOE).
+base.js: klik serwera → iframe `src = lk1.supremejav.com/supjav.php?l=<data-link>&bg=`.
+supjav.php robi `OLID = data-link.reverse()` (odwrócenie stringa hex) i ładuje
+`supjav.php?c=<OLID>`, które 302-uje na realny hoster (RE 2026-07-10):
+  TV → turbovidhls.com, FST → fc2stream.tv, ST → streamtape.com, VOE → voe.sx
+
+Flow ekstraktora (on-demand, play time):
+  1. fetch detail `/<id>.html` PRZEZ proxy (Bright Data — supjav CF-blokuje VPS IP),
+  2. parse `data-link` z `.btn-server`,
+  3. reverse hex → GET `lk1.supremejav.com/supjav.php?c=<rev>` (direct z VPS, follow 302)
+     → finalny hoster URL,
+  4. zwróć type='hoster' → telefon resolwuje (dood/filemoon natywnie, reszta WebView
+     fallback z residential IP). lk1 osiągalny z VPS bez proxy; tylko detail wymaga proxy.
+"""
+from __future__ import annotations
+
+import logging
+import re
+
+import curl_cffi.requests as _rq
+
+from app.config import get_settings
+from app.extractors import browser_get
+from app.extractors._fetch import _DEFAULT_IMPERSONATE
+from app.extractors._models import StreamSource
+
+log = logging.getLogger(__name__)
+
+_LK = "https://lk1.supremejav.com/supjav.php"
+_DATA_LINK_RE = re.compile(r'class="btn-server[^"]*"\s+data-link="([0-9a-f]{16,})"', re.IGNORECASE)
+# reklama/tracker domeny które lk1 czasem zwraca zamiast hostera — odrzucamy.
+_AD_RE = re.compile(r"(snaptrckr|trackwilltrk|mayzaent|eix304|doppiocdn|/ad\?)", re.IGNORECASE)
+
+
+def fetch_supjav_html(url: str, *, proxy: str | None, timeout: float, tries: int = 6) -> str:
+    """Fetch supjav przez proxy z retry. Bright Data rotuje IP per-request, a CF
+    przepuszcza tylko część IP (reszta dostaje ~6KB challenge). Retry aż trafimy IP
+    który przechodzi (realna strona ma `data-link=`/`<h1>` i jest duża). Zwraca '' gdy
+    wszystkie próby padły."""
+    last = ""
+    for _ in range(max(1, tries)):
+        try:
+            res = browser_get(url, timeout=timeout, proxy=proxy)
+            html = res.text if hasattr(res, "text") else res
+        except Exception:
+            html = ""
+        last = html or last
+        if html and len(html) > 15000 and ("data-link=" in html or "<h1>" in html):
+            return html
+    return last
+
+
+def _resolve_hoster(data_link: str, timeout: float) -> str | None:
+    """reverse hex → lk1 supjav.php?c= → follow 302 → finalny hoster URL."""
+    olid = data_link[::-1]
+    try:
+        s = _rq.Session(impersonate=_DEFAULT_IMPERSONATE)
+        r = s.get(
+            f"{_LK}?c={olid}",
+            headers={"Referer": "https://lk1.supremejav.com/"},
+            allow_redirects=True,
+            timeout=timeout,
+        )
+        final = str(r.url)
+        r.close()
+    except Exception as e:
+        log.info("supjav: lk1 resolve failed (%s): %s", data_link[:12], e)
+        return None
+    # supjav.php bez realnego hostera zostaje na lk1 (albo leci na ad) → odrzuć.
+    if "supremejav.com" in final or _AD_RE.search(final):
+        return None
+    # lk1 dokleja fragment `#supjav.com@<code>` (metadata) — hostery go ignorują, tniemy.
+    return final.split("#", 1)[0]
+
+
+def extract(page_url: str, *, timeout: float = 60.0) -> list[StreamSource] | None:
+    proxy = get_settings().brightdata_proxy_url
+    if not proxy:
+        log.info("supjav: brak proxy — nie mogę pobrać CF-blokowanego detalu %s", page_url)
+        return None
+    html = fetch_supjav_html(page_url, proxy=proxy, timeout=timeout)
+    if not html:
+        log.info("supjav: detail fetch failed (CF) %s", page_url)
+        return None
+
+    data_links = list(dict.fromkeys(_DATA_LINK_RE.findall(html)))
+    if not data_links:
+        log.info("supjav: brak data-link na %s", page_url)
+        return None
+
+    seen: set[str] = set()
+    out: list[StreamSource] = []
+    for dl in data_links:
+        hoster = _resolve_hoster(dl, timeout=min(timeout, 40.0))
+        if not hoster or hoster in seen:
+            continue
+        seen.add(hoster)
+        host = hoster.split("/")[2] if "://" in hoster else hoster
+        out.append(
+            StreamSource(
+                link=hoster,
+                type="hoster",
+                quality=host,
+                referer="https://supjav.com/",
+            )
+        )
+    if not out:
+        log.info("supjav: żaden data-link nie rozwiązał się na hoster (%s)", page_url)
+        return None
+    return out