diff --git a/app/connectors/direct_scrapers/__init__.py b/app/connectors/direct_scrapers/__init__.py index ca31c6f..e6b8c48 100644 --- a/app/connectors/direct_scrapers/__init__.py +++ b/app/connectors/direct_scrapers/__init__.py @@ -137,6 +137,7 @@ from app.connectors.direct_scrapers.hdporngg import HDPornGGScraper # noqa: E40 from app.connectors.direct_scrapers.hqfap import HQFapScraper # noqa: E402 from app.connectors.direct_scrapers.javflix import JavflixScraper # noqa: E402 from app.connectors.direct_scrapers.vjav import VjavScraper # noqa: E402 +from app.connectors.direct_scrapers.supjav import SupjavScraper # noqa: E402 from app.connectors.direct_scrapers.neporn import NepornScraper # noqa: E402 from app.connectors.direct_scrapers.superporn import SuperpornScraper # noqa: E402 from app.connectors.direct_scrapers.eporner_api import EpornerApiScraper # noqa: E402 @@ -227,6 +228,10 @@ ALL_BROWSE_SCRAPERS: list[type[BaseBrowseScraper]] = [ # TXXX network: sitemap id-walk (newest=max id) + JSON metadata API. Stream # videofile.php → get_file HLS (portable, /proxy/hls passthrough). RE 2026-07-10. VjavScraper, + # SupjavScraper — JAV vertical (osobna sekcja, origin tube:supjav w JAV_ORIGINS). + # CF-blokuje VPS → browse homepage przez Bright Data proxy. Embed-aggregator: + # data-link reverse-hex → lk1.supremejav.com → hoster (extractor supjav). RE 2026-07-10. + SupjavScraper, # NepornScraper — dołączony 2026-06-10 (user request). KVS engine (jak freshporno/ # porn00), /latest-updates/N/. JSON-LD (title+desc+uploadDate+thumb) + video:duration # meta + /models/ performerzy + /categories/ tagi. Brak studio (tytuł bywa diff --git a/app/connectors/direct_scrapers/supjav.py b/app/connectors/direct_scrapers/supjav.py new file mode 100644 index 0000000..8f2cb66 --- /dev/null +++ b/app/connectors/direct_scrapers/supjav.py @@ -0,0 +1,126 @@ +"""supjav.com — JAV browse scraper (przez Bright Data proxy, CF-blokuje VPS IP). + +Osobna pula JAV (sekcja JAV), origin `tube:supjav` w JAV_ORIGINS. supjav agreguje +embedy (serwery TV/FST/ST/VOE), gdzie realny hoster jest schowany za `data-link` +(reverse-hex → lk1.supremejav.com); rozwiązuje go ekstraktor `app/extractors/tubes/supjav.py`. + +Browse (RE 2026-07-10): + - listing: homepage `/` (najnowsze ~66, WordPress). Głębsza paginacja `/page/N/` leci + CF 403 nawet przez proxy → browse ogranicza się do strony 1 (scheduler odświeża). + - karty: `
`. + - Metadane bierzemy Z KARTY (tytuł/thumb/URL/kod JAV/data z roku-miesiąca ścieżki + thumb), NIE z detalu — detal przez proxy jest flaky (CF przepuszcza ~40% IP), a + per-post fetch byłby wolny i lossy. Studio/tagi (poza kodem) dorabia enrich później. + +supjav blokuje VPS IP twardym CF → listing idzie przez Bright Data ISP proxy z retry +(`fetch_supjav_html`, rotacja IP aż trafi przepuszczający). Bez proxy scraper no-op. +""" +from __future__ import annotations + +import html as _html +import logging +import re +from datetime import date + +from app.config import get_settings +from app.connectors.base import RawPlaybackSource, RawScene, RawTag +from app.connectors.direct_scrapers._browse_base import BaseBrowseScraper +from app.extractors.tubes.supjav import fetch_supjav_html +from app.normalize.text import slugify + +log = logging.getLogger(__name__) + +_BASE = "https://supjav.com" +# Karta: link-obraz z href + title + thumb. title="([^"]*)" jest bezpieczne (wartość +# atrybutu nie zawiera surowego "), więc tytuły z `>` (np. `<<3-Day Offer>>`) przechodzą. +# Link-obraz karty: . +# supjav miesza kolejność atrybutów (href/rel/class="img"/title w dwóch wariantach), więc +# NIE wymagamy pozycji class="img" — wymóg `>\s*` ma tekst, nie ). Thumb eager `src=` lub lazy +# `data-original=` (karty poniżej folda: src="data:..." placeholder + realny data-original). +_CARD_RE = re.compile( + r'href="(https://supjav\.com/\d+\.html)"[^>]*?\btitle="([^"]*)"[^>]*?>\s*' + r']*?\b(?:src|data-original|data-src|data-lazy-src)="(https://img\.supjav\.com/[^"]+)"', + re.IGNORECASE, +) +_THUMB_DATE_RE = re.compile(r"/images/(\d{4})/(\d{2})/") +# Kod JAV: FC2PPV ma cyfrę w prefiksie (FC2PPV 4931572), studia mają myślnik (MGOLD-053, +# DLDSS-510, SNOS-258). Prefiks liter+cyfr, separator - lub spacja, numer. +_CODE_RE = re.compile(r"^([A-Z][A-Z0-9]{1,7}[-\s]\d{2,7})", re.IGNORECASE) + + +class SupjavScraper(BaseBrowseScraper): + sitetag = "supjav" + _timeout = 70.0 + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + # CF-blok VPS IP → listing przez Bright Data proxy (jak superporn). + self._proxy = get_settings().brightdata_proxy_url + if not self._proxy: + log.warning("supjav: BRIGHTDATA_PROXY_URL unset — scraper disabled") + + def _listing_url(self, page: int) -> str: + return f"{_BASE}/" if page <= 1 else f"{_BASE}/page/{page}/" + + def crawl_page(self, page: int) -> list[RawScene] | None: + if not self._proxy: + return [] # bez proxy gwarantowany CF 403 (exhausted) + html = fetch_supjav_html(self._listing_url(page), proxy=self._proxy, timeout=self._timeout) + if not html or len(html) < 15000: + # sam challenge / fetch fail. page 1 = transient (None, retry potem); + # page > 1 = najpewniej CF-403 głębszej paginacji = koniec (exhausted). + return None if page <= 1 else [] + seen: set[str] = set() + out: list[RawScene] = [] + for m in _CARD_RE.finditer(html): + url, title_raw, thumb_raw = m.group(1), m.group(2), m.group(3) + if url in seen: + continue + seen.add(url) + raw = self._card_to_scene(url, title_raw, thumb_raw) + if raw is not None: + out.append(raw) + return out + + def _card_to_scene(self, url: str, title_raw: str, thumb_raw: str) -> RawScene | None: + title = _html.unescape(title_raw).strip() + if not title: + return None + thumb = thumb_raw.split("!")[0] or None # utnij `!320x216.jpg` → pełna rozdz. + release_date: date | None = None + dm = _THUMB_DATE_RE.search(thumb_raw) + if dm: + try: + release_date = date(int(dm.group(1)), int(dm.group(2)), 1) + except ValueError: + release_date = None + + tags: list[RawTag] = [] + cm = _CODE_RE.match(title) + if cm: + code = re.sub(r"[ ]", "-", cm.group(1).upper()) + tags.append(RawTag(external_id=f"javcode:{code}", name=code, slug=slugify(code))) + + return RawScene( + external_id=f"{self.sitetag}:{url}", + title=title, + release_date=release_date, + url=url, + tags=tags, + playback_sources=[ + RawPlaybackSource( + origin=f"tube:{self.sitetag}", + page_url=url, + thumbnail_url=thumb, + ) + ], + ) + + # HTML detail nieużywany (browse z kart), ale ABC wymaga. + def _extract_scene_urls(self, listing_html: str) -> list[str]: # pragma: no cover + return [] + + def _parse_detail(self, scene_url: str, detail_html: str) -> RawScene | None: # pragma: no cover + return None diff --git a/app/extractors/__init__.py b/app/extractors/__init__.py index 14d8e63..6fa7e69 100644 --- a/app/extractors/__init__.py +++ b/app/extractors/__init__.py @@ -39,6 +39,7 @@ from app.extractors.tubes import ( paradisehill, porn00, porntrex, + supjav, sxyprn, vjav, watchporn, @@ -179,6 +180,8 @@ _REGISTRY: dict[str, Callable[[str], list[StreamSource] | None]] = { "javflix": javflix.extract, # vjav (JAV, TXXX network) — videofile.php -> get_file HLS (patrz tubes/vjav.py). "vjav": vjav.extract, + # supjav (JAV, embed-aggregator) — data-link reverse-hex -> lk1 -> hoster (type=hoster). + "supjav": supjav.extract, # neporn — KVS function/0 + license (jak freshporno). Server-side _kvs resolve → # data001.neporn.com/remote_control.php portable (cross-IP 206, 2026-06-10). "neporncom": neporn.extract, diff --git a/app/extractors/tubes/supjav.py b/app/extractors/tubes/supjav.py new file mode 100644 index 0000000..bace464 --- /dev/null +++ b/app/extractors/tubes/supjav.py @@ -0,0 +1,112 @@ +"""supjav.com — JAV embed-aggregator. Resolve data-link → hoster (phone-side). + +supjav chowa realny hoster za `data-link` (hex) na przyciskach serwerów (TV/FST/ST/VOE). +base.js: klik serwera → iframe `src = lk1.supremejav.com/supjav.php?l=&bg=`. +supjav.php robi `OLID = data-link.reverse()` (odwrócenie stringa hex) i ładuje +`supjav.php?c=`, które 302-uje na realny hoster (RE 2026-07-10): + TV → turbovidhls.com, FST → fc2stream.tv, ST → streamtape.com, VOE → voe.sx + +Flow ekstraktora (on-demand, play time): + 1. fetch detail `/.html` PRZEZ proxy (Bright Data — supjav CF-blokuje VPS IP), + 2. parse `data-link` z `.btn-server`, + 3. reverse hex → GET `lk1.supremejav.com/supjav.php?c=` (direct z VPS, follow 302) + → finalny hoster URL, + 4. zwróć type='hoster' → telefon resolwuje (dood/filemoon natywnie, reszta WebView + fallback z residential IP). lk1 osiągalny z VPS bez proxy; tylko detail wymaga proxy. +""" +from __future__ import annotations + +import logging +import re + +import curl_cffi.requests as _rq + +from app.config import get_settings +from app.extractors import browser_get +from app.extractors._fetch import _DEFAULT_IMPERSONATE +from app.extractors._models import StreamSource + +log = logging.getLogger(__name__) + +_LK = "https://lk1.supremejav.com/supjav.php" +_DATA_LINK_RE = re.compile(r'class="btn-server[^"]*"\s+data-link="([0-9a-f]{16,})"', re.IGNORECASE) +# reklama/tracker domeny które lk1 czasem zwraca zamiast hostera — odrzucamy. +_AD_RE = re.compile(r"(snaptrckr|trackwilltrk|mayzaent|eix304|doppiocdn|/ad\?)", re.IGNORECASE) + + +def fetch_supjav_html(url: str, *, proxy: str | None, timeout: float, tries: int = 6) -> str: + """Fetch supjav przez proxy z retry. Bright Data rotuje IP per-request, a CF + przepuszcza tylko część IP (reszta dostaje ~6KB challenge). Retry aż trafimy IP + który przechodzi (realna strona ma `data-link=`/`

` i jest duża). Zwraca '' gdy + wszystkie próby padły.""" + last = "" + for _ in range(max(1, tries)): + try: + res = browser_get(url, timeout=timeout, proxy=proxy) + html = res.text if hasattr(res, "text") else res + except Exception: + html = "" + last = html or last + if html and len(html) > 15000 and ("data-link=" in html or "

" in html): + return html + return last + + +def _resolve_hoster(data_link: str, timeout: float) -> str | None: + """reverse hex → lk1 supjav.php?c= → follow 302 → finalny hoster URL.""" + olid = data_link[::-1] + try: + s = _rq.Session(impersonate=_DEFAULT_IMPERSONATE) + r = s.get( + f"{_LK}?c={olid}", + headers={"Referer": "https://lk1.supremejav.com/"}, + allow_redirects=True, + timeout=timeout, + ) + final = str(r.url) + r.close() + except Exception as e: + log.info("supjav: lk1 resolve failed (%s): %s", data_link[:12], e) + return None + # supjav.php bez realnego hostera zostaje na lk1 (albo leci na ad) → odrzuć. + if "supremejav.com" in final or _AD_RE.search(final): + return None + # lk1 dokleja fragment `#supjav.com@` (metadata) — hostery go ignorują, tniemy. + return final.split("#", 1)[0] + + +def extract(page_url: str, *, timeout: float = 60.0) -> list[StreamSource] | None: + proxy = get_settings().brightdata_proxy_url + if not proxy: + log.info("supjav: brak proxy — nie mogę pobrać CF-blokowanego detalu %s", page_url) + return None + html = fetch_supjav_html(page_url, proxy=proxy, timeout=timeout) + if not html: + log.info("supjav: detail fetch failed (CF) %s", page_url) + return None + + data_links = list(dict.fromkeys(_DATA_LINK_RE.findall(html))) + if not data_links: + log.info("supjav: brak data-link na %s", page_url) + return None + + seen: set[str] = set() + out: list[StreamSource] = [] + for dl in data_links: + hoster = _resolve_hoster(dl, timeout=min(timeout, 40.0)) + if not hoster or hoster in seen: + continue + seen.add(hoster) + host = hoster.split("/")[2] if "://" in hoster else hoster + out.append( + StreamSource( + link=hoster, + type="hoster", + quality=host, + referer="https://supjav.com/", + ) + ) + if not out: + log.info("supjav: żaden data-link nie rozwiązał się na hoster (%s)", page_url) + return None + return out