feat(pornbusy): browse scraper + loadvid POST-manifest producer
pornbusy.com exposes every field as a discrete itemprop node (title, duration, uploadDate, thumbnail, description) plus a dedicated <div id="video-actors"> cast block with no sidebar pollution. Measured orphan risk is low: 80% of a 100-scene sample have a performer that already carries a tpdb/stashdb ref, and 21% of titles strongly match scenes we already hold. No studio field exists on the site. Homepage pagination is broken (pages 1/2/3 return an identical set), so the listing is driven off sitemap_index.xml -> 10 post-sitemaps sorted by lastmod, chunked in crawl_page the same way the PlayTube base does it. Playback dispatches on embedURL and covers ~69% of the catalog: the seekplayer family (already handled by the engine after the earlier host regex widening) and zpi.cx (the embedURL is the file itself), plus loadvid at ~41%, which needed new plumbing. loadvid hands back the CONTENT of an m3u8 over POST /videos/resolve-token (CSRF + videoToken from the embed page) and has no manifest URL at all: GET on that endpoint is 405 and the guessable .m3u8 paths are 404. So make_token grew an optional `producer` marker and /proxy/hls calls the producer instead of GETting a URL. Segments in the returned manifest are absolute and fully portable (verified: 206 on a Range request with no headers, no referer, no token), so the phone still pulls them straight from the CDN and only the manifest travels through the VPS. upload18 (~12%) and the tail are deliberately left unresolved: their token embeds the fetcher's /24, so resolving server-side would force the whole video through the VPS. Verified: 19 scenes/page, 19/19 with duration, 17/19 with cast, 19/19 with tags (names read from the title attribute since an icon element precedes the anchor text), playback 8/8 via loadvid, and /proxy/hls returning a 116KB manifest with 869 absolute segments. Pilot ingest of 3 pages: 59 seen, 35 merged into existing scenes, 24 new, 0 errors. Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
goon-foss
parent
2bf9179467
commit
d716b0c75d
7 changed files with 501 additions and 0 deletions
|
|
@ -476,9 +476,13 @@ def _proxify_link(link: StreamLink, referer: str) -> StreamLink:
|
|||
mobile_direct_ok = True
|
||||
refetch_url = (link.raw or {}).get("refetch_url")
|
||||
refetch_hoster = (link.raw or {}).get("refetch_hoster")
|
||||
# manifest_producer: hoster oddaje TREŚĆ manifestu (POST), nie URL — `/proxy/hls`
|
||||
# zawoła producenta zamiast GET-a. Patrz extractors/hosters/loadvid.py.
|
||||
manifest_producer = (link.raw or {}).get("manifest_producer")
|
||||
token = make_token(
|
||||
raw_url, referer, impersonate=use_impersonate,
|
||||
refresh=refetch_url, refresh_hoster=refetch_hoster,
|
||||
producer=manifest_producer,
|
||||
)
|
||||
# Decyzja na BASIE link.type (zaufanie do extractora), z fallback path-hint.
|
||||
# Pornhat: raw URL `.../get_file/.../<id>.mp4/` ale CDN 302 → HLS manifest.
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ oglądać dłuższe sceny + pause/seek bez ryzyka expired token.
|
|||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import functools
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
|
|
@ -163,6 +164,7 @@ def make_token(
|
|||
refresh_hoster: str | None = None,
|
||||
impersonate: bool = False,
|
||||
stable_bucket_sec: int | None = None,
|
||||
producer: str | None = None,
|
||||
) -> str:
|
||||
"""Build proxy token.
|
||||
|
||||
|
|
@ -187,6 +189,10 @@ def make_token(
|
|||
payload["rh"] = refresh_hoster
|
||||
if impersonate:
|
||||
payload["i"] = 1
|
||||
if producer:
|
||||
# Hoster oddający TREŚĆ manifestu (nie URL) — `/proxy/hls` zawoła producenta
|
||||
# zamiast GET-a. Patrz app/extractors/hosters/loadvid.py.
|
||||
payload["mp"] = producer
|
||||
raw = json.dumps(payload, separators=(",", ":")).encode("utf-8")
|
||||
body = base64.urlsafe_b64encode(raw).rstrip(b"=").decode("ascii")
|
||||
sig = base64.urlsafe_b64encode(
|
||||
|
|
@ -586,6 +592,24 @@ async def _curl_cffi_stream(
|
|||
raise HTTPException(status_code=502, detail=f"proxy error: {e}") from e
|
||||
|
||||
|
||||
async def _produce_manifest(producer: str, embed_url: str) -> str | None:
|
||||
"""Zawołaj hoster-producenta manifestu w threadpoolu (curl_cffi jest sync)."""
|
||||
import anyio
|
||||
|
||||
if producer == "loadvid":
|
||||
from app.extractors.hosters import loadvid
|
||||
|
||||
try:
|
||||
return await anyio.to_thread.run_sync(
|
||||
functools.partial(loadvid.fetch_manifest, embed_url, timeout=25.0)
|
||||
)
|
||||
except Exception as e:
|
||||
log.info("proxy-hls producer loadvid failed %s: %s", embed_url, e)
|
||||
return None
|
||||
log.warning("proxy-hls: unknown manifest producer %r", producer)
|
||||
return None
|
||||
|
||||
|
||||
@router.get("/hls/{token}/{_basename:path}")
|
||||
async def proxy_hls_manifest(token: str, _basename: str) -> Response:
|
||||
"""Direct-HLS manifest passthrough dla time-bound (mobile_direct_ok) m3u8 hosterów.
|
||||
|
|
@ -605,6 +629,20 @@ async def proxy_hls_manifest(token: str, _basename: str) -> Response:
|
|||
payload = parse_token(token)
|
||||
target = payload["u"]
|
||||
referer = payload.get("r") or None
|
||||
|
||||
# Hoster-producent: manifest nie ma własnego URL-a (loadvid oddaje jego TREŚĆ
|
||||
# dopiero na POST /videos/resolve-token), więc zamiast GET-a wołamy producenta.
|
||||
producer = payload.get("mp")
|
||||
if producer:
|
||||
manifest = await _produce_manifest(producer, target)
|
||||
if manifest is None:
|
||||
raise HTTPException(status_code=502, detail="manifest producer failed")
|
||||
return Response(
|
||||
content=_absolutize_m3u8(manifest, base_url=target),
|
||||
media_type="application/vnd.apple.mpegurl",
|
||||
headers={"Cache-Control": "no-store"},
|
||||
)
|
||||
|
||||
headers = _build_headers(referer)
|
||||
async with httpx.AsyncClient(follow_redirects=True, timeout=20.0) as client:
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -44,6 +44,7 @@ from app.connectors.direct_scrapers.siska import SiskaScraper
|
|||
from app.connectors.direct_scrapers.sxyland import SxyLandScraper
|
||||
from app.connectors.direct_scrapers.sxyprn import SxyPrnScraper
|
||||
from app.connectors.direct_scrapers.galaxyporn import GalaxyPornScraper
|
||||
from app.connectors.direct_scrapers.pornbusy import PornBusyScraper
|
||||
from app.connectors.direct_scrapers.watchporn import WatchPornScraper
|
||||
from app.connectors.direct_scrapers.youperv import YoupervScraper
|
||||
from app.connectors.direct_scrapers.xhamster import XHamsterScraper
|
||||
|
|
@ -165,6 +166,11 @@ ALL_BROWSE_SCRAPERS: list[type[BaseBrowseScraper]] = [
|
|||
# w HTML, a NULL ukryłby sceny pod filtrem min_duration_sec). Playback: iframe
|
||||
# seekplayer → istniejący silnik, HLS przez /proxy/hls.
|
||||
GalaxyPornScraper,
|
||||
# PornBusyScraper — dodany 2026-07-27 (ocena 4.5/5). Metadane w `itemprop` (duration/
|
||||
# uploadDate/thumb/desc) + obsada w `<div id="video-actors">` bez pollution. Homepage
|
||||
# pagination ZEPSUTA (str. 1/2/3 identyczne) → listing z sitemapy, newest-first.
|
||||
# Brak pola studio. Playback: loadvid/seekplayer/zpi ≈ 69% katalogu.
|
||||
PornBusyScraper,
|
||||
# Browse równolegle do istniejącego search scrapera (wzorzec xvideos/eporner):
|
||||
# search zostaje (pokrycie back-catalogu performerów), browse gwarantuje świeżość
|
||||
# wprost z feedu (watchdog 48h zamiast 168h). Konwersja 2026-06-24 (user request).
|
||||
|
|
|
|||
238
app/connectors/direct_scrapers/pornbusy.py
Normal file
238
app/connectors/direct_scrapers/pornbusy.py
Normal file
|
|
@ -0,0 +1,238 @@
|
|||
"""pornbusy.com — browse scraper (WordPress + Rank Math). Dodany 2026-07-27.
|
||||
|
||||
Ripy studyjne (~60%) + JAV z kodami (~30%). Ocena orphan-risk LOW z pomiarem:
|
||||
80% scen z próbki 100 ma performera, który u nas MA już ref tpdb/stashdb, a 21%
|
||||
tytułów mocno matchuje sceny które już mamy (czyli się scalą, nie zorphanują).
|
||||
|
||||
**Paginacja homepage jest ZEPSUTA** — `/page/2/` i `/page/3/` zwracają ten sam
|
||||
zestaw co strona 1 (zweryfikowane). Dlatego listing bierzemy z SITEMAPY:
|
||||
`sitemap_index.xml` → 10× `post-sitemapN.xml` po ~500 URL-i, każdy z `<lastmod>`,
|
||||
posortowane od najnowszych. `crawl_page` tnie ten katalog na strony po `_PAGE_SIZE`
|
||||
(wzorzec jak `_playtube.BasePlayTubeScraper`).
|
||||
|
||||
Metadane — każde pole to osobny węzeł `itemprop`, nic nie trzeba wyłuskiwać z
|
||||
tytułu-slug:
|
||||
- title `<h1 class="entry-title">`, duration ISO, uploadDate, thumbnailUrl, description
|
||||
- obsada: `<div id="video-actors">` — CZYSTA, bez pollution (na stronie sceny jest
|
||||
dokładnie tyle linków `/actor/` ilu realnych aktorów; brak sekcji Related)
|
||||
- kategorie + tagi: `<div class="tags-list">` (`/category/` i `/tag/`)
|
||||
- studio: **brak** — pornbusy nie ma pola studia ani prefiksu w tytule
|
||||
|
||||
Playback: `itemprop="embedURL"` → extractor `pornbusycom` (loadvid / seekplayer /
|
||||
zpi.cx ≈ 69% katalogu).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import logging
|
||||
import re
|
||||
|
||||
from app.connectors.base import (
|
||||
RawPerformer,
|
||||
RawPlaybackSource,
|
||||
RawScene,
|
||||
RawTag,
|
||||
)
|
||||
from app.connectors.direct_scrapers._browse_base import BaseBrowseScraper
|
||||
from app.connectors.direct_scrapers._playtube import _parse_iso_date
|
||||
from app.extractors import browser_get
|
||||
from app.normalize.text import slugify
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
_BASE = "https://pornbusy.com"
|
||||
_PAGE_SIZE = 20
|
||||
|
||||
_SITEMAP_INDEX = f"{_BASE}/sitemap_index.xml"
|
||||
_LOC_RE = re.compile(r"<loc>\s*([^<]+?)\s*</loc>")
|
||||
_URL_BLOCK_RE = re.compile(r"<url>(.*?)</url>", re.DOTALL | re.IGNORECASE)
|
||||
_LASTMOD_RE = re.compile(r"<lastmod>\s*([^<]+?)\s*</lastmod>")
|
||||
|
||||
_TITLE_RE = re.compile(r'<h1 class="entry-title"[^>]*>(.*?)</h1>', re.IGNORECASE | re.DOTALL)
|
||||
_DUR_RE = re.compile(r'itemprop="duration"\s+content="([^"]+)"', re.IGNORECASE)
|
||||
_DATE_RE = re.compile(r'itemprop="uploadDate"\s+content="([^"]+)"', re.IGNORECASE)
|
||||
_THUMB_RE = re.compile(r'itemprop="thumbnailUrl"\s+content="([^"]+)"', re.IGNORECASE)
|
||||
_DESC_RE = re.compile(r'itemprop="description"\s+content="([^"]*)"', re.IGNORECASE)
|
||||
_ACTORS_BLOCK_RE = re.compile(r'<div id="video-actors">(.*?)</div>', re.IGNORECASE | re.DOTALL)
|
||||
_ACTOR_RE = re.compile(r'/actor/[^/"]+/"[^>]*title="([^"]+)"', re.IGNORECASE)
|
||||
_ACTOR_TEXT_RE = re.compile(r">([^<>]+)</a>")
|
||||
_TAGS_BLOCK_RE = re.compile(r'<div class="tags-list">(.*?)</div>', re.IGNORECASE | re.DOTALL)
|
||||
# Nazwa z atrybutu `title`, NIE z tekstu anchora: tekst poprzedza zagnieżdżona ikona
|
||||
# (`<a ... title="Blonde"><i class="fa fa-tag"></i>Blonde</a>`), więc `>([^<]+)</a>`
|
||||
# nie matchuje. `title` jest czysty i zawsze obecny.
|
||||
_TAXO_RE = re.compile(
|
||||
r'/(?:category|tag)/([a-z0-9\-]+)/"[^>]*title="([^"]+)"', re.IGNORECASE
|
||||
)
|
||||
# `P0DT0H42M52S`
|
||||
_ISO_DUR_RE = re.compile(
|
||||
r"P(?:(\d+)D)?T?(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?", re.IGNORECASE
|
||||
)
|
||||
|
||||
# Junk-performer guard: pornbusy wrzuca do /actor/ także ogólniki.
|
||||
_JUNK_ACTORS = frozenset({"amateur", "unknown", "anonymous", "n-a", "na", "various"})
|
||||
|
||||
|
||||
def _parse_iso_duration(value: str | None) -> int | None:
|
||||
if not value:
|
||||
return None
|
||||
m = _ISO_DUR_RE.match(value.strip())
|
||||
if not m:
|
||||
return None
|
||||
d, h, mn, s = (int(g or 0) for g in m.groups())
|
||||
total = d * 86400 + h * 3600 + mn * 60 + s
|
||||
return total or None
|
||||
|
||||
|
||||
def _clean(raw: str) -> str:
|
||||
return html.unescape(re.sub(r"\s+", " ", re.sub(r"<[^>]+>", " ", raw))).strip()
|
||||
|
||||
|
||||
class PornBusyScraper(BaseBrowseScraper):
|
||||
sitetag = "pornbusycom"
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
# Katalog z sitemapy, newest-first. Lazy raz per instancję (browse_latest i
|
||||
# deep_crawl tworzą instancję per run, więc 10 fetchy XML amortyzuje się).
|
||||
self._catalog: list[str] | None = None
|
||||
|
||||
# Listing nie jest stronicowalny GET-em (homepage pagination zepsuta) —
|
||||
# paginację robi sitemap w `crawl_page`. Te dwie metody są abstrakcyjne w bazie.
|
||||
def _listing_url(self, page: int) -> str: # pragma: no cover - nieużywane
|
||||
return _SITEMAP_INDEX
|
||||
|
||||
def _extract_scene_urls(self, listing_html: str) -> list[str]: # pragma: no cover
|
||||
return []
|
||||
|
||||
def _load_catalog(self) -> list[str] | None:
|
||||
if self._catalog is not None:
|
||||
return self._catalog
|
||||
try:
|
||||
idx = browser_get(_SITEMAP_INDEX, timeout=self._timeout)
|
||||
idx.raise_for_status()
|
||||
except Exception as e:
|
||||
log.warning("pornbusy: sitemap index fetch failed: %s", e)
|
||||
return None
|
||||
maps = [u for u in _LOC_RE.findall(idx.text) if "post-sitemap" in u]
|
||||
if not maps:
|
||||
log.warning("pornbusy: no post-sitemaps in index")
|
||||
return None
|
||||
|
||||
entries: list[tuple[str, str]] = []
|
||||
for sm_url in maps:
|
||||
try:
|
||||
sm = browser_get(sm_url, timeout=self._timeout)
|
||||
sm.raise_for_status()
|
||||
except Exception as e:
|
||||
log.warning("pornbusy: sitemap fetch failed %s: %s", sm_url, e)
|
||||
continue
|
||||
for block in _URL_BLOCK_RE.findall(sm.text):
|
||||
loc = _LOC_RE.search(block)
|
||||
if not loc:
|
||||
continue
|
||||
url = loc.group(1)
|
||||
# Pomijamy strony taksonomii/nav — sceny to `/<slug>/` jednosegmentowe.
|
||||
if any(x in url for x in ("/actor/", "/category/", "/tag/", "/page/")):
|
||||
continue
|
||||
lm = _LASTMOD_RE.search(block)
|
||||
entries.append((lm.group(1) if lm else "", url))
|
||||
if not entries:
|
||||
return None
|
||||
entries.sort(key=lambda e: e[0], reverse=True)
|
||||
seen: set[str] = set()
|
||||
catalog: list[str] = []
|
||||
for _, url in entries:
|
||||
if url in seen:
|
||||
continue
|
||||
seen.add(url)
|
||||
catalog.append(url)
|
||||
log.info("pornbusy: catalog loaded — %d scenes from %d sitemaps", len(catalog), len(maps))
|
||||
self._catalog = catalog
|
||||
return catalog
|
||||
|
||||
def crawl_page(self, page: int) -> list[RawScene] | None:
|
||||
catalog = self._load_catalog()
|
||||
if catalog is None:
|
||||
return None
|
||||
chunk = catalog[(page - 1) * _PAGE_SIZE: page * _PAGE_SIZE]
|
||||
if not chunk:
|
||||
return []
|
||||
out: list[RawScene] = []
|
||||
for scene_url in chunk:
|
||||
try:
|
||||
res = browser_get(scene_url, timeout=self._timeout)
|
||||
res.raise_for_status()
|
||||
except Exception as e:
|
||||
log.info("pornbusy detail fetch failed %s: %s", scene_url, e)
|
||||
continue
|
||||
try:
|
||||
raw = self._parse_detail(scene_url, res.text)
|
||||
except Exception as e:
|
||||
log.warning("pornbusy detail parse failed %s: %s", scene_url, e)
|
||||
continue
|
||||
if raw is not None:
|
||||
out.append(raw)
|
||||
return out
|
||||
|
||||
def _parse_detail(self, scene_url: str, detail_html: str) -> RawScene | None:
|
||||
tm = _TITLE_RE.search(detail_html)
|
||||
title = _clean(tm.group(1)) if tm else ""
|
||||
if not title:
|
||||
return None
|
||||
|
||||
dm = _DUR_RE.search(detail_html)
|
||||
duration_sec = _parse_iso_duration(dm.group(1)) if dm else None
|
||||
um = _DATE_RE.search(detail_html)
|
||||
release_date = _parse_iso_date(um.group(1)) if um else None
|
||||
thm = _THUMB_RE.search(detail_html)
|
||||
thumbnail_url = thm.group(1) if thm else None
|
||||
dsm = _DESC_RE.search(detail_html)
|
||||
description = html.unescape(dsm.group(1)).strip() if dsm else None
|
||||
if description and description.strip().lower() == title.strip().lower():
|
||||
description = None # opis bywa kopią tytułu — nie duplikujemy
|
||||
|
||||
performers: list[RawPerformer] = []
|
||||
seen_p: set[str] = set()
|
||||
ab = _ACTORS_BLOCK_RE.search(detail_html)
|
||||
if ab:
|
||||
names = _ACTOR_RE.findall(ab.group(1)) or _ACTOR_TEXT_RE.findall(ab.group(1))
|
||||
for name in names:
|
||||
name = html.unescape(name).strip()
|
||||
sl = slugify(name)
|
||||
if not sl or sl in seen_p or sl in _JUNK_ACTORS:
|
||||
continue
|
||||
seen_p.add(sl)
|
||||
performers.append(
|
||||
RawPerformer(external_id=f"{self.sitetag}:performer:{sl}", name=name)
|
||||
)
|
||||
|
||||
tags: list[RawTag] = []
|
||||
seen_t: set[str] = set()
|
||||
tb = _TAGS_BLOCK_RE.search(detail_html)
|
||||
if tb:
|
||||
for slug, name in _TAXO_RE.findall(tb.group(1)):
|
||||
name = html.unescape(name).strip()
|
||||
if not name or slug in seen_t or slug in seen_p:
|
||||
continue
|
||||
seen_t.add(slug)
|
||||
tags.append(RawTag(external_id=f"{self.sitetag}:tag:{slug}", name=name, slug=slug))
|
||||
|
||||
return RawScene(
|
||||
external_id=f"{self.sitetag}:{scene_url}",
|
||||
title=title,
|
||||
description=description,
|
||||
release_date=release_date,
|
||||
duration_sec=duration_sec,
|
||||
url=scene_url,
|
||||
# studio: pornbusy go nie ma (ani pole, ani prefiks tytułu)
|
||||
performers=performers,
|
||||
tags=tags,
|
||||
playback_sources=[
|
||||
RawPlaybackSource(
|
||||
origin=f"tube:{self.sitetag}",
|
||||
page_url=scene_url,
|
||||
duration_sec=duration_sec,
|
||||
thumbnail_url=thumbnail_url,
|
||||
)
|
||||
],
|
||||
)
|
||||
|
|
@ -39,6 +39,7 @@ from app.extractors.tubes import (
|
|||
latestpornvideo,
|
||||
paradisehill,
|
||||
porn00,
|
||||
pornbusy,
|
||||
porntrex,
|
||||
supjav,
|
||||
sxyprn,
|
||||
|
|
@ -112,6 +113,10 @@ _REGISTRY: dict[str, Callable[[str], list[StreamSource] | None]] = {
|
|||
# galaxyporn — iframe rodziny seekplayer (upns/4meplayer/seekplays); silnik już mamy,
|
||||
# extractor tylko wyłuskuje iframe. HLS Referer+IP-bound → /proxy/hls.
|
||||
"galaxyporncom": galaxyporn.extract,
|
||||
# pornbusy — dispatch po embedURL: loadvid (POST-manifest przez /proxy/hls),
|
||||
# rodzina seekplayer (istniejący silnik), zpi.cx (embedURL to gotowy plik).
|
||||
# upload18 + ogon → None (token IP-bound, resolve wymusiłby proxy całego wideo).
|
||||
"pornbusycom": pornbusy.extract,
|
||||
"siskavideo": _embed_iframe.extract,
|
||||
"porn4dayspw": _embed_iframe.extract,
|
||||
"porndishcom": _embed_iframe.extract,
|
||||
|
|
|
|||
128
app/extractors/hosters/loadvid.py
Normal file
128
app/extractors/hosters/loadvid.py
Normal file
|
|
@ -0,0 +1,128 @@
|
|||
"""cdn.loadvid.com — HLS hoster oddający manifest przez POST (nie URL).
|
||||
|
||||
Protokół (reverse-engineer 2026-07-27, potwierdzony live):
|
||||
1. GET /videos/play/<hash> → HTML z `<meta name="csrf-token">` + inline
|
||||
`window.LoadVidConfig = { videoHash, videoToken, ... }`
|
||||
2. POST /videos/resolve-token {token, hash} + nagłówek `X-CSRF-TOKEN`
|
||||
→ **treść manifestu m3u8** (200, `application/vnd.apple.mpegurl`, ~116 KB,
|
||||
~870 segmentów), a NIE URL do manifestu.
|
||||
|
||||
Dlatego to nie jest zwykły extractor "URL → URL": nie da się oddać linku do
|
||||
manifestu, bo taki link nie istnieje (GET na resolve-token → 405, zgadywane
|
||||
`/index.m3u8` → 404). Manifest musi wyprodukować backend.
|
||||
|
||||
Rozwiązanie: `extract()` zwraca **embed URL** oznaczony `manifest_producer=loadvid`.
|
||||
`/proxy/hls/<token>/play.m3u8` rozpoznaje ten marker i zamiast GET-a woła
|
||||
`fetch_manifest()`, po czym serwuje manifest telefonowi. Segmenty w manifeście są
|
||||
ABSOLUTNE i w pełni przenośne (zweryfikowane: 206 `bytes 0-1023` BEZ jakichkolwiek
|
||||
nagłówków, bez Referera, bez tokena — serwowane jako `image/png`, w środku TS),
|
||||
więc telefon ciągnie je bezpośrednio z CDN. Przez VPS idzie tylko manifest.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from app.extractors._fetch import _DEFAULT_IMPERSONATE, _DEFAULT_UA, _HAS_CURL_CFFI
|
||||
from app.extractors._models import HosterDead, StreamSource
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
_HOST_RE = re.compile(r"^(?:[a-z0-9]+\.)?loadvid\.com$", re.IGNORECASE)
|
||||
_CSRF_RE = re.compile(r'<meta name="csrf-token" content="([^"]+)"', re.IGNORECASE)
|
||||
_CONFIG_RE = re.compile(r"LoadVidConfig\s*=\s*(\{.*?\})\s*;", re.DOTALL)
|
||||
_TOKEN_RE = re.compile(r"""videoToken\s*:\s*['"]([^'"]+)""")
|
||||
_HASH_RE = re.compile(r"""videoHash\s*:\s*['"]([^'"]+)""")
|
||||
_RESOLVE_PATH = "/videos/resolve-token"
|
||||
|
||||
|
||||
def matches(url: str) -> bool:
|
||||
try:
|
||||
return bool(_HOST_RE.match(urlparse(url).hostname or ""))
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def fetch_manifest(embed_url: str, *, timeout: float = 30.0) -> str | None:
|
||||
"""Embed URL → treść manifestu m3u8 (albo None).
|
||||
|
||||
CSRF-token i cookie sesji muszą pochodzić z TEGO SAMEGO requestu co POST,
|
||||
dlatego jedna sesja curl_cffi na całą operację."""
|
||||
if not _HAS_CURL_CFFI:
|
||||
log.info("loadvid: curl_cffi unavailable")
|
||||
return None
|
||||
from curl_cffi import requests as cf
|
||||
|
||||
parsed = urlparse(embed_url)
|
||||
origin = f"{parsed.scheme}://{parsed.hostname}"
|
||||
session = cf.Session(impersonate=_DEFAULT_IMPERSONATE)
|
||||
try:
|
||||
r = session.get(
|
||||
embed_url,
|
||||
headers={"User-Agent": _DEFAULT_UA, "Accept": "text/html,application/xhtml+xml"},
|
||||
timeout=timeout,
|
||||
)
|
||||
except Exception as e:
|
||||
log.info("loadvid: embed fetch failed %s: %s", embed_url, e)
|
||||
return None
|
||||
if r.status_code in (404, 410):
|
||||
raise HosterDead(f"loadvid {embed_url}: HTTP {r.status_code}")
|
||||
if r.status_code != 200 or not r.text:
|
||||
log.info("loadvid: embed status=%s for %s", r.status_code, embed_url)
|
||||
return None
|
||||
|
||||
csrf = _CSRF_RE.search(r.text)
|
||||
cfg = _CONFIG_RE.search(r.text)
|
||||
if not csrf or not cfg:
|
||||
log.info("loadvid: no csrf/LoadVidConfig on %s", embed_url)
|
||||
return None
|
||||
tok = _TOKEN_RE.search(cfg.group(1))
|
||||
hsh = _HASH_RE.search(cfg.group(1))
|
||||
if not tok or not hsh:
|
||||
log.info("loadvid: no videoToken/videoHash on %s", embed_url)
|
||||
return None
|
||||
|
||||
try:
|
||||
pr = session.post(
|
||||
origin + _RESOLVE_PATH,
|
||||
headers={
|
||||
"User-Agent": _DEFAULT_UA,
|
||||
"X-CSRF-TOKEN": csrf.group(1),
|
||||
"X-Requested-With": "XMLHttpRequest",
|
||||
"Accept": "application/vnd.apple.mpegurl",
|
||||
"Referer": embed_url,
|
||||
},
|
||||
json={"token": tok.group(1), "hash": hsh.group(1)},
|
||||
timeout=timeout,
|
||||
)
|
||||
except Exception as e:
|
||||
log.info("loadvid: resolve-token failed %s: %s", embed_url, e)
|
||||
return None
|
||||
if pr.status_code != 200 or "#EXTM3U" not in pr.text:
|
||||
log.info("loadvid: resolve-token status=%s len=%d", pr.status_code, len(pr.text or ""))
|
||||
return None
|
||||
return pr.text
|
||||
|
||||
|
||||
def extract(page_url: str, *, timeout: float = 30.0) -> list[StreamSource] | None:
|
||||
"""Zwraca embed URL z markerem producenta — manifest powstaje dopiero w
|
||||
`/proxy/hls` (patrz docstring modułu). Weryfikujemy tu, że manifest realnie
|
||||
da się wyprodukować, żeby nie oddawać martwego źródła."""
|
||||
if not matches(page_url):
|
||||
return None
|
||||
manifest = fetch_manifest(page_url, timeout=timeout)
|
||||
if not manifest:
|
||||
return None
|
||||
return [
|
||||
StreamSource(
|
||||
link=page_url,
|
||||
type="m3u8",
|
||||
raw={
|
||||
# Segmenty absolutne i portable → telefon ciągnie je direct z CDN,
|
||||
# przez VPS leci tylko manifest.
|
||||
"mobile_direct_ok": True,
|
||||
"manifest_producer": "loadvid",
|
||||
},
|
||||
)
|
||||
]
|
||||
82
app/extractors/tubes/pornbusy.py
Normal file
82
app/extractors/tubes/pornbusy.py
Normal file
|
|
@ -0,0 +1,82 @@
|
|||
"""pornbusy.com — dispatch po hosterze z `itemprop="embedURL"`.
|
||||
|
||||
Katalog jest rozproszony po hosterach (próbka 120 scen): ~41% loadvid,
|
||||
~20% rodzina seekplayer (av.ezplayer.me / av.seeks.cloud / 4k.upn.one /
|
||||
4k.player4me.vip — objęte poszerzonym `_HOST_RE` silnika), ~8% zpi.cx,
|
||||
~12% upload18 (token z wbitym `i=<ip>/24` = IP-bound), reszta ogon.
|
||||
|
||||
Obsługujemy trzy pierwsze (~69% katalogu):
|
||||
- loadvid → manifest przez POST, marker `manifest_producer` (patrz hosters/loadvid.py)
|
||||
- seekplayer → istniejący silnik (ten sam AES key/IV)
|
||||
- zpi.cx → `embedURL` JEST plikiem wideo (mimo `.webm` w nazwie to mp4);
|
||||
zweryfikowane: Range 206 `video/mp4`, bez Referera, bez tokena
|
||||
|
||||
upload18 i ogon zwracamy jako None — resolve na VPS wymusiłby proxowanie CAŁEGO
|
||||
wideo (token IP-bound), co łamie zasadę no-video-proxy.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from app.extractors._fetch import fetch_tube_html
|
||||
from app.extractors._models import StreamSource
|
||||
from app.extractors.hosters import loadvid, seekplayer_engine
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
_BASE = "https://pornbusy.com"
|
||||
_EMBED_RE = re.compile(r'itemprop="embedURL"\s+content="([^"]+)"', re.IGNORECASE)
|
||||
_IFRAME_RE = re.compile(r'<iframe[^>]+src="([^"]+)"', re.IGNORECASE)
|
||||
_ZPI_HOST_RE = re.compile(r"^(?:[a-z0-9]+\.)?zpi\.cx$", re.IGNORECASE)
|
||||
|
||||
|
||||
def extract(page_url: str, *, timeout: float = 60.0) -> list[StreamSource] | None:
|
||||
html_text = fetch_tube_html(page_url, timeout=timeout)
|
||||
m = _EMBED_RE.search(html_text) or _IFRAME_RE.search(html_text)
|
||||
if not m:
|
||||
log.info("pornbusy: no embedURL/iframe on %s", page_url)
|
||||
return None
|
||||
embed = m.group(1).strip()
|
||||
if embed.startswith("//"):
|
||||
embed = "https:" + embed
|
||||
|
||||
if loadvid.matches(embed):
|
||||
return loadvid.extract(embed, timeout=timeout)
|
||||
|
||||
if seekplayer_engine.matches(embed):
|
||||
sources = seekplayer_engine.extract(embed, timeout=timeout)
|
||||
if not sources:
|
||||
return None
|
||||
player_origin = f"https://{urlparse(embed).hostname}/"
|
||||
out: list[StreamSource] = []
|
||||
for s in sources:
|
||||
raw = dict(s.raw or {})
|
||||
raw["proxy_no_verify"] = True
|
||||
out.append(
|
||||
StreamSource(
|
||||
link=s.link,
|
||||
type=s.type or "m3u8",
|
||||
quality=s.quality,
|
||||
# Referer = origin PLAYERA, nie strony (jak przy galaxyporn:
|
||||
# z Refererem strony CDN zwraca 403).
|
||||
referer=s.referer or player_origin,
|
||||
raw=raw,
|
||||
)
|
||||
)
|
||||
return out
|
||||
|
||||
host = (urlparse(embed).hostname or "").lower()
|
||||
if _ZPI_HOST_RE.match(host):
|
||||
# embedURL to bezpośrednio plik (rozszerzenie `.webm` myli — to mp4).
|
||||
return [
|
||||
StreamSource(
|
||||
link=embed,
|
||||
type="mp4",
|
||||
raw={"mobile_direct_ok": True},
|
||||
)
|
||||
]
|
||||
|
||||
log.info("pornbusy: unsupported hoster %s (%s)", host, page_url)
|
||||
return None
|
||||
Loading…
Add table
Reference in a new issue