feat(pornbusy): browse scraper + loadvid POST-manifest producer
pornbusy.com exposes every field as a discrete itemprop node (title, duration, uploadDate, thumbnail, description) plus a dedicated <div id="video-actors"> cast block with no sidebar pollution. Measured orphan risk is low: 80% of a 100-scene sample have a performer that already carries a tpdb/stashdb ref, and 21% of titles strongly match scenes we already hold. No studio field exists on the site. Homepage pagination is broken (pages 1/2/3 return an identical set), so the listing is driven off sitemap_index.xml -> 10 post-sitemaps sorted by lastmod, chunked in crawl_page the same way the PlayTube base does it. Playback dispatches on embedURL and covers ~69% of the catalog: the seekplayer family (already handled by the engine after the earlier host regex widening) and zpi.cx (the embedURL is the file itself), plus loadvid at ~41%, which needed new plumbing. loadvid hands back the CONTENT of an m3u8 over POST /videos/resolve-token (CSRF + videoToken from the embed page) and has no manifest URL at all: GET on that endpoint is 405 and the guessable .m3u8 paths are 404. So make_token grew an optional `producer` marker and /proxy/hls calls the producer instead of GETting a URL. Segments in the returned manifest are absolute and fully portable (verified: 206 on a Range request with no headers, no referer, no token), so the phone still pulls them straight from the CDN and only the manifest travels through the VPS. upload18 (~12%) and the tail are deliberately left unresolved: their token embeds the fetcher's /24, so resolving server-side would force the whole video through the VPS. Verified: 19 scenes/page, 19/19 with duration, 17/19 with cast, 19/19 with tags (names read from the title attribute since an icon element precedes the anchor text), playback 8/8 via loadvid, and /proxy/hls returning a 116KB manifest with 869 absolute segments. Pilot ingest of 3 pages: 59 seen, 35 merged into existing scenes, 24 new, 0 errors. Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
parent
2bf9179467
commit
d716b0c75d
7 changed files with 501 additions and 0 deletions
|
|
@ -476,9 +476,13 @@ def _proxify_link(link: StreamLink, referer: str) -> StreamLink:
|
||||||
mobile_direct_ok = True
|
mobile_direct_ok = True
|
||||||
refetch_url = (link.raw or {}).get("refetch_url")
|
refetch_url = (link.raw or {}).get("refetch_url")
|
||||||
refetch_hoster = (link.raw or {}).get("refetch_hoster")
|
refetch_hoster = (link.raw or {}).get("refetch_hoster")
|
||||||
|
# manifest_producer: hoster oddaje TREŚĆ manifestu (POST), nie URL — `/proxy/hls`
|
||||||
|
# zawoła producenta zamiast GET-a. Patrz extractors/hosters/loadvid.py.
|
||||||
|
manifest_producer = (link.raw or {}).get("manifest_producer")
|
||||||
token = make_token(
|
token = make_token(
|
||||||
raw_url, referer, impersonate=use_impersonate,
|
raw_url, referer, impersonate=use_impersonate,
|
||||||
refresh=refetch_url, refresh_hoster=refetch_hoster,
|
refresh=refetch_url, refresh_hoster=refetch_hoster,
|
||||||
|
producer=manifest_producer,
|
||||||
)
|
)
|
||||||
# Decyzja na BASIE link.type (zaufanie do extractora), z fallback path-hint.
|
# Decyzja na BASIE link.type (zaufanie do extractora), z fallback path-hint.
|
||||||
# Pornhat: raw URL `.../get_file/.../<id>.mp4/` ale CDN 302 → HLS manifest.
|
# Pornhat: raw URL `.../get_file/.../<id>.mp4/` ale CDN 302 → HLS manifest.
|
||||||
|
|
|
||||||
|
|
@ -20,6 +20,7 @@ oglądać dłuższe sceny + pause/seek bez ryzyka expired token.
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import base64
|
import base64
|
||||||
|
import functools
|
||||||
import hashlib
|
import hashlib
|
||||||
import hmac
|
import hmac
|
||||||
import json
|
import json
|
||||||
|
|
@ -163,6 +164,7 @@ def make_token(
|
||||||
refresh_hoster: str | None = None,
|
refresh_hoster: str | None = None,
|
||||||
impersonate: bool = False,
|
impersonate: bool = False,
|
||||||
stable_bucket_sec: int | None = None,
|
stable_bucket_sec: int | None = None,
|
||||||
|
producer: str | None = None,
|
||||||
) -> str:
|
) -> str:
|
||||||
"""Build proxy token.
|
"""Build proxy token.
|
||||||
|
|
||||||
|
|
@ -187,6 +189,10 @@ def make_token(
|
||||||
payload["rh"] = refresh_hoster
|
payload["rh"] = refresh_hoster
|
||||||
if impersonate:
|
if impersonate:
|
||||||
payload["i"] = 1
|
payload["i"] = 1
|
||||||
|
if producer:
|
||||||
|
# Hoster oddający TREŚĆ manifestu (nie URL) — `/proxy/hls` zawoła producenta
|
||||||
|
# zamiast GET-a. Patrz app/extractors/hosters/loadvid.py.
|
||||||
|
payload["mp"] = producer
|
||||||
raw = json.dumps(payload, separators=(",", ":")).encode("utf-8")
|
raw = json.dumps(payload, separators=(",", ":")).encode("utf-8")
|
||||||
body = base64.urlsafe_b64encode(raw).rstrip(b"=").decode("ascii")
|
body = base64.urlsafe_b64encode(raw).rstrip(b"=").decode("ascii")
|
||||||
sig = base64.urlsafe_b64encode(
|
sig = base64.urlsafe_b64encode(
|
||||||
|
|
@ -586,6 +592,24 @@ async def _curl_cffi_stream(
|
||||||
raise HTTPException(status_code=502, detail=f"proxy error: {e}") from e
|
raise HTTPException(status_code=502, detail=f"proxy error: {e}") from e
|
||||||
|
|
||||||
|
|
||||||
|
async def _produce_manifest(producer: str, embed_url: str) -> str | None:
|
||||||
|
"""Zawołaj hoster-producenta manifestu w threadpoolu (curl_cffi jest sync)."""
|
||||||
|
import anyio
|
||||||
|
|
||||||
|
if producer == "loadvid":
|
||||||
|
from app.extractors.hosters import loadvid
|
||||||
|
|
||||||
|
try:
|
||||||
|
return await anyio.to_thread.run_sync(
|
||||||
|
functools.partial(loadvid.fetch_manifest, embed_url, timeout=25.0)
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
log.info("proxy-hls producer loadvid failed %s: %s", embed_url, e)
|
||||||
|
return None
|
||||||
|
log.warning("proxy-hls: unknown manifest producer %r", producer)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
@router.get("/hls/{token}/{_basename:path}")
|
@router.get("/hls/{token}/{_basename:path}")
|
||||||
async def proxy_hls_manifest(token: str, _basename: str) -> Response:
|
async def proxy_hls_manifest(token: str, _basename: str) -> Response:
|
||||||
"""Direct-HLS manifest passthrough dla time-bound (mobile_direct_ok) m3u8 hosterów.
|
"""Direct-HLS manifest passthrough dla time-bound (mobile_direct_ok) m3u8 hosterów.
|
||||||
|
|
@ -605,6 +629,20 @@ async def proxy_hls_manifest(token: str, _basename: str) -> Response:
|
||||||
payload = parse_token(token)
|
payload = parse_token(token)
|
||||||
target = payload["u"]
|
target = payload["u"]
|
||||||
referer = payload.get("r") or None
|
referer = payload.get("r") or None
|
||||||
|
|
||||||
|
# Hoster-producent: manifest nie ma własnego URL-a (loadvid oddaje jego TREŚĆ
|
||||||
|
# dopiero na POST /videos/resolve-token), więc zamiast GET-a wołamy producenta.
|
||||||
|
producer = payload.get("mp")
|
||||||
|
if producer:
|
||||||
|
manifest = await _produce_manifest(producer, target)
|
||||||
|
if manifest is None:
|
||||||
|
raise HTTPException(status_code=502, detail="manifest producer failed")
|
||||||
|
return Response(
|
||||||
|
content=_absolutize_m3u8(manifest, base_url=target),
|
||||||
|
media_type="application/vnd.apple.mpegurl",
|
||||||
|
headers={"Cache-Control": "no-store"},
|
||||||
|
)
|
||||||
|
|
||||||
headers = _build_headers(referer)
|
headers = _build_headers(referer)
|
||||||
async with httpx.AsyncClient(follow_redirects=True, timeout=20.0) as client:
|
async with httpx.AsyncClient(follow_redirects=True, timeout=20.0) as client:
|
||||||
try:
|
try:
|
||||||
|
|
|
||||||
|
|
@ -44,6 +44,7 @@ from app.connectors.direct_scrapers.siska import SiskaScraper
|
||||||
from app.connectors.direct_scrapers.sxyland import SxyLandScraper
|
from app.connectors.direct_scrapers.sxyland import SxyLandScraper
|
||||||
from app.connectors.direct_scrapers.sxyprn import SxyPrnScraper
|
from app.connectors.direct_scrapers.sxyprn import SxyPrnScraper
|
||||||
from app.connectors.direct_scrapers.galaxyporn import GalaxyPornScraper
|
from app.connectors.direct_scrapers.galaxyporn import GalaxyPornScraper
|
||||||
|
from app.connectors.direct_scrapers.pornbusy import PornBusyScraper
|
||||||
from app.connectors.direct_scrapers.watchporn import WatchPornScraper
|
from app.connectors.direct_scrapers.watchporn import WatchPornScraper
|
||||||
from app.connectors.direct_scrapers.youperv import YoupervScraper
|
from app.connectors.direct_scrapers.youperv import YoupervScraper
|
||||||
from app.connectors.direct_scrapers.xhamster import XHamsterScraper
|
from app.connectors.direct_scrapers.xhamster import XHamsterScraper
|
||||||
|
|
@ -165,6 +166,11 @@ ALL_BROWSE_SCRAPERS: list[type[BaseBrowseScraper]] = [
|
||||||
# w HTML, a NULL ukryłby sceny pod filtrem min_duration_sec). Playback: iframe
|
# w HTML, a NULL ukryłby sceny pod filtrem min_duration_sec). Playback: iframe
|
||||||
# seekplayer → istniejący silnik, HLS przez /proxy/hls.
|
# seekplayer → istniejący silnik, HLS przez /proxy/hls.
|
||||||
GalaxyPornScraper,
|
GalaxyPornScraper,
|
||||||
|
# PornBusyScraper — dodany 2026-07-27 (ocena 4.5/5). Metadane w `itemprop` (duration/
|
||||||
|
# uploadDate/thumb/desc) + obsada w `<div id="video-actors">` bez pollution. Homepage
|
||||||
|
# pagination ZEPSUTA (str. 1/2/3 identyczne) → listing z sitemapy, newest-first.
|
||||||
|
# Brak pola studio. Playback: loadvid/seekplayer/zpi ≈ 69% katalogu.
|
||||||
|
PornBusyScraper,
|
||||||
# Browse równolegle do istniejącego search scrapera (wzorzec xvideos/eporner):
|
# Browse równolegle do istniejącego search scrapera (wzorzec xvideos/eporner):
|
||||||
# search zostaje (pokrycie back-catalogu performerów), browse gwarantuje świeżość
|
# search zostaje (pokrycie back-catalogu performerów), browse gwarantuje świeżość
|
||||||
# wprost z feedu (watchdog 48h zamiast 168h). Konwersja 2026-06-24 (user request).
|
# wprost z feedu (watchdog 48h zamiast 168h). Konwersja 2026-06-24 (user request).
|
||||||
|
|
|
||||||
238
app/connectors/direct_scrapers/pornbusy.py
Normal file
238
app/connectors/direct_scrapers/pornbusy.py
Normal file
|
|
@ -0,0 +1,238 @@
|
||||||
|
"""pornbusy.com — browse scraper (WordPress + Rank Math). Dodany 2026-07-27.
|
||||||
|
|
||||||
|
Ripy studyjne (~60%) + JAV z kodami (~30%). Ocena orphan-risk LOW z pomiarem:
|
||||||
|
80% scen z próbki 100 ma performera, który u nas MA już ref tpdb/stashdb, a 21%
|
||||||
|
tytułów mocno matchuje sceny które już mamy (czyli się scalą, nie zorphanują).
|
||||||
|
|
||||||
|
**Paginacja homepage jest ZEPSUTA** — `/page/2/` i `/page/3/` zwracają ten sam
|
||||||
|
zestaw co strona 1 (zweryfikowane). Dlatego listing bierzemy z SITEMAPY:
|
||||||
|
`sitemap_index.xml` → 10× `post-sitemapN.xml` po ~500 URL-i, każdy z `<lastmod>`,
|
||||||
|
posortowane od najnowszych. `crawl_page` tnie ten katalog na strony po `_PAGE_SIZE`
|
||||||
|
(wzorzec jak `_playtube.BasePlayTubeScraper`).
|
||||||
|
|
||||||
|
Metadane — każde pole to osobny węzeł `itemprop`, nic nie trzeba wyłuskiwać z
|
||||||
|
tytułu-slug:
|
||||||
|
- title `<h1 class="entry-title">`, duration ISO, uploadDate, thumbnailUrl, description
|
||||||
|
- obsada: `<div id="video-actors">` — CZYSTA, bez pollution (na stronie sceny jest
|
||||||
|
dokładnie tyle linków `/actor/` ilu realnych aktorów; brak sekcji Related)
|
||||||
|
- kategorie + tagi: `<div class="tags-list">` (`/category/` i `/tag/`)
|
||||||
|
- studio: **brak** — pornbusy nie ma pola studia ani prefiksu w tytule
|
||||||
|
|
||||||
|
Playback: `itemprop="embedURL"` → extractor `pornbusycom` (loadvid / seekplayer /
|
||||||
|
zpi.cx ≈ 69% katalogu).
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import html
|
||||||
|
import logging
|
||||||
|
import re
|
||||||
|
|
||||||
|
from app.connectors.base import (
|
||||||
|
RawPerformer,
|
||||||
|
RawPlaybackSource,
|
||||||
|
RawScene,
|
||||||
|
RawTag,
|
||||||
|
)
|
||||||
|
from app.connectors.direct_scrapers._browse_base import BaseBrowseScraper
|
||||||
|
from app.connectors.direct_scrapers._playtube import _parse_iso_date
|
||||||
|
from app.extractors import browser_get
|
||||||
|
from app.normalize.text import slugify
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
_BASE = "https://pornbusy.com"
|
||||||
|
_PAGE_SIZE = 20
|
||||||
|
|
||||||
|
_SITEMAP_INDEX = f"{_BASE}/sitemap_index.xml"
|
||||||
|
_LOC_RE = re.compile(r"<loc>\s*([^<]+?)\s*</loc>")
|
||||||
|
_URL_BLOCK_RE = re.compile(r"<url>(.*?)</url>", re.DOTALL | re.IGNORECASE)
|
||||||
|
_LASTMOD_RE = re.compile(r"<lastmod>\s*([^<]+?)\s*</lastmod>")
|
||||||
|
|
||||||
|
_TITLE_RE = re.compile(r'<h1 class="entry-title"[^>]*>(.*?)</h1>', re.IGNORECASE | re.DOTALL)
|
||||||
|
_DUR_RE = re.compile(r'itemprop="duration"\s+content="([^"]+)"', re.IGNORECASE)
|
||||||
|
_DATE_RE = re.compile(r'itemprop="uploadDate"\s+content="([^"]+)"', re.IGNORECASE)
|
||||||
|
_THUMB_RE = re.compile(r'itemprop="thumbnailUrl"\s+content="([^"]+)"', re.IGNORECASE)
|
||||||
|
_DESC_RE = re.compile(r'itemprop="description"\s+content="([^"]*)"', re.IGNORECASE)
|
||||||
|
_ACTORS_BLOCK_RE = re.compile(r'<div id="video-actors">(.*?)</div>', re.IGNORECASE | re.DOTALL)
|
||||||
|
_ACTOR_RE = re.compile(r'/actor/[^/"]+/"[^>]*title="([^"]+)"', re.IGNORECASE)
|
||||||
|
_ACTOR_TEXT_RE = re.compile(r">([^<>]+)</a>")
|
||||||
|
_TAGS_BLOCK_RE = re.compile(r'<div class="tags-list">(.*?)</div>', re.IGNORECASE | re.DOTALL)
|
||||||
|
# Nazwa z atrybutu `title`, NIE z tekstu anchora: tekst poprzedza zagnieżdżona ikona
|
||||||
|
# (`<a ... title="Blonde"><i class="fa fa-tag"></i>Blonde</a>`), więc `>([^<]+)</a>`
|
||||||
|
# nie matchuje. `title` jest czysty i zawsze obecny.
|
||||||
|
_TAXO_RE = re.compile(
|
||||||
|
r'/(?:category|tag)/([a-z0-9\-]+)/"[^>]*title="([^"]+)"', re.IGNORECASE
|
||||||
|
)
|
||||||
|
# `P0DT0H42M52S`
|
||||||
|
_ISO_DUR_RE = re.compile(
|
||||||
|
r"P(?:(\d+)D)?T?(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?", re.IGNORECASE
|
||||||
|
)
|
||||||
|
|
||||||
|
# Junk-performer guard: pornbusy wrzuca do /actor/ także ogólniki.
|
||||||
|
_JUNK_ACTORS = frozenset({"amateur", "unknown", "anonymous", "n-a", "na", "various"})
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_iso_duration(value: str | None) -> int | None:
|
||||||
|
if not value:
|
||||||
|
return None
|
||||||
|
m = _ISO_DUR_RE.match(value.strip())
|
||||||
|
if not m:
|
||||||
|
return None
|
||||||
|
d, h, mn, s = (int(g or 0) for g in m.groups())
|
||||||
|
total = d * 86400 + h * 3600 + mn * 60 + s
|
||||||
|
return total or None
|
||||||
|
|
||||||
|
|
||||||
|
def _clean(raw: str) -> str:
|
||||||
|
return html.unescape(re.sub(r"\s+", " ", re.sub(r"<[^>]+>", " ", raw))).strip()
|
||||||
|
|
||||||
|
|
||||||
|
class PornBusyScraper(BaseBrowseScraper):
|
||||||
|
sitetag = "pornbusycom"
|
||||||
|
|
||||||
|
def __init__(self) -> None:
|
||||||
|
super().__init__()
|
||||||
|
# Katalog z sitemapy, newest-first. Lazy raz per instancję (browse_latest i
|
||||||
|
# deep_crawl tworzą instancję per run, więc 10 fetchy XML amortyzuje się).
|
||||||
|
self._catalog: list[str] | None = None
|
||||||
|
|
||||||
|
# Listing nie jest stronicowalny GET-em (homepage pagination zepsuta) —
|
||||||
|
# paginację robi sitemap w `crawl_page`. Te dwie metody są abstrakcyjne w bazie.
|
||||||
|
def _listing_url(self, page: int) -> str: # pragma: no cover - nieużywane
|
||||||
|
return _SITEMAP_INDEX
|
||||||
|
|
||||||
|
def _extract_scene_urls(self, listing_html: str) -> list[str]: # pragma: no cover
|
||||||
|
return []
|
||||||
|
|
||||||
|
def _load_catalog(self) -> list[str] | None:
|
||||||
|
if self._catalog is not None:
|
||||||
|
return self._catalog
|
||||||
|
try:
|
||||||
|
idx = browser_get(_SITEMAP_INDEX, timeout=self._timeout)
|
||||||
|
idx.raise_for_status()
|
||||||
|
except Exception as e:
|
||||||
|
log.warning("pornbusy: sitemap index fetch failed: %s", e)
|
||||||
|
return None
|
||||||
|
maps = [u for u in _LOC_RE.findall(idx.text) if "post-sitemap" in u]
|
||||||
|
if not maps:
|
||||||
|
log.warning("pornbusy: no post-sitemaps in index")
|
||||||
|
return None
|
||||||
|
|
||||||
|
entries: list[tuple[str, str]] = []
|
||||||
|
for sm_url in maps:
|
||||||
|
try:
|
||||||
|
sm = browser_get(sm_url, timeout=self._timeout)
|
||||||
|
sm.raise_for_status()
|
||||||
|
except Exception as e:
|
||||||
|
log.warning("pornbusy: sitemap fetch failed %s: %s", sm_url, e)
|
||||||
|
continue
|
||||||
|
for block in _URL_BLOCK_RE.findall(sm.text):
|
||||||
|
loc = _LOC_RE.search(block)
|
||||||
|
if not loc:
|
||||||
|
continue
|
||||||
|
url = loc.group(1)
|
||||||
|
# Pomijamy strony taksonomii/nav — sceny to `/<slug>/` jednosegmentowe.
|
||||||
|
if any(x in url for x in ("/actor/", "/category/", "/tag/", "/page/")):
|
||||||
|
continue
|
||||||
|
lm = _LASTMOD_RE.search(block)
|
||||||
|
entries.append((lm.group(1) if lm else "", url))
|
||||||
|
if not entries:
|
||||||
|
return None
|
||||||
|
entries.sort(key=lambda e: e[0], reverse=True)
|
||||||
|
seen: set[str] = set()
|
||||||
|
catalog: list[str] = []
|
||||||
|
for _, url in entries:
|
||||||
|
if url in seen:
|
||||||
|
continue
|
||||||
|
seen.add(url)
|
||||||
|
catalog.append(url)
|
||||||
|
log.info("pornbusy: catalog loaded — %d scenes from %d sitemaps", len(catalog), len(maps))
|
||||||
|
self._catalog = catalog
|
||||||
|
return catalog
|
||||||
|
|
||||||
|
def crawl_page(self, page: int) -> list[RawScene] | None:
|
||||||
|
catalog = self._load_catalog()
|
||||||
|
if catalog is None:
|
||||||
|
return None
|
||||||
|
chunk = catalog[(page - 1) * _PAGE_SIZE: page * _PAGE_SIZE]
|
||||||
|
if not chunk:
|
||||||
|
return []
|
||||||
|
out: list[RawScene] = []
|
||||||
|
for scene_url in chunk:
|
||||||
|
try:
|
||||||
|
res = browser_get(scene_url, timeout=self._timeout)
|
||||||
|
res.raise_for_status()
|
||||||
|
except Exception as e:
|
||||||
|
log.info("pornbusy detail fetch failed %s: %s", scene_url, e)
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
raw = self._parse_detail(scene_url, res.text)
|
||||||
|
except Exception as e:
|
||||||
|
log.warning("pornbusy detail parse failed %s: %s", scene_url, e)
|
||||||
|
continue
|
||||||
|
if raw is not None:
|
||||||
|
out.append(raw)
|
||||||
|
return out
|
||||||
|
|
||||||
|
def _parse_detail(self, scene_url: str, detail_html: str) -> RawScene | None:
|
||||||
|
tm = _TITLE_RE.search(detail_html)
|
||||||
|
title = _clean(tm.group(1)) if tm else ""
|
||||||
|
if not title:
|
||||||
|
return None
|
||||||
|
|
||||||
|
dm = _DUR_RE.search(detail_html)
|
||||||
|
duration_sec = _parse_iso_duration(dm.group(1)) if dm else None
|
||||||
|
um = _DATE_RE.search(detail_html)
|
||||||
|
release_date = _parse_iso_date(um.group(1)) if um else None
|
||||||
|
thm = _THUMB_RE.search(detail_html)
|
||||||
|
thumbnail_url = thm.group(1) if thm else None
|
||||||
|
dsm = _DESC_RE.search(detail_html)
|
||||||
|
description = html.unescape(dsm.group(1)).strip() if dsm else None
|
||||||
|
if description and description.strip().lower() == title.strip().lower():
|
||||||
|
description = None # opis bywa kopią tytułu — nie duplikujemy
|
||||||
|
|
||||||
|
performers: list[RawPerformer] = []
|
||||||
|
seen_p: set[str] = set()
|
||||||
|
ab = _ACTORS_BLOCK_RE.search(detail_html)
|
||||||
|
if ab:
|
||||||
|
names = _ACTOR_RE.findall(ab.group(1)) or _ACTOR_TEXT_RE.findall(ab.group(1))
|
||||||
|
for name in names:
|
||||||
|
name = html.unescape(name).strip()
|
||||||
|
sl = slugify(name)
|
||||||
|
if not sl or sl in seen_p or sl in _JUNK_ACTORS:
|
||||||
|
continue
|
||||||
|
seen_p.add(sl)
|
||||||
|
performers.append(
|
||||||
|
RawPerformer(external_id=f"{self.sitetag}:performer:{sl}", name=name)
|
||||||
|
)
|
||||||
|
|
||||||
|
tags: list[RawTag] = []
|
||||||
|
seen_t: set[str] = set()
|
||||||
|
tb = _TAGS_BLOCK_RE.search(detail_html)
|
||||||
|
if tb:
|
||||||
|
for slug, name in _TAXO_RE.findall(tb.group(1)):
|
||||||
|
name = html.unescape(name).strip()
|
||||||
|
if not name or slug in seen_t or slug in seen_p:
|
||||||
|
continue
|
||||||
|
seen_t.add(slug)
|
||||||
|
tags.append(RawTag(external_id=f"{self.sitetag}:tag:{slug}", name=name, slug=slug))
|
||||||
|
|
||||||
|
return RawScene(
|
||||||
|
external_id=f"{self.sitetag}:{scene_url}",
|
||||||
|
title=title,
|
||||||
|
description=description,
|
||||||
|
release_date=release_date,
|
||||||
|
duration_sec=duration_sec,
|
||||||
|
url=scene_url,
|
||||||
|
# studio: pornbusy go nie ma (ani pole, ani prefiks tytułu)
|
||||||
|
performers=performers,
|
||||||
|
tags=tags,
|
||||||
|
playback_sources=[
|
||||||
|
RawPlaybackSource(
|
||||||
|
origin=f"tube:{self.sitetag}",
|
||||||
|
page_url=scene_url,
|
||||||
|
duration_sec=duration_sec,
|
||||||
|
thumbnail_url=thumbnail_url,
|
||||||
|
)
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
@ -39,6 +39,7 @@ from app.extractors.tubes import (
|
||||||
latestpornvideo,
|
latestpornvideo,
|
||||||
paradisehill,
|
paradisehill,
|
||||||
porn00,
|
porn00,
|
||||||
|
pornbusy,
|
||||||
porntrex,
|
porntrex,
|
||||||
supjav,
|
supjav,
|
||||||
sxyprn,
|
sxyprn,
|
||||||
|
|
@ -112,6 +113,10 @@ _REGISTRY: dict[str, Callable[[str], list[StreamSource] | None]] = {
|
||||||
# galaxyporn — iframe rodziny seekplayer (upns/4meplayer/seekplays); silnik już mamy,
|
# galaxyporn — iframe rodziny seekplayer (upns/4meplayer/seekplays); silnik już mamy,
|
||||||
# extractor tylko wyłuskuje iframe. HLS Referer+IP-bound → /proxy/hls.
|
# extractor tylko wyłuskuje iframe. HLS Referer+IP-bound → /proxy/hls.
|
||||||
"galaxyporncom": galaxyporn.extract,
|
"galaxyporncom": galaxyporn.extract,
|
||||||
|
# pornbusy — dispatch po embedURL: loadvid (POST-manifest przez /proxy/hls),
|
||||||
|
# rodzina seekplayer (istniejący silnik), zpi.cx (embedURL to gotowy plik).
|
||||||
|
# upload18 + ogon → None (token IP-bound, resolve wymusiłby proxy całego wideo).
|
||||||
|
"pornbusycom": pornbusy.extract,
|
||||||
"siskavideo": _embed_iframe.extract,
|
"siskavideo": _embed_iframe.extract,
|
||||||
"porn4dayspw": _embed_iframe.extract,
|
"porn4dayspw": _embed_iframe.extract,
|
||||||
"porndishcom": _embed_iframe.extract,
|
"porndishcom": _embed_iframe.extract,
|
||||||
|
|
|
||||||
128
app/extractors/hosters/loadvid.py
Normal file
128
app/extractors/hosters/loadvid.py
Normal file
|
|
@ -0,0 +1,128 @@
|
||||||
|
"""cdn.loadvid.com — HLS hoster oddający manifest przez POST (nie URL).
|
||||||
|
|
||||||
|
Protokół (reverse-engineer 2026-07-27, potwierdzony live):
|
||||||
|
1. GET /videos/play/<hash> → HTML z `<meta name="csrf-token">` + inline
|
||||||
|
`window.LoadVidConfig = { videoHash, videoToken, ... }`
|
||||||
|
2. POST /videos/resolve-token {token, hash} + nagłówek `X-CSRF-TOKEN`
|
||||||
|
→ **treść manifestu m3u8** (200, `application/vnd.apple.mpegurl`, ~116 KB,
|
||||||
|
~870 segmentów), a NIE URL do manifestu.
|
||||||
|
|
||||||
|
Dlatego to nie jest zwykły extractor "URL → URL": nie da się oddać linku do
|
||||||
|
manifestu, bo taki link nie istnieje (GET na resolve-token → 405, zgadywane
|
||||||
|
`/index.m3u8` → 404). Manifest musi wyprodukować backend.
|
||||||
|
|
||||||
|
Rozwiązanie: `extract()` zwraca **embed URL** oznaczony `manifest_producer=loadvid`.
|
||||||
|
`/proxy/hls/<token>/play.m3u8` rozpoznaje ten marker i zamiast GET-a woła
|
||||||
|
`fetch_manifest()`, po czym serwuje manifest telefonowi. Segmenty w manifeście są
|
||||||
|
ABSOLUTNE i w pełni przenośne (zweryfikowane: 206 `bytes 0-1023` BEZ jakichkolwiek
|
||||||
|
nagłówków, bez Referera, bez tokena — serwowane jako `image/png`, w środku TS),
|
||||||
|
więc telefon ciągnie je bezpośrednio z CDN. Przez VPS idzie tylko manifest.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import re
|
||||||
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
|
from app.extractors._fetch import _DEFAULT_IMPERSONATE, _DEFAULT_UA, _HAS_CURL_CFFI
|
||||||
|
from app.extractors._models import HosterDead, StreamSource
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
_HOST_RE = re.compile(r"^(?:[a-z0-9]+\.)?loadvid\.com$", re.IGNORECASE)
|
||||||
|
_CSRF_RE = re.compile(r'<meta name="csrf-token" content="([^"]+)"', re.IGNORECASE)
|
||||||
|
_CONFIG_RE = re.compile(r"LoadVidConfig\s*=\s*(\{.*?\})\s*;", re.DOTALL)
|
||||||
|
_TOKEN_RE = re.compile(r"""videoToken\s*:\s*['"]([^'"]+)""")
|
||||||
|
_HASH_RE = re.compile(r"""videoHash\s*:\s*['"]([^'"]+)""")
|
||||||
|
_RESOLVE_PATH = "/videos/resolve-token"
|
||||||
|
|
||||||
|
|
||||||
|
def matches(url: str) -> bool:
|
||||||
|
try:
|
||||||
|
return bool(_HOST_RE.match(urlparse(url).hostname or ""))
|
||||||
|
except Exception:
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def fetch_manifest(embed_url: str, *, timeout: float = 30.0) -> str | None:
|
||||||
|
"""Embed URL → treść manifestu m3u8 (albo None).
|
||||||
|
|
||||||
|
CSRF-token i cookie sesji muszą pochodzić z TEGO SAMEGO requestu co POST,
|
||||||
|
dlatego jedna sesja curl_cffi na całą operację."""
|
||||||
|
if not _HAS_CURL_CFFI:
|
||||||
|
log.info("loadvid: curl_cffi unavailable")
|
||||||
|
return None
|
||||||
|
from curl_cffi import requests as cf
|
||||||
|
|
||||||
|
parsed = urlparse(embed_url)
|
||||||
|
origin = f"{parsed.scheme}://{parsed.hostname}"
|
||||||
|
session = cf.Session(impersonate=_DEFAULT_IMPERSONATE)
|
||||||
|
try:
|
||||||
|
r = session.get(
|
||||||
|
embed_url,
|
||||||
|
headers={"User-Agent": _DEFAULT_UA, "Accept": "text/html,application/xhtml+xml"},
|
||||||
|
timeout=timeout,
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
log.info("loadvid: embed fetch failed %s: %s", embed_url, e)
|
||||||
|
return None
|
||||||
|
if r.status_code in (404, 410):
|
||||||
|
raise HosterDead(f"loadvid {embed_url}: HTTP {r.status_code}")
|
||||||
|
if r.status_code != 200 or not r.text:
|
||||||
|
log.info("loadvid: embed status=%s for %s", r.status_code, embed_url)
|
||||||
|
return None
|
||||||
|
|
||||||
|
csrf = _CSRF_RE.search(r.text)
|
||||||
|
cfg = _CONFIG_RE.search(r.text)
|
||||||
|
if not csrf or not cfg:
|
||||||
|
log.info("loadvid: no csrf/LoadVidConfig on %s", embed_url)
|
||||||
|
return None
|
||||||
|
tok = _TOKEN_RE.search(cfg.group(1))
|
||||||
|
hsh = _HASH_RE.search(cfg.group(1))
|
||||||
|
if not tok or not hsh:
|
||||||
|
log.info("loadvid: no videoToken/videoHash on %s", embed_url)
|
||||||
|
return None
|
||||||
|
|
||||||
|
try:
|
||||||
|
pr = session.post(
|
||||||
|
origin + _RESOLVE_PATH,
|
||||||
|
headers={
|
||||||
|
"User-Agent": _DEFAULT_UA,
|
||||||
|
"X-CSRF-TOKEN": csrf.group(1),
|
||||||
|
"X-Requested-With": "XMLHttpRequest",
|
||||||
|
"Accept": "application/vnd.apple.mpegurl",
|
||||||
|
"Referer": embed_url,
|
||||||
|
},
|
||||||
|
json={"token": tok.group(1), "hash": hsh.group(1)},
|
||||||
|
timeout=timeout,
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
log.info("loadvid: resolve-token failed %s: %s", embed_url, e)
|
||||||
|
return None
|
||||||
|
if pr.status_code != 200 or "#EXTM3U" not in pr.text:
|
||||||
|
log.info("loadvid: resolve-token status=%s len=%d", pr.status_code, len(pr.text or ""))
|
||||||
|
return None
|
||||||
|
return pr.text
|
||||||
|
|
||||||
|
|
||||||
|
def extract(page_url: str, *, timeout: float = 30.0) -> list[StreamSource] | None:
|
||||||
|
"""Zwraca embed URL z markerem producenta — manifest powstaje dopiero w
|
||||||
|
`/proxy/hls` (patrz docstring modułu). Weryfikujemy tu, że manifest realnie
|
||||||
|
da się wyprodukować, żeby nie oddawać martwego źródła."""
|
||||||
|
if not matches(page_url):
|
||||||
|
return None
|
||||||
|
manifest = fetch_manifest(page_url, timeout=timeout)
|
||||||
|
if not manifest:
|
||||||
|
return None
|
||||||
|
return [
|
||||||
|
StreamSource(
|
||||||
|
link=page_url,
|
||||||
|
type="m3u8",
|
||||||
|
raw={
|
||||||
|
# Segmenty absolutne i portable → telefon ciągnie je direct z CDN,
|
||||||
|
# przez VPS leci tylko manifest.
|
||||||
|
"mobile_direct_ok": True,
|
||||||
|
"manifest_producer": "loadvid",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
]
|
||||||
82
app/extractors/tubes/pornbusy.py
Normal file
82
app/extractors/tubes/pornbusy.py
Normal file
|
|
@ -0,0 +1,82 @@
|
||||||
|
"""pornbusy.com — dispatch po hosterze z `itemprop="embedURL"`.
|
||||||
|
|
||||||
|
Katalog jest rozproszony po hosterach (próbka 120 scen): ~41% loadvid,
|
||||||
|
~20% rodzina seekplayer (av.ezplayer.me / av.seeks.cloud / 4k.upn.one /
|
||||||
|
4k.player4me.vip — objęte poszerzonym `_HOST_RE` silnika), ~8% zpi.cx,
|
||||||
|
~12% upload18 (token z wbitym `i=<ip>/24` = IP-bound), reszta ogon.
|
||||||
|
|
||||||
|
Obsługujemy trzy pierwsze (~69% katalogu):
|
||||||
|
- loadvid → manifest przez POST, marker `manifest_producer` (patrz hosters/loadvid.py)
|
||||||
|
- seekplayer → istniejący silnik (ten sam AES key/IV)
|
||||||
|
- zpi.cx → `embedURL` JEST plikiem wideo (mimo `.webm` w nazwie to mp4);
|
||||||
|
zweryfikowane: Range 206 `video/mp4`, bez Referera, bez tokena
|
||||||
|
|
||||||
|
upload18 i ogon zwracamy jako None — resolve na VPS wymusiłby proxowanie CAŁEGO
|
||||||
|
wideo (token IP-bound), co łamie zasadę no-video-proxy.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import re
|
||||||
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
|
from app.extractors._fetch import fetch_tube_html
|
||||||
|
from app.extractors._models import StreamSource
|
||||||
|
from app.extractors.hosters import loadvid, seekplayer_engine
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
_BASE = "https://pornbusy.com"
|
||||||
|
_EMBED_RE = re.compile(r'itemprop="embedURL"\s+content="([^"]+)"', re.IGNORECASE)
|
||||||
|
_IFRAME_RE = re.compile(r'<iframe[^>]+src="([^"]+)"', re.IGNORECASE)
|
||||||
|
_ZPI_HOST_RE = re.compile(r"^(?:[a-z0-9]+\.)?zpi\.cx$", re.IGNORECASE)
|
||||||
|
|
||||||
|
|
||||||
|
def extract(page_url: str, *, timeout: float = 60.0) -> list[StreamSource] | None:
|
||||||
|
html_text = fetch_tube_html(page_url, timeout=timeout)
|
||||||
|
m = _EMBED_RE.search(html_text) or _IFRAME_RE.search(html_text)
|
||||||
|
if not m:
|
||||||
|
log.info("pornbusy: no embedURL/iframe on %s", page_url)
|
||||||
|
return None
|
||||||
|
embed = m.group(1).strip()
|
||||||
|
if embed.startswith("//"):
|
||||||
|
embed = "https:" + embed
|
||||||
|
|
||||||
|
if loadvid.matches(embed):
|
||||||
|
return loadvid.extract(embed, timeout=timeout)
|
||||||
|
|
||||||
|
if seekplayer_engine.matches(embed):
|
||||||
|
sources = seekplayer_engine.extract(embed, timeout=timeout)
|
||||||
|
if not sources:
|
||||||
|
return None
|
||||||
|
player_origin = f"https://{urlparse(embed).hostname}/"
|
||||||
|
out: list[StreamSource] = []
|
||||||
|
for s in sources:
|
||||||
|
raw = dict(s.raw or {})
|
||||||
|
raw["proxy_no_verify"] = True
|
||||||
|
out.append(
|
||||||
|
StreamSource(
|
||||||
|
link=s.link,
|
||||||
|
type=s.type or "m3u8",
|
||||||
|
quality=s.quality,
|
||||||
|
# Referer = origin PLAYERA, nie strony (jak przy galaxyporn:
|
||||||
|
# z Refererem strony CDN zwraca 403).
|
||||||
|
referer=s.referer or player_origin,
|
||||||
|
raw=raw,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return out
|
||||||
|
|
||||||
|
host = (urlparse(embed).hostname or "").lower()
|
||||||
|
if _ZPI_HOST_RE.match(host):
|
||||||
|
# embedURL to bezpośrednio plik (rozszerzenie `.webm` myli — to mp4).
|
||||||
|
return [
|
||||||
|
StreamSource(
|
||||||
|
link=embed,
|
||||||
|
type="mp4",
|
||||||
|
raw={"mobile_direct_ok": True},
|
||||||
|
)
|
||||||
|
]
|
||||||
|
|
||||||
|
log.info("pornbusy: unsupported hoster %s (%s)", host, page_url)
|
||||||
|
return None
|
||||||
Loading…
Add table
Reference in a new issue