feat(watchporn): re-enable as KVS browse scraper + native mp4 extractor
watchporn.to was disabled 2026-05-12 (all iframes were DoodStream/CAPTCHA -> black-screen WebView). The site has since been rebuilt as a KVS tube: the DoodStream hosters are gone, scenes now expose flashvars get_file direct mp4. Rewrote the dead ?s= search scraper as a /latest-updates/ browse scraper (og:title, JSON-LD duration/uploadDate, /models/ performers, /tags/ tags, /categories/ studio) and added a KVS get_file extractor (porntrex-style same-session 302 resolve; token not IP-bound, plays from VPS). Moved to ALL_BROWSE_SCRAPERS, extractor swapped from _embed_iframe. Verified: 35 fresh scenes/page (today), multi-performer, playback 1080p mp4 206. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
jtrzupek
parent
6583fdd185
commit
c419d3ab4b
4 changed files with 236 additions and 16 deletions
|
|
@ -77,11 +77,9 @@ ALL_DIRECT_SCRAPERS: list[type[BaseDirectTubeScraper]] = [
|
|||
# Mixdrop zrebrandował na m1xdrop.bz, yt-dlp out-of-date, packer/JS extract = fail.
|
||||
# Playmogo = DoodStream CAPTCHA. Porn-app sam olewa xmoviesforyou (brak handlera w
|
||||
# jadx). 1,321 solo-orphan scen.
|
||||
# WatchPornScraper — wyłączony 2026-05-12 (user bug-report). Wszystkie iframes to
|
||||
# DoodStream variants (playmogo/d0000d/dooood/mivalyo) z CAPTCHA gate. WebView na
|
||||
# mobile = black screen (player JS nie inicjalizuje się przez Turnstile). 16%
|
||||
# scen solo (no backup tube), 84% multi-source — user może użyć innego tube. yt-dlp
|
||||
# nie wspiera DoodStream ("Piracy"), własny resolver TBD jeśli warto.
|
||||
# WatchPornScraper — RE-ENABLE 2026-07-02, przeniesiony do ALL_BROWSE_SCRAPERS.
|
||||
# Site przebudowany na KVS (DoodStream-CAPTCHA z 2026-05-12 zniknął); browse
|
||||
# /latest-updates/ + flashvars get_file direct mp4 (extractor watchporn). Patrz watchporn.py.
|
||||
# SiskaScraper — przeniesiony do ALL_BROWSE_SCRAPERS (browse-konwersja 2026-06-20,
|
||||
# bo search siski zepsuty site-side — `?s=` ignoruje query). Patrz siska.py.
|
||||
# Porn4DaysScraper — wyłączony 2026-05-12 (post audit fix). 100% scen na streamtape
|
||||
|
|
@ -144,6 +142,11 @@ from app.connectors.direct_scrapers.xvideos_browse import XVideosBrowseScraper
|
|||
ALL_BROWSE_SCRAPERS: list[type[BaseBrowseScraper]] = [
|
||||
PerverzijaScraper,
|
||||
PornDishScraper,
|
||||
# WatchPornScraper — re-enabled 2026-07-02 jako browse (site przebudowany na KVS,
|
||||
# DoodStream-CAPTCHA zniknął). Browse /latest-updates/: og:title, JSON-LD duration/
|
||||
# uploadDate, /models/ performerzy, /tags/ tagi, /categories/ studio. Playback KVS
|
||||
# get_file direct mp4 (extractor watchporn, VPS-side, token nie IP-bound).
|
||||
WatchPornScraper,
|
||||
# Browse równolegle do istniejącego search scrapera (wzorzec xvideos/eporner):
|
||||
# search zostaje (pokrycie back-catalogu performerów), browse gwarantuje świeżość
|
||||
# wprost z feedu (watchdog 48h zamiast 168h). Konwersja 2026-06-24 (user request).
|
||||
|
|
|
|||
|
|
@ -1,19 +1,108 @@
|
|||
"""watchporn.to — direct HTML scrape.
|
||||
"""watchporn.to — browse scraper (KVS engine). Re-enabled 2026-07-02.
|
||||
|
||||
Search: `https://watchporn.to/page/<n>/?s=<q>` (WordPress).
|
||||
Scene URL: `https://watchporn.to/videos/<slug>/`.
|
||||
Był search-scraperem (`?s=`), zamarzł, a potem site przebudowano na KVS z nowym
|
||||
layoutem. DoodStream-CAPTCHA (powód wyłączenia 2026-05-12) zniknął — teraz KVS
|
||||
flashvars `get_file` direct mp4 (extractor `watchporn`, VPS-side, token nie IP-bound).
|
||||
|
||||
Browse `/latest-updates/` → detail page:
|
||||
- title: og:title ("Studio/Creator - Scene Title")
|
||||
- duration + release_date: JSON-LD "duration" (ISO) + "uploadDate"
|
||||
- performerzy: `/models/<slug>/` (pomijamy numeryczne id-slugi), nazwa z tekstu linku
|
||||
- tagi: `/tags/<slug>/`
|
||||
- studio: pierwszy `/categories/<slug>/` (np. EvilAngel, ManyVids)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import re
|
||||
|
||||
from app.connectors.direct_scrapers._search_base import BaseSearchScraper
|
||||
from app.connectors.base import (
|
||||
RawPerformer,
|
||||
RawPlaybackSource,
|
||||
RawScene,
|
||||
RawStudio,
|
||||
RawTag,
|
||||
)
|
||||
from app.connectors.direct_scrapers._browse_base import BaseBrowseScraper, meta_content
|
||||
from app.connectors.direct_scrapers._playtube import _parse_iso_date, _parse_iso_duration
|
||||
from app.normalize.text import slugify
|
||||
|
||||
_BASE = "https://watchporn.to"
|
||||
_SCENE_URL_RE = re.compile(r'href="(https://watchporn\.to/video/\d+/[a-z0-9\-]+/)"', re.IGNORECASE)
|
||||
_MODEL_RE = re.compile(r'href="https://watchporn\.to/models/([a-z0-9\-]+)/"[^>]*>([^<]+)', re.IGNORECASE)
|
||||
_TAG_RE = re.compile(r'href="https://watchporn\.to/tags/([a-z0-9\-]+)/"[^>]*>([^<]+)', re.IGNORECASE)
|
||||
_CAT_RE = re.compile(r'href="https://watchporn\.to/categories/([a-z0-9\-]+)/"[^>]*>([^<]+)', re.IGNORECASE)
|
||||
_DUR_RE = re.compile(r'"duration"\s*:\s*"([^"]+)"')
|
||||
_UPLOAD_RE = re.compile(r'"uploadDate"\s*:\s*"([^"]+)"')
|
||||
|
||||
|
||||
class WatchPornScraper(BaseSearchScraper):
|
||||
class WatchPornScraper(BaseBrowseScraper):
|
||||
sitetag = "watchporn"
|
||||
_search_url_template = "https://watchporn.to/page/{page}/?s={query}"
|
||||
_scene_url_re = re.compile(
|
||||
r'href="(?P<url>https://watchporn\.to/videos/(?P<slug>[a-z0-9][a-z0-9\-]+))/"',
|
||||
re.IGNORECASE,
|
||||
|
||||
def _listing_url(self, page: int) -> str:
|
||||
return f"{_BASE}/latest-updates/" if page <= 1 else f"{_BASE}/latest-updates/{page}/"
|
||||
|
||||
def _extract_scene_urls(self, listing_html: str) -> list[str]:
|
||||
seen: set[str] = set()
|
||||
out: list[str] = []
|
||||
for m in _SCENE_URL_RE.finditer(listing_html):
|
||||
url = m.group(1)
|
||||
if url not in seen:
|
||||
seen.add(url)
|
||||
out.append(url)
|
||||
return out
|
||||
|
||||
def _parse_detail(self, scene_url: str, detail_html: str) -> RawScene | None:
|
||||
title = (meta_content(detail_html, property="og:title") or "").strip()
|
||||
if not title:
|
||||
return None
|
||||
|
||||
dm = _DUR_RE.search(detail_html)
|
||||
duration_sec = _parse_iso_duration(dm.group(1)) if dm else None
|
||||
um = _UPLOAD_RE.search(detail_html)
|
||||
release_date = _parse_iso_date(um.group(1)) if um else None
|
||||
|
||||
performers: list[RawPerformer] = []
|
||||
seen_p: set[str] = set()
|
||||
for m in _MODEL_RE.finditer(detail_html):
|
||||
slug = m.group(1)
|
||||
name = html.unescape(m.group(2)).strip()
|
||||
if slug.isdigit() or slug in seen_p or not name:
|
||||
continue
|
||||
seen_p.add(slug)
|
||||
performers.append(RawPerformer(external_id=f"{self.sitetag}:model:{slug}", name=name))
|
||||
|
||||
tags: list[RawTag] = []
|
||||
seen_t: set[str] = set()
|
||||
for m in _TAG_RE.finditer(detail_html):
|
||||
slug = m.group(1)
|
||||
name = html.unescape(m.group(2)).strip()
|
||||
if slug in seen_t or not name:
|
||||
continue
|
||||
seen_t.add(slug)
|
||||
tags.append(RawTag(external_id=f"{self.sitetag}:tag:{slug}", name=name, slug=slug))
|
||||
|
||||
studio: RawStudio | None = None
|
||||
cm = _CAT_RE.search(detail_html)
|
||||
if cm:
|
||||
cname = html.unescape(cm.group(2)).strip()
|
||||
if cname:
|
||||
studio = RawStudio(
|
||||
external_id=f"{self.sitetag}:studio:{slugify(cname)}", name=cname, slug=slugify(cname)
|
||||
)
|
||||
|
||||
return RawScene(
|
||||
external_id=f"{self.sitetag}:{scene_url}",
|
||||
title=title,
|
||||
release_date=release_date,
|
||||
duration_sec=duration_sec,
|
||||
url=scene_url,
|
||||
studio=studio,
|
||||
performers=performers,
|
||||
tags=tags,
|
||||
playback_sources=[
|
||||
RawPlaybackSource(
|
||||
origin=f"tube:{self.sitetag}", page_url=scene_url, duration_sec=duration_sec
|
||||
)
|
||||
],
|
||||
)
|
||||
|
|
|
|||
|
|
@ -38,6 +38,7 @@ from app.extractors.tubes import (
|
|||
porn00,
|
||||
porntrex,
|
||||
sxyprn,
|
||||
watchporn,
|
||||
xhamster,
|
||||
yespornvip,
|
||||
)
|
||||
|
|
@ -96,7 +97,9 @@ _REGISTRY: dict[str, Callable[[str], list[StreamSource] | None]] = {
|
|||
# Aggregator tubes — generic embed-iframe → hoster unpacker
|
||||
"latestpornvideocom": latestpornvideo.extract,
|
||||
"xmoviesforyoucom": _embed_iframe.extract,
|
||||
"watchporn": _embed_iframe.extract,
|
||||
# watchporn — 2026-07-02 przebudowany na KVS (DoodStream-CAPTCHA zniknął).
|
||||
# flashvars get_file direct mp4, same-session 302 resolve, token nie IP-bound.
|
||||
"watchporn": watchporn.extract,
|
||||
"siskavideo": _embed_iframe.extract,
|
||||
"porn4dayspw": _embed_iframe.extract,
|
||||
"porndishcom": _embed_iframe.extract,
|
||||
|
|
|
|||
125
app/extractors/tubes/watchporn.py
Normal file
125
app/extractors/tubes/watchporn.py
Normal file
|
|
@ -0,0 +1,125 @@
|
|||
"""watchporn.to — KVS engine direct stream extractor (re-enabled 2026-07-02).
|
||||
|
||||
Site przebudowany na KVS: detail page ma flashvars `video_url`/`video_alt_url*`,
|
||||
każdy to `get_file/<srv>/<token>/<path>_<q>p.mp4/?v-acctoken=...`. Ten sam wzorzec co
|
||||
porntrex. get_file 302 → finalny CDN url; resolvujemy same-session (token bywa
|
||||
cookie/session-bound) i oddajemy finalny url per jakość. Token nie IP-bound
|
||||
(cross-IP 206 z VPS zweryfikowane 2026-07-02) → mobile gra direct, zero proxy.
|
||||
Wcześniejszy DoodStream-CAPTCHA zniknął.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
|
||||
from app.extractors._fetch import _DEFAULT_IMPERSONATE, _DEFAULT_UA, _HAS_CURL_CFFI, fetch_tube_html
|
||||
from app.extractors._models import HosterDead, StreamSource
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
_BASE = "https://watchporn.to"
|
||||
|
||||
_DEAD_RE = re.compile(
|
||||
r"this video (?:was|has been) deleted|video (?:was|has been) removed"
|
||||
r"|no longer available|video is unavailable",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_URL_RE = re.compile(
|
||||
r"(video(?:_alt)?_url\d*)\s*:\s*'(https?://[^']+/get_file/[^']+)'",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_TEXT_RE = re.compile(
|
||||
r"(video(?:_alt)?_url\d*)_text\s*:\s*'([^']*)'",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def _quality_rank(label: str | None) -> int:
|
||||
if not label:
|
||||
return -1
|
||||
m = re.search(r"(\d{3,4})\s*p", label, re.IGNORECASE)
|
||||
return int(m.group(1)) if m else -1
|
||||
|
||||
|
||||
def _resolve_get_file(session, get_file_url: str, timeout: float) -> str | None:
|
||||
sep = "&" if "?" in get_file_url else "?"
|
||||
url = f"{get_file_url}{sep}rnd={int(time.time() * 1000)}"
|
||||
try:
|
||||
r = session.get(
|
||||
url,
|
||||
timeout=timeout,
|
||||
allow_redirects=True,
|
||||
stream=True,
|
||||
headers={"Referer": _BASE + "/", "Range": "bytes=0-1"},
|
||||
)
|
||||
final = str(r.url)
|
||||
status = r.status_code
|
||||
r.close()
|
||||
except Exception as e:
|
||||
log.info("watchporn: get_file resolve failed (%s): %s", get_file_url[:60], e)
|
||||
return None
|
||||
if status >= 400 or "/get_file/" in final:
|
||||
log.info("watchporn: get_file resolve bad status=%s final=%s", status, final[:70])
|
||||
return None
|
||||
return final
|
||||
|
||||
|
||||
def extract(page_url: str, *, timeout: float = 60.0) -> list[StreamSource] | None:
|
||||
session = None
|
||||
if _HAS_CURL_CFFI:
|
||||
from curl_cffi import requests as _cf_requests
|
||||
session = _cf_requests.Session(impersonate=_DEFAULT_IMPERSONATE)
|
||||
try:
|
||||
resp = session.get(
|
||||
page_url,
|
||||
headers={"User-Agent": _DEFAULT_UA, "Accept": "text/html,application/xhtml+xml"},
|
||||
timeout=timeout,
|
||||
allow_redirects=True,
|
||||
)
|
||||
html_text = resp.text if resp.status_code < 400 else ""
|
||||
except Exception as e:
|
||||
log.info("watchporn: page fetch failed %s: %s", page_url, e)
|
||||
html_text = ""
|
||||
if not html_text:
|
||||
html_text = fetch_tube_html(page_url, timeout=timeout)
|
||||
session = None
|
||||
else:
|
||||
html_text = fetch_tube_html(page_url, timeout=timeout)
|
||||
|
||||
if html_text and _DEAD_RE.search(html_text):
|
||||
raise HosterDead(f"watchporn {page_url}: video deleted/removed")
|
||||
|
||||
quality_by_var: dict[str, str] = {}
|
||||
for m in _TEXT_RE.finditer(html_text):
|
||||
quality_by_var[m.group(1).lower()] = m.group(2).strip()
|
||||
|
||||
seen: set[str] = set()
|
||||
result: list[StreamSource] = []
|
||||
for m in _URL_RE.finditer(html_text):
|
||||
var_name = m.group(1).lower()
|
||||
url = m.group(2)
|
||||
if url in seen:
|
||||
continue
|
||||
seen.add(url)
|
||||
quality = quality_by_var.get(var_name)
|
||||
final_link = url
|
||||
if session is not None:
|
||||
resolved = _resolve_get_file(session, url, timeout)
|
||||
if resolved:
|
||||
final_link = resolved
|
||||
result.append(
|
||||
StreamSource(
|
||||
link=final_link,
|
||||
type="mp4",
|
||||
quality=quality or None,
|
||||
referer=_BASE + "/",
|
||||
raw={"mobile_direct_ok": True},
|
||||
)
|
||||
)
|
||||
|
||||
if not result:
|
||||
log.info("watchporn: no KVS video_url in flashvars on %s", page_url)
|
||||
return None
|
||||
result.sort(key=lambda s: _quality_rank(s.quality), reverse=True)
|
||||
return result
|
||||
Loading…
Add table
Reference in a new issue