From 015bd5bb7aa26c5c9961f6da3a43a011a571f891 Mon Sep 17 00:00:00 2001 From: goon-foss Date: Thu, 2 Jul 2026 14:32:36 +0200 Subject: [PATCH] feat(watchporn): re-enable as KVS browse scraper + native mp4 extractor watchporn.to was disabled 2026-05-12 (all iframes were DoodStream/CAPTCHA -> black-screen WebView). The site has since been rebuilt as a KVS tube: the DoodStream hosters are gone, scenes now expose flashvars get_file direct mp4. Rewrote the dead ?s= search scraper as a /latest-updates/ browse scraper (og:title, JSON-LD duration/uploadDate, /models/ performers, /tags/ tags, /categories/ studio) and added a KVS get_file extractor (porntrex-style same-session 302 resolve; token not IP-bound, plays from VPS). Moved to ALL_BROWSE_SCRAPERS, extractor swapped from _embed_iframe. Verified: 35 fresh scenes/page (today), multi-performer, playback 1080p mp4 206. Co-Authored-By: Claude Opus 4.8 --- app/connectors/direct_scrapers/__init__.py | 13 +- app/connectors/direct_scrapers/watchporn.py | 109 +++++++++++++++-- app/extractors/__init__.py | 5 +- app/extractors/tubes/watchporn.py | 125 ++++++++++++++++++++ 4 files changed, 236 insertions(+), 16 deletions(-) create mode 100644 app/extractors/tubes/watchporn.py diff --git a/app/connectors/direct_scrapers/__init__.py b/app/connectors/direct_scrapers/__init__.py index ff08368..7a84ca3 100644 --- a/app/connectors/direct_scrapers/__init__.py +++ b/app/connectors/direct_scrapers/__init__.py @@ -77,11 +77,9 @@ ALL_DIRECT_SCRAPERS: list[type[BaseDirectTubeScraper]] = [ # Mixdrop zrebrandował na m1xdrop.bz, yt-dlp out-of-date, packer/JS extract = fail. # Playmogo = DoodStream CAPTCHA. Porn-app sam olewa xmoviesforyou (brak handlera w # jadx). 1,321 solo-orphan scen. - # WatchPornScraper — wyłączony 2026-05-12 (user bug-report). Wszystkie iframes to - # DoodStream variants (playmogo/d0000d/dooood/mivalyo) z CAPTCHA gate. WebView na - # mobile = black screen (player JS nie inicjalizuje się przez Turnstile). 16% - # scen solo (no backup tube), 84% multi-source — user może użyć innego tube. yt-dlp - # nie wspiera DoodStream ("Piracy"), własny resolver TBD jeśli warto. + # WatchPornScraper — RE-ENABLE 2026-07-02, przeniesiony do ALL_BROWSE_SCRAPERS. + # Site przebudowany na KVS (DoodStream-CAPTCHA z 2026-05-12 zniknął); browse + # /latest-updates/ + flashvars get_file direct mp4 (extractor watchporn). Patrz watchporn.py. # SiskaScraper — przeniesiony do ALL_BROWSE_SCRAPERS (browse-konwersja 2026-06-20, # bo search siski zepsuty site-side — `?s=` ignoruje query). Patrz siska.py. # Porn4DaysScraper — wyłączony 2026-05-12 (post audit fix). 100% scen na streamtape @@ -144,6 +142,11 @@ from app.connectors.direct_scrapers.xvideos_browse import XVideosBrowseScraper ALL_BROWSE_SCRAPERS: list[type[BaseBrowseScraper]] = [ PerverzijaScraper, PornDishScraper, + # WatchPornScraper — re-enabled 2026-07-02 jako browse (site przebudowany na KVS, + # DoodStream-CAPTCHA zniknął). Browse /latest-updates/: og:title, JSON-LD duration/ + # uploadDate, /models/ performerzy, /tags/ tagi, /categories/ studio. Playback KVS + # get_file direct mp4 (extractor watchporn, VPS-side, token nie IP-bound). + WatchPornScraper, # Browse równolegle do istniejącego search scrapera (wzorzec xvideos/eporner): # search zostaje (pokrycie back-catalogu performerów), browse gwarantuje świeżość # wprost z feedu (watchdog 48h zamiast 168h). Konwersja 2026-06-24 (user request). diff --git a/app/connectors/direct_scrapers/watchporn.py b/app/connectors/direct_scrapers/watchporn.py index 5c3afd1..b95b6e9 100644 --- a/app/connectors/direct_scrapers/watchporn.py +++ b/app/connectors/direct_scrapers/watchporn.py @@ -1,19 +1,108 @@ -"""watchporn.to — direct HTML scrape. +"""watchporn.to — browse scraper (KVS engine). Re-enabled 2026-07-02. -Search: `https://watchporn.to/page//?s=` (WordPress). -Scene URL: `https://watchporn.to/videos//`. +Był search-scraperem (`?s=`), zamarzł, a potem site przebudowano na KVS z nowym +layoutem. DoodStream-CAPTCHA (powód wyłączenia 2026-05-12) zniknął — teraz KVS +flashvars `get_file` direct mp4 (extractor `watchporn`, VPS-side, token nie IP-bound). + +Browse `/latest-updates/` → detail page: + - title: og:title ("Studio/Creator - Scene Title") + - duration + release_date: JSON-LD "duration" (ISO) + "uploadDate" + - performerzy: `/models//` (pomijamy numeryczne id-slugi), nazwa z tekstu linku + - tagi: `/tags//` + - studio: pierwszy `/categories//` (np. EvilAngel, ManyVids) """ from __future__ import annotations +import html import re -from app.connectors.direct_scrapers._search_base import BaseSearchScraper +from app.connectors.base import ( + RawPerformer, + RawPlaybackSource, + RawScene, + RawStudio, + RawTag, +) +from app.connectors.direct_scrapers._browse_base import BaseBrowseScraper, meta_content +from app.connectors.direct_scrapers._playtube import _parse_iso_date, _parse_iso_duration +from app.normalize.text import slugify + +_BASE = "https://watchporn.to" +_SCENE_URL_RE = re.compile(r'href="(https://watchporn\.to/video/\d+/[a-z0-9\-]+/)"', re.IGNORECASE) +_MODEL_RE = re.compile(r'href="https://watchporn\.to/models/([a-z0-9\-]+)/"[^>]*>([^<]+)', re.IGNORECASE) +_TAG_RE = re.compile(r'href="https://watchporn\.to/tags/([a-z0-9\-]+)/"[^>]*>([^<]+)', re.IGNORECASE) +_CAT_RE = re.compile(r'href="https://watchporn\.to/categories/([a-z0-9\-]+)/"[^>]*>([^<]+)', re.IGNORECASE) +_DUR_RE = re.compile(r'"duration"\s*:\s*"([^"]+)"') +_UPLOAD_RE = re.compile(r'"uploadDate"\s*:\s*"([^"]+)"') -class WatchPornScraper(BaseSearchScraper): +class WatchPornScraper(BaseBrowseScraper): sitetag = "watchporn" - _search_url_template = "https://watchporn.to/page/{page}/?s={query}" - _scene_url_re = re.compile( - r'href="(?Phttps://watchporn\.to/videos/(?P[a-z0-9][a-z0-9\-]+))/"', - re.IGNORECASE, - ) + + def _listing_url(self, page: int) -> str: + return f"{_BASE}/latest-updates/" if page <= 1 else f"{_BASE}/latest-updates/{page}/" + + def _extract_scene_urls(self, listing_html: str) -> list[str]: + seen: set[str] = set() + out: list[str] = [] + for m in _SCENE_URL_RE.finditer(listing_html): + url = m.group(1) + if url not in seen: + seen.add(url) + out.append(url) + return out + + def _parse_detail(self, scene_url: str, detail_html: str) -> RawScene | None: + title = (meta_content(detail_html, property="og:title") or "").strip() + if not title: + return None + + dm = _DUR_RE.search(detail_html) + duration_sec = _parse_iso_duration(dm.group(1)) if dm else None + um = _UPLOAD_RE.search(detail_html) + release_date = _parse_iso_date(um.group(1)) if um else None + + performers: list[RawPerformer] = [] + seen_p: set[str] = set() + for m in _MODEL_RE.finditer(detail_html): + slug = m.group(1) + name = html.unescape(m.group(2)).strip() + if slug.isdigit() or slug in seen_p or not name: + continue + seen_p.add(slug) + performers.append(RawPerformer(external_id=f"{self.sitetag}:model:{slug}", name=name)) + + tags: list[RawTag] = [] + seen_t: set[str] = set() + for m in _TAG_RE.finditer(detail_html): + slug = m.group(1) + name = html.unescape(m.group(2)).strip() + if slug in seen_t or not name: + continue + seen_t.add(slug) + tags.append(RawTag(external_id=f"{self.sitetag}:tag:{slug}", name=name, slug=slug)) + + studio: RawStudio | None = None + cm = _CAT_RE.search(detail_html) + if cm: + cname = html.unescape(cm.group(2)).strip() + if cname: + studio = RawStudio( + external_id=f"{self.sitetag}:studio:{slugify(cname)}", name=cname, slug=slugify(cname) + ) + + return RawScene( + external_id=f"{self.sitetag}:{scene_url}", + title=title, + release_date=release_date, + duration_sec=duration_sec, + url=scene_url, + studio=studio, + performers=performers, + tags=tags, + playback_sources=[ + RawPlaybackSource( + origin=f"tube:{self.sitetag}", page_url=scene_url, duration_sec=duration_sec + ) + ], + ) diff --git a/app/extractors/__init__.py b/app/extractors/__init__.py index 870c96a..9a720f2 100644 --- a/app/extractors/__init__.py +++ b/app/extractors/__init__.py @@ -38,6 +38,7 @@ from app.extractors.tubes import ( porn00, porntrex, sxyprn, + watchporn, xhamster, yespornvip, ) @@ -96,7 +97,9 @@ _REGISTRY: dict[str, Callable[[str], list[StreamSource] | None]] = { # Aggregator tubes — generic embed-iframe → hoster unpacker "latestpornvideocom": latestpornvideo.extract, "xmoviesforyoucom": _embed_iframe.extract, - "watchporn": _embed_iframe.extract, + # watchporn — 2026-07-02 przebudowany na KVS (DoodStream-CAPTCHA zniknął). + # flashvars get_file direct mp4, same-session 302 resolve, token nie IP-bound. + "watchporn": watchporn.extract, "siskavideo": _embed_iframe.extract, "porn4dayspw": _embed_iframe.extract, "porndishcom": _embed_iframe.extract, diff --git a/app/extractors/tubes/watchporn.py b/app/extractors/tubes/watchporn.py new file mode 100644 index 0000000..af46628 --- /dev/null +++ b/app/extractors/tubes/watchporn.py @@ -0,0 +1,125 @@ +"""watchporn.to — KVS engine direct stream extractor (re-enabled 2026-07-02). + +Site przebudowany na KVS: detail page ma flashvars `video_url`/`video_alt_url*`, +każdy to `get_file///_p.mp4/?v-acctoken=...`. Ten sam wzorzec co +porntrex. get_file 302 → finalny CDN url; resolvujemy same-session (token bywa +cookie/session-bound) i oddajemy finalny url per jakość. Token nie IP-bound +(cross-IP 206 z VPS zweryfikowane 2026-07-02) → mobile gra direct, zero proxy. +Wcześniejszy DoodStream-CAPTCHA zniknął. +""" +from __future__ import annotations + +import logging +import re +import time + +from app.extractors._fetch import _DEFAULT_IMPERSONATE, _DEFAULT_UA, _HAS_CURL_CFFI, fetch_tube_html +from app.extractors._models import HosterDead, StreamSource + +log = logging.getLogger(__name__) + +_BASE = "https://watchporn.to" + +_DEAD_RE = re.compile( + r"this video (?:was|has been) deleted|video (?:was|has been) removed" + r"|no longer available|video is unavailable", + re.IGNORECASE, +) +_URL_RE = re.compile( + r"(video(?:_alt)?_url\d*)\s*:\s*'(https?://[^']+/get_file/[^']+)'", + re.IGNORECASE, +) +_TEXT_RE = re.compile( + r"(video(?:_alt)?_url\d*)_text\s*:\s*'([^']*)'", + re.IGNORECASE, +) + + +def _quality_rank(label: str | None) -> int: + if not label: + return -1 + m = re.search(r"(\d{3,4})\s*p", label, re.IGNORECASE) + return int(m.group(1)) if m else -1 + + +def _resolve_get_file(session, get_file_url: str, timeout: float) -> str | None: + sep = "&" if "?" in get_file_url else "?" + url = f"{get_file_url}{sep}rnd={int(time.time() * 1000)}" + try: + r = session.get( + url, + timeout=timeout, + allow_redirects=True, + stream=True, + headers={"Referer": _BASE + "/", "Range": "bytes=0-1"}, + ) + final = str(r.url) + status = r.status_code + r.close() + except Exception as e: + log.info("watchporn: get_file resolve failed (%s): %s", get_file_url[:60], e) + return None + if status >= 400 or "/get_file/" in final: + log.info("watchporn: get_file resolve bad status=%s final=%s", status, final[:70]) + return None + return final + + +def extract(page_url: str, *, timeout: float = 60.0) -> list[StreamSource] | None: + session = None + if _HAS_CURL_CFFI: + from curl_cffi import requests as _cf_requests + session = _cf_requests.Session(impersonate=_DEFAULT_IMPERSONATE) + try: + resp = session.get( + page_url, + headers={"User-Agent": _DEFAULT_UA, "Accept": "text/html,application/xhtml+xml"}, + timeout=timeout, + allow_redirects=True, + ) + html_text = resp.text if resp.status_code < 400 else "" + except Exception as e: + log.info("watchporn: page fetch failed %s: %s", page_url, e) + html_text = "" + if not html_text: + html_text = fetch_tube_html(page_url, timeout=timeout) + session = None + else: + html_text = fetch_tube_html(page_url, timeout=timeout) + + if html_text and _DEAD_RE.search(html_text): + raise HosterDead(f"watchporn {page_url}: video deleted/removed") + + quality_by_var: dict[str, str] = {} + for m in _TEXT_RE.finditer(html_text): + quality_by_var[m.group(1).lower()] = m.group(2).strip() + + seen: set[str] = set() + result: list[StreamSource] = [] + for m in _URL_RE.finditer(html_text): + var_name = m.group(1).lower() + url = m.group(2) + if url in seen: + continue + seen.add(url) + quality = quality_by_var.get(var_name) + final_link = url + if session is not None: + resolved = _resolve_get_file(session, url, timeout) + if resolved: + final_link = resolved + result.append( + StreamSource( + link=final_link, + type="mp4", + quality=quality or None, + referer=_BASE + "/", + raw={"mobile_direct_ok": True}, + ) + ) + + if not result: + log.info("watchporn: no KVS video_url in flashvars on %s", page_url) + return None + result.sort(key=lambda s: _quality_rank(s.quality), reverse=True) + return result