1ad16ceae9
Stary globalny regex /models/ gubil prawdziwa obsade (nazwa w spanie, pusty tekst linku) i dopinal performerow z kart related (srednio 12 na scene). Kategorie to teraz gatunki, nie studio; tagi bez przedrostka #.
170 lines
7.2 KiB
Python
170 lines
7.2 KiB
Python
"""watchporn.to, browse scraper (KVS engine). Re-enabled 2026-07-02.
|
|
|
|
Był search-scraperem (`?s=`), zamarzł, a potem site przebudowano na KVS z nowym
|
|
layoutem. DoodStream-CAPTCHA (powód wyłączenia 2026-05-12) zniknął, teraz KVS
|
|
flashvars `get_file` direct mp4 (extractor `watchporn`, VPS-side, token nie IP-bound).
|
|
|
|
Browse `/latest-updates/` → detail page:
|
|
- title: og:title ("Studio/Creator - Scene Title")
|
|
- duration + release_date: JSON-LD "duration" (ISO) + "uploadDate"
|
|
- performerzy: `/models/<slug>/` (pomijamy numeryczne id-slugi), nazwa z tekstu linku
|
|
- tagi: `/tags/<slug>/`
|
|
- studio: pierwszy `/categories/<slug>/` (np. EvilAngel, ManyVids)
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import html
|
|
import re
|
|
|
|
from app.connectors.base import (
|
|
RawPerformer,
|
|
RawPlaybackSource,
|
|
RawScene,
|
|
RawStudio,
|
|
RawTag,
|
|
)
|
|
from app.connectors.direct_scrapers._browse_base import BaseBrowseScraper, meta_content
|
|
from app.connectors.direct_scrapers._playtube import _parse_iso_date, _parse_iso_duration
|
|
from app.normalize.text import slugify
|
|
|
|
_BASE = "https://watchporn.to"
|
|
_SCENE_URL_RE = re.compile(r'href="(https://watchporn\.to/video/\d+/[a-z0-9\-]+/)"', re.IGNORECASE)
|
|
_MODEL_RE = re.compile(r'href="https://watchporn\.to/models/([a-z0-9\-]+)/"[^>]*>([^<]+)', re.IGNORECASE)
|
|
_TAG_RE = re.compile(r'href="https://watchporn\.to/tags/([a-z0-9\-]+)/"[^>]*>([^<]+)', re.IGNORECASE)
|
|
_CAT_RE = re.compile(r'href="https://watchporn\.to/categories/([a-z0-9\-]+)/"[^>]*>([^<]+)', re.IGNORECASE)
|
|
# Nowy motyw od ok. 2026-10-10 (prefiks klas `wp-`). Obsada sceny siedzi TYLKO w bloku
|
|
# `<div class="wp-castrow">`, nazwa w `<span class="wp-cast__n">` (nie w tekscie linku),
|
|
# a karty "related" ponizej maja wlasne linki `/models/` z nazwa w tekscie. Stary globalny
|
|
# regex gubil prawdziwa obsade (pusty tekst linku) i dokladal performerow z kart related:
|
|
# srednio 12,5 na scene zamiast 2 (10 do 13.10, 193 sceny). Kategorie to teraz gatunki
|
|
# (chipy `wp-vchip`), studio jest w linku `/sites/<slug>/` i w JSON-LD "publisher".
|
|
_NEW_THEME_MARK = 'class="wp-vi__panel"'
|
|
_CASTROW_RE = re.compile(r'<div class="wp-castrow">(.*?)</div>', re.IGNORECASE | re.DOTALL)
|
|
_CAST_RE = re.compile(
|
|
r'href="https://watchporn\.to/models/([a-z0-9\-]+)/"[^>]*>.*?<span class="wp-cast__n">([^<]+)</span>',
|
|
re.IGNORECASE | re.DOTALL,
|
|
)
|
|
_CHIPS_RE = re.compile(r'<div class="wp-chips">(.*?)</div>', re.IGNORECASE | re.DOTALL)
|
|
_SITE_RE = re.compile(r'href="https://watchporn\.to/sites/([a-z0-9\-]+)/"', re.IGNORECASE)
|
|
_PUBLISHER_RE = re.compile(r'"publisher"\s*:\s*\{[^}]*?"name"\s*:\s*"([^"]+)"', re.DOTALL)
|
|
_DUR_RE = re.compile(r'"duration"\s*:\s*"([^"]+)"')
|
|
_UPLOAD_RE = re.compile(r'"uploadDate"\s*:\s*"([^"]+)"')
|
|
_VIDEO_ID_RE = re.compile(r"/video/(\d+)/")
|
|
|
|
|
|
def _derive_thumb(scene_url: str) -> str | None:
|
|
"""KVS trzyma poster pod stałym wzorem `contents/videos_screenshots/<id//1000*1000>/
|
|
<id>/preview.jpg` (zweryfikowane). Fallback gdy detail page nie ma og:image, żeby
|
|
kafelek na liście miał miniaturkę OD RAZU (bez czekania na auto-enrich w SceneDetail)."""
|
|
m = _VIDEO_ID_RE.search(scene_url)
|
|
if not m:
|
|
return None
|
|
vid = int(m.group(1))
|
|
return f"{_BASE}/contents/videos_screenshots/{vid // 1000 * 1000}/{vid}/preview.jpg"
|
|
|
|
|
|
class WatchPornScraper(BaseBrowseScraper):
|
|
sitetag = "watchporn"
|
|
|
|
def _listing_url(self, page: int) -> str:
|
|
return f"{_BASE}/latest-updates/" if page <= 1 else f"{_BASE}/latest-updates/{page}/"
|
|
|
|
def _extract_scene_urls(self, listing_html: str) -> list[str]:
|
|
seen: set[str] = set()
|
|
out: list[str] = []
|
|
for m in _SCENE_URL_RE.finditer(listing_html):
|
|
url = m.group(1)
|
|
if url not in seen:
|
|
seen.add(url)
|
|
out.append(url)
|
|
return out
|
|
|
|
def _parse_detail(self, scene_url: str, detail_html: str) -> RawScene | None:
|
|
title = (meta_content(detail_html, property="og:title") or "").strip()
|
|
if not title:
|
|
return None
|
|
|
|
dm = _DUR_RE.search(detail_html)
|
|
duration_sec = _parse_iso_duration(dm.group(1)) if dm else None
|
|
um = _UPLOAD_RE.search(detail_html)
|
|
release_date = _parse_iso_date(um.group(1)) if um else None
|
|
|
|
new_theme = _NEW_THEME_MARK in detail_html
|
|
|
|
performers: list[RawPerformer] = []
|
|
seen_p: set[str] = set()
|
|
if new_theme:
|
|
# Tylko blok obsady; brak bloku = scena bez obsady, NIE wracamy do globalnego
|
|
# regexu, bo ten zbiera karty related.
|
|
crm = _CASTROW_RE.search(detail_html)
|
|
cast_iter = _CAST_RE.finditer(crm.group(1)) if crm else iter(())
|
|
else:
|
|
cast_iter = _MODEL_RE.finditer(detail_html)
|
|
for m in cast_iter:
|
|
slug = m.group(1)
|
|
name = html.unescape(m.group(2)).strip()
|
|
if slug.isdigit() or slug in seen_p or not name:
|
|
continue
|
|
seen_p.add(slug)
|
|
performers.append(RawPerformer(external_id=f"{self.sitetag}:model:{slug}", name=name))
|
|
|
|
tags: list[RawTag] = []
|
|
seen_t: set[str] = set()
|
|
|
|
def _add_tag(slug: str, raw_name: str) -> None:
|
|
name = html.unescape(raw_name).strip().lstrip("#").strip()
|
|
if slug in seen_t or not name:
|
|
return
|
|
seen_t.add(slug)
|
|
tags.append(RawTag(external_id=f"{self.sitetag}:tag:{slug}", name=name, slug=slug))
|
|
|
|
for m in _TAG_RE.finditer(detail_html):
|
|
_add_tag(m.group(1), m.group(2))
|
|
if new_theme:
|
|
# Kategorie-chipy sceny (Blowjob, MILF...) to gatunki, ida do tagow.
|
|
chm = _CHIPS_RE.search(detail_html)
|
|
if chm:
|
|
for m in _CAT_RE.finditer(chm.group(1)):
|
|
_add_tag(m.group(1), m.group(2))
|
|
|
|
thumbnail_url = (
|
|
meta_content(detail_html, property="og:image") or _derive_thumb(scene_url)
|
|
)
|
|
|
|
studio: RawStudio | None = None
|
|
if new_theme:
|
|
sm = _SITE_RE.search(detail_html)
|
|
if sm:
|
|
pm = _PUBLISHER_RE.search(detail_html)
|
|
cname = html.unescape(pm.group(1)).strip() if pm else sm.group(1).replace("-", " ").title()
|
|
if cname:
|
|
studio = RawStudio(
|
|
external_id=f"{self.sitetag}:studio:{slugify(cname)}", name=cname, slug=slugify(cname)
|
|
)
|
|
else:
|
|
cm = _CAT_RE.search(detail_html)
|
|
if cm:
|
|
cname = html.unescape(cm.group(2)).strip()
|
|
if cname:
|
|
studio = RawStudio(
|
|
external_id=f"{self.sitetag}:studio:{slugify(cname)}", name=cname, slug=slugify(cname)
|
|
)
|
|
|
|
return RawScene(
|
|
external_id=f"{self.sitetag}:{scene_url}",
|
|
title=title,
|
|
release_date=release_date,
|
|
duration_sec=duration_sec,
|
|
url=scene_url,
|
|
studio=studio,
|
|
performers=performers,
|
|
tags=tags,
|
|
playback_sources=[
|
|
RawPlaybackSource(
|
|
origin=f"tube:{self.sitetag}",
|
|
page_url=scene_url,
|
|
duration_sec=duration_sec,
|
|
thumbnail_url=thumbnail_url,
|
|
)
|
|
],
|
|
)
|