11400c01cb
- superporn: feed /videos/latest/N zamiast rotujacej strony glownej - pornbusy: karty ze strony glownej przed spoznion sitemapa - dooplay/streamporn: nowy motyw (card__th, h1 vid__t, data-servers JSON, st--duration, time datetime, chipy gatunkow), odstep i ponowienia w _fetch - porndoe: listing z sitemap_videos.xml; None gdy zadna strona sceny nie parsuje - hentaimama: /episodes/feed/ poza regexem odcinkow - watchdog: progi per zrodlo dla serwisow publikujacych rzadko
657 lines
27 KiB
Python
657 lines
27 KiB
Python
"""dooplay (a.k.a. PsyPlay) WordPress theme scraper, generic dla mangoporn/streamporn/pandamovies.
|
||
|
||
Te 3 strony to dokładnie ten sam template (theme=dooplay + PsyPlay player plugin),
|
||
więc parametryzujemy connector po `(base_url, source_name)` i odpalamy 3 instancje.
|
||
|
||
Listing: `/movies/page/N/` zwraca <a href="/movies/<slug>/"> per item.
|
||
Detail: `/movies/<slug>/` ma rich meta:
|
||
- <h1> tytuł (w class="data" wrapper)
|
||
- <a href="/year/YYYY/" rel="tag"> rok produkcji
|
||
- <a href="/studios/<slug>/" rel="tag"> studio
|
||
- <span class='duration'>NN mins.</span> długość
|
||
- <a href="/pornstar/<slug>/"> cast (multi)
|
||
- <a href="/genre/<slug>/"> tagi (multi)
|
||
- <div itemprop="description"><p>...</p></div> opis
|
||
- <span class="dt_rating_vgs" itemprop="ratingValue">N</span> rating 0-10
|
||
- <li ... data-fl-source="<embed_url>"><a href="<embed_link>">Host</a></li> player options
|
||
|
||
Player ma multi-host options (DoodStream, LuluStream, RPMShare etc.), każdy embed
|
||
URL idzie jako osobny `playback_source` z origin=`{site}:{host}` żeby później mobile
|
||
mógł wybrać czyim embedem chce odpalić scenę.
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import logging
|
||
import re
|
||
import time
|
||
from collections.abc import Iterator
|
||
from datetime import date, datetime
|
||
from typing import Any
|
||
|
||
import httpx
|
||
|
||
from app.connectors.base import (
|
||
BaseMovieConnector,
|
||
RawMovie,
|
||
RawPerformer,
|
||
RawPlaybackSource,
|
||
RawStudio,
|
||
RawTag,
|
||
)
|
||
from app.extractors import browser_get
|
||
from app.models.source import SourceKind
|
||
|
||
log = logging.getLogger(__name__)
|
||
|
||
USER_AGENT = (
|
||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||
"(KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36"
|
||
)
|
||
|
||
|
||
# ---- selektory (theme-agnostic, działa dla dowolnego dooplay) -----------
|
||
|
||
# Listing item, dwa wzorce w zależności od witryny:
|
||
# 1. mangoporn: zwykłe <a href="https://site/movies/<slug>/"> bez klasy
|
||
# (theme wyrendurował SEO-friendly URL bezpośrednio w grid)
|
||
# 2. streamporn/pandamovies: <a class="ml-mask jt" href="<base>/<slug>/">
|
||
# (slug bez /movies/ prefix, np. /watch-xxx-...-adult-movie-online-free/)
|
||
# Łapiemy oba przez alternatywę.
|
||
# 2026-10-02 mangoporn przeszedl na `<a class="thumb" href=...>` (klasa PRZED href) i
|
||
# wzorzec z href na poczatku nie lapal juz nic: 0 filmow od 2026-09-23. Teraz href moze
|
||
# stac w dowolnym miejscu znacznika; linki menu do /movies/category/ odpadaja.
|
||
# 2026-10-13 streamporn.li zmienil motyw: karta to `<article class="card"><a class="card__th"
|
||
# href=...>` (klasa PRZED href, slug bez /movies/), stary wzorzec dawal 0 linkow od 2.10.
|
||
_LIST_ITEM_RE = re.compile(
|
||
r'<a\s+href="(?P<url>https?://[^"]+)"[^>]*\bclass="ml-mask\b[^"]*"'
|
||
r"|"
|
||
r'<a\s+(?:[^>]*?\s)?href="(?P<url2>https?://[^"]+/movies/(?!category/)[a-z0-9-]+/)"'
|
||
r"|"
|
||
r'<a\s+class="(?:card__th|ml-mask)[^"]*"\s+href="(?P<url3>https?://[^"]+)"',
|
||
re.IGNORECASE,
|
||
)
|
||
# Tolerantny title, mangoporn (dooplay) używa <h1> w class="data", streamporn/pandamovies
|
||
# (raw PsyPlay theme) używają <h3 itemprop="name">. Łapiemy oba przez itemprop="name".
|
||
_TITLE_RE = re.compile(
|
||
r'<h[1-6][^>]*\sitemprop="name"[^>]*>([^<]+)</h[1-6]>'
|
||
r'|class="data"[^>]*>\s*<h[1-6][^>]*>([^<]+)</h[1-6]>',
|
||
re.IGNORECASE | re.DOTALL,
|
||
)
|
||
# Zapas: mangoporn od 2026-10 ma tytul w golym <h1>, bez itemprop i bez class="data".
|
||
# Tylko gdy dwa wzorce wyzej nic nie znajda, zeby inne motywy nie braly logo z <h1>.
|
||
_H1_TITLE_RE = re.compile(r"<h1[^>]*>\s*([^<]{2,300}?)\s*</h1>", re.IGNORECASE)
|
||
# dooplay uses /year/, raw PsyPlay uses /release-year/. Same dla pozostałych slugów,
|
||
# różne thema dziedziczą podstawowy markup ale customizują URL słowniki.
|
||
_YEAR_RE = re.compile(
|
||
r'/(?:year|release-year)/(\d{4})/"\s*rel="tag"', re.IGNORECASE
|
||
)
|
||
_STUDIO_RE = re.compile(
|
||
r'href="https?://[^/]+/(?:studios?|director)/([a-z0-9-]+)/"\s+rel="tag"[^>]*>([^<]+)</a>',
|
||
re.IGNORECASE,
|
||
)
|
||
# Duration: span class='duration' (dooplay) lub <p><strong>Duration:</strong> X hrs. Y mins.</p> (PsyPlay)
|
||
_DURATION_SPAN_RE = re.compile(
|
||
r"<span\s+class=['\"]duration['\"][^>]*>([^<]+)</span>", re.IGNORECASE
|
||
)
|
||
_DURATION_TEXT_RE = re.compile(
|
||
r"<strong>\s*Duration:\s*</strong>\s*([^<]+)<", re.IGNORECASE
|
||
)
|
||
# Release date: span class='release_date' (dooplay) lub <p><strong>Released Date:</strong> X</p> (PsyPlay)
|
||
_RELEASE_DATE_SPAN_RE = re.compile(
|
||
r"<span\s+class=['\"]release_date['\"]'?[^>]*>([^<]+)</span>", re.IGNORECASE
|
||
)
|
||
_RELEASE_DATE_TEXT_RE = re.compile(
|
||
r"<strong>\s*Released?\s*Date:\s*</strong>\s*([^<]+)<", re.IGNORECASE
|
||
)
|
||
_DESCRIPTION_RE = re.compile(
|
||
r'itemprop="description"[^>]*>(.*?)</div>', re.IGNORECASE | re.DOTALL
|
||
)
|
||
_RATING_RE = re.compile(
|
||
r'itemprop="ratingValue"[^>]*>([\d.]+)</span>', re.IGNORECASE
|
||
)
|
||
# Cast: dooplay /pornstar/, PsyPlay /actor/
|
||
_PORNSTAR_RE = re.compile(
|
||
r'href="https?://[^/]+/(?:pornstar|actor)/([a-z0-9-]+)/"\s+rel="tag"[^>]*>([^<]+)</a>',
|
||
re.IGNORECASE,
|
||
)
|
||
# Genre: same /genre(s)/ w obu themach
|
||
_GENRE_TAG_RE = re.compile(
|
||
r'href="https?://[^/]+/genres?/([a-z0-9-]+)/"\s+rel="tag"[^>]*>([^<]+)</a>',
|
||
re.IGNORECASE,
|
||
)
|
||
# Player options: data-fl-source jest oryginalnym embed URL hostera, data-fl-url
|
||
# to page URL u hostera. Stare theme (mangoporn): `<li class="hosts-buttons-wpx">`.
|
||
# Nowe theme (pandamovies od ~2026-04): `<div class="Rtable1-cell" data-fl-url=...
|
||
# data-fl-source=...>`. Trzeba też tolerować order-independent attrs, nowe theme
|
||
# emituje url BEFORE source, stare odwrotnie. Łapiemy oba wzorce dwoma osobnymi
|
||
# regexami i konsolidujemy w `_iter_player_options`.
|
||
_PLAYER_OPTION_RE = re.compile(
|
||
r'<li[^>]*\bclass="hosts-buttons-wpx"[^>]*'
|
||
r'(?:data-fl-source="(?P<source>[^"]*)"[^>]*)?'
|
||
r'(?:data-fl-url="(?P<page>[^"]*)"[^>]*)?'
|
||
r'>\s*<a[^>]*href="(?P<href>[^"]+)"[^>]*'
|
||
r'(?:[^<]*<img[^>]+>)?\s*([^<]+?)\s*</a>',
|
||
re.IGNORECASE | re.DOTALL,
|
||
)
|
||
# Nowy markup pandamovies: `<div class="Rtable1-cell" data-fl-* ...><a href=...>HostName</a></div>`.
|
||
# Attrs są w kolejności url→source, source często pusty (`data-fl-source=""` dla
|
||
# doodstream/mixdrop/easyvidplayer). Capturujemy CAŁY opening tag w group(1)
|
||
# żeby data-fl-source należał gwarantowanie do TEGO konkretnego div (wcześniejszy
|
||
# window-lookback 600 chars mógł pickować poprzedni cell, cross-attribution
|
||
# doodstream→mixdrop entry, code-review #14).
|
||
_PLAYER_OPTION_DIV_RE = re.compile(
|
||
r'(<div[^>]*\bclass="Rtable1-cell"[^>]*>)\s*'
|
||
r'<a[^>]*href="(?P<href>[^"]+)"[^>]*'
|
||
r'(?:[^<]*<img[^>]+>)?\s*([^<]+?)\s*</a>',
|
||
re.IGNORECASE | re.DOTALL,
|
||
)
|
||
_DATA_FL_SOURCE_RE = re.compile(r'data-fl-source="([^"]*)"', re.IGNORECASE)
|
||
# Poster, JSON-LD `thumbnailUrl` jest najbardziej stabilny (każdy dooplay/PsyPlay
|
||
# theme z SEO ma JSON-LD VideoObject schema). Fallback na class="poster" img dla starych
|
||
# instalacji bez schema. Trzeci fallback: og:image meta tag.
|
||
_POSTER_JSONLD_RE = re.compile(
|
||
r'"thumbnailUrl"\s*:\s*"([^"]+\.(?:jpg|jpeg|png|webp)[^"]*)"', re.IGNORECASE
|
||
)
|
||
_POSTER_RE = re.compile(
|
||
r'class="poster"[^>]*>\s*<img\s+[^>]*src="([^"]+)"', re.IGNORECASE
|
||
)
|
||
_POSTER_OG_RE = re.compile(
|
||
r'<meta\s+property="og:image"\s+content="([^"]+)"', re.IGNORECASE
|
||
)
|
||
_DURATION_MINS_RE = re.compile(r"(\d+)\s*min", re.IGNORECASE)
|
||
|
||
# Motyw streamporn.li od 2026-10 (wtyczka host-links-manager). Tytul siedzi w
|
||
# `<h1 class="vid__t"><span>Watch TYTUL ROK by STUDIO Porn Movie Online Free</span></h1>`,
|
||
# pasek `st st--duration` ma MM:SS, data w `<time datetime=...>`, gatunki to
|
||
# `<a class="chip" href=".../genre/<slug>/">` bez rel="tag", a hostery nie sa linkami
|
||
# w HTML, tylko JSON-em w atrybucie `data-servers` sekcji `<section class="hlm">`:
|
||
# [{"u": "https://playmogo.com/e/..", "t": "iframe", "l": "DoodStream"}, ...].
|
||
_VID_TITLE_RE = re.compile(
|
||
r'<h1[^>]*\bclass="vid__t[^"]*"[^>]*>\s*(?:<span[^>]*>)?\s*(.+?)\s*(?:</span>)?\s*</h1>',
|
||
re.IGNORECASE | re.DOTALL,
|
||
)
|
||
_VID_TITLE_PARTS_RE = re.compile(
|
||
r"^(?:Watch\s+)?(?P<title>.+?)"
|
||
r"(?:\s+(?P<year>(?:19|20)\d{2})\s+by\s+(?P<studio>.+?))?"
|
||
r"(?:\s+Porn\s+Movie\s+Online\s+Free)?\s*$",
|
||
re.IGNORECASE | re.DOTALL,
|
||
)
|
||
_ST_DURATION_RE = re.compile(
|
||
r'class="st st--duration"[^>]*>[^<]*(?:<svg.*?</svg>)?\s*(\d{1,2}):(\d{2})(?::(\d{2}))?\s*<',
|
||
re.IGNORECASE | re.DOTALL,
|
||
)
|
||
_ST_DATE_RE = re.compile(
|
||
r'class="st st--date"[^>]*>.*?<time\s+datetime="(\d{4}-\d{2}-\d{2})', re.IGNORECASE | re.DOTALL
|
||
)
|
||
_YEAR_LOOSE_RE = re.compile(r'/(?:year|release-year)/(\d{4})/"', re.IGNORECASE)
|
||
_GENRE_CHIP_RE = re.compile(
|
||
r'<a\s+class="chip"\s+href="https?://[^/]+/genres?/([a-z0-9-]+)/"[^>]*>([^<]+)</a>',
|
||
re.IGNORECASE,
|
||
)
|
||
_DATA_SERVERS_RE = re.compile(r'data-servers="([^"]*)"', re.IGNORECASE)
|
||
|
||
|
||
def _slug(name: str) -> str:
|
||
return re.sub(r"-{2,}", "-", re.sub(r"[^a-z0-9]+", "-", name.lower())).strip("-")
|
||
|
||
|
||
class DooplayConnector(BaseMovieConnector):
|
||
"""Generic dooplay scraper. Instantiated per-site via subclasses below."""
|
||
|
||
kind = SourceKind.scraper
|
||
base_url: str
|
||
name: str
|
||
|
||
def __init__(self, *, timeout: float = 30.0):
|
||
if not getattr(self, "base_url", None):
|
||
raise RuntimeError(f"{type(self).__name__} requires class-level `base_url`")
|
||
if not getattr(self, "name", None):
|
||
raise RuntimeError(f"{type(self).__name__} requires class-level `name`")
|
||
self._timeout = timeout
|
||
self._last_fetch_at = 0.0
|
||
|
||
# Odstep miedzy zapytaniami do jednej strony. streamporn.li (2026-10-13) odpowiada
|
||
# na pojedyncze zapytanie w 0,2 s, ale drugie zapytanie zaraz po pierwszym wisialo
|
||
# do timeoutu (30 s, curl 28), a wyjatek curl_cffi nie jest httpx.HTTPError, wiec
|
||
# wywracal caly przebieg ingestu po pierwszym filmie.
|
||
_pace_sec = 1.0
|
||
_fetch_attempts = 3
|
||
|
||
def close(self) -> None:
|
||
pass
|
||
|
||
def _fetch(self, url: str) -> str:
|
||
"""browser_get z chrome120 impersonation, psyplay sites czasem blokują
|
||
czysty httpx (Python TLS fingerprint) zwracając 500/403. curl_cffi fixuje to.
|
||
Timeout i bledy transportu: do `_fetch_attempts` prob z rosnacym odstepem, potem
|
||
httpx.TransportError, zeby petla w fetch_movies pominela film zamiast paść."""
|
||
if not url.startswith("http"):
|
||
url = self.base_url.rstrip("/") + url
|
||
headers = {
|
||
"User-Agent": USER_AGENT,
|
||
"Accept-Language": "en-US,en;q=0.9",
|
||
"Accept": "text/html,application/xhtml+xml",
|
||
"Referer": self.base_url + "/",
|
||
}
|
||
last_exc: Exception | None = None
|
||
r = None
|
||
for attempt in range(self._fetch_attempts):
|
||
wait = self._pace_sec - (time.monotonic() - self._last_fetch_at)
|
||
if wait > 0:
|
||
time.sleep(wait)
|
||
try:
|
||
r = browser_get(url, headers=headers, timeout=self._timeout, follow_redirects=True)
|
||
self._last_fetch_at = time.monotonic()
|
||
break
|
||
except Exception as e: # noqa: BLE001 - curl_cffi/httpx, rozne klasy
|
||
last_exc = e
|
||
self._last_fetch_at = time.monotonic()
|
||
log.info("%s: fetch attempt %d failed for %s: %s", self.name, attempt + 1, url, e)
|
||
time.sleep(2.0 * (attempt + 1))
|
||
if r is None:
|
||
raise httpx.TransportError(f"fetch failed after {self._fetch_attempts} attempts: {url}: {last_exc}")
|
||
if r.status_code >= 400:
|
||
raise httpx.HTTPStatusError(
|
||
f"{r.status_code} for {url}",
|
||
request=None, # type: ignore[arg-type]
|
||
response=httpx.Response(r.status_code, text=r.text[:200]),
|
||
)
|
||
return r.text
|
||
|
||
# Bezpiecznik, dooplay listing potrafi mieć tysiące stron (streamporn.nl ma
|
||
# >2k filmów). Bez tego ingest wisi godzinami, jest killowany przy restartcie
|
||
# schedulera, blokując kolejne connectory w queue (bug-report 2026-05-28: "od
|
||
# wczoraj nie ma nowych filmów", streamporn wisiał od 5-24, blokował
|
||
# mangoporn + pandamovies). Listing jest sortowany po dacie uploadu (NIE
|
||
# release_date filmu, release może być z 2013 a upload z dziś), ale upload
|
||
# date nie jest w markupie, więc filtrowanie po `since` przez release_date
|
||
# nie działa. Pragmatyczny cap stron: 3 dla delta (≈150 nowych pozycji/dzień
|
||
# to znacznie powyżej realnego upload-rate), 50 dla full ingestu (`since=None`).
|
||
_MAX_PAGES_DELTA = 3
|
||
_MAX_PAGES_FULL = 50
|
||
|
||
def fetch_movies(
|
||
self,
|
||
*,
|
||
since: datetime | None = None,
|
||
limit: int | None = None,
|
||
) -> Iterator[RawMovie]:
|
||
seen = 0
|
||
page = 1
|
||
seen_urls: set[str] = set()
|
||
max_pages = self._MAX_PAGES_DELTA if since is not None else self._MAX_PAGES_FULL
|
||
while page <= max_pages:
|
||
try:
|
||
urls = list(self._fetch_listing(page))
|
||
except httpx.HTTPError as e:
|
||
log.warning("%s listing page=%d failed: %s", self.name, page, e)
|
||
return
|
||
if not urls:
|
||
log.info("%s: empty page=%d, stop", self.name, page)
|
||
return
|
||
for url in urls:
|
||
if url in seen_urls:
|
||
continue
|
||
seen_urls.add(url)
|
||
try:
|
||
movie = self._fetch_detail(url)
|
||
except httpx.HTTPError as e:
|
||
log.warning("%s detail %s failed: %s", self.name, url, e)
|
||
continue
|
||
if movie is None:
|
||
continue
|
||
yield movie
|
||
seen += 1
|
||
if limit is not None and seen >= limit:
|
||
return
|
||
page += 1
|
||
log.info(
|
||
"%s: hit max_pages=%d cap (delta=%s), stopping after seen=%d",
|
||
self.name, max_pages, since is not None, seen,
|
||
)
|
||
|
||
def _fetch_listing(self, page: int) -> Iterator[str]:
|
||
path = self._listing_path(page)
|
||
text = self._fetch(path)
|
||
from urllib.parse import urlparse
|
||
site_host = urlparse(self.base_url).hostname
|
||
for m in _LIST_ITEM_RE.finditer(text):
|
||
url = m.group("url") or m.group("url2") or m.group("url3")
|
||
if not url:
|
||
continue
|
||
try:
|
||
if urlparse(url).hostname != site_host:
|
||
continue
|
||
except Exception:
|
||
continue
|
||
yield url
|
||
|
||
def _listing_path(self, page: int) -> str:
|
||
return "/movies/" if page == 1 else f"/movies/page/{page}/"
|
||
|
||
def _fetch_detail(self, url: str) -> RawMovie | None:
|
||
from urllib.parse import urlparse
|
||
path = urlparse(url).path.rstrip("/")
|
||
slug = path.split("/")[-1] or "root"
|
||
text = self._fetch(url)
|
||
return _parse_dooplay_detail(
|
||
slug=slug, page_url=url, html=text,
|
||
source_name=self.name, base_url=self.base_url,
|
||
)
|
||
|
||
|
||
def _parse_dooplay_detail(
|
||
*, slug: str, html: str, source_name: str, base_url: str, page_url: str | None = None
|
||
) -> RawMovie | None:
|
||
vid_year: int | None = None
|
||
vid_studio: str | None = None
|
||
m_title = _TITLE_RE.search(html) or _H1_TITLE_RE.search(html)
|
||
if m_title:
|
||
title = _decode_html(next(g for g in m_title.groups() if g).strip())
|
||
else:
|
||
# Motyw streamporn.li 2026-10: h1 z zagniezdzonym <span>, tytul, rok i studio w
|
||
# jednym napisie ("Watch TYTUL 2026 by STUDIO Porn Movie Online Free").
|
||
m_vid = _VID_TITLE_RE.search(html)
|
||
if not m_vid:
|
||
log.warning("%s: no title in %s", source_name, slug)
|
||
return None
|
||
raw_title = _decode_html(_strip_tags(m_vid.group(1))).strip()
|
||
m_parts = _VID_TITLE_PARTS_RE.match(raw_title)
|
||
title = (m_parts.group("title") if m_parts else raw_title).strip()
|
||
if m_parts and m_parts.group("year"):
|
||
vid_year = int(m_parts.group("year"))
|
||
if m_parts and m_parts.group("studio"):
|
||
vid_studio = m_parts.group("studio").strip()
|
||
if not title:
|
||
return None
|
||
|
||
m_year = _YEAR_RE.search(html) or _YEAR_LOOSE_RE.search(html)
|
||
release_year = int(m_year.group(1)) if m_year else vid_year
|
||
|
||
studio: RawStudio | None = None
|
||
m_studio = _STUDIO_RE.search(html)
|
||
if m_studio:
|
||
studio_slug = m_studio.group(1)
|
||
studio_name = _decode_html(m_studio.group(2).strip())
|
||
studio = RawStudio(
|
||
external_id=f"{source_name}:{studio_slug}",
|
||
name=studio_name,
|
||
slug=studio_slug,
|
||
)
|
||
elif vid_studio:
|
||
studio_slug = _slug(vid_studio)
|
||
if studio_slug:
|
||
studio = RawStudio(
|
||
external_id=f"{source_name}:{studio_slug}",
|
||
name=vid_studio,
|
||
slug=studio_slug,
|
||
)
|
||
|
||
duration_sec: int | None = None
|
||
m_dur = _DURATION_SPAN_RE.search(html) or _DURATION_TEXT_RE.search(html)
|
||
if m_dur:
|
||
text = m_dur.group(1)
|
||
# Może być "32 mins." (dooplay) albo "1 hrs. 12 mins." (PsyPlay)
|
||
m_h = re.search(r"(\d+)\s*hr", text, re.IGNORECASE)
|
||
m_m = re.search(r"(\d+)\s*min", text, re.IGNORECASE)
|
||
if m_h or m_m:
|
||
duration_sec = (int(m_h.group(1)) * 3600 if m_h else 0) + (int(m_m.group(1)) * 60 if m_m else 0)
|
||
if duration_sec is None:
|
||
# streamporn.li 2026-10: pasek "29:00" (MM:SS) albo "1:29:00" (H:MM:SS).
|
||
m_st = _ST_DURATION_RE.search(html)
|
||
if m_st:
|
||
a, b, c = m_st.group(1), m_st.group(2), m_st.group(3)
|
||
if c is not None:
|
||
duration_sec = int(a) * 3600 + int(b) * 60 + int(c)
|
||
else:
|
||
duration_sec = int(a) * 60 + int(b)
|
||
|
||
release_date: date | None = None
|
||
m_rd = _RELEASE_DATE_SPAN_RE.search(html) or _RELEASE_DATE_TEXT_RE.search(html)
|
||
if m_rd:
|
||
text = m_rd.group(1).strip()
|
||
for fmt in ("%B %d, %Y", "%b %d, %Y", "%Y-%m-%d"):
|
||
try:
|
||
release_date = datetime.strptime(text, fmt).date()
|
||
break
|
||
except ValueError:
|
||
continue
|
||
if release_date is None:
|
||
m_sd = _ST_DATE_RE.search(html)
|
||
if m_sd:
|
||
try:
|
||
release_date = datetime.strptime(m_sd.group(1), "%Y-%m-%d").date()
|
||
except ValueError:
|
||
pass
|
||
|
||
description: str | None = None
|
||
m_desc = _DESCRIPTION_RE.search(html)
|
||
if m_desc:
|
||
description = _decode_html(_strip_tags(m_desc.group(1))).strip() or None
|
||
|
||
rating: float | None = None
|
||
m_rating = _RATING_RE.search(html)
|
||
if m_rating:
|
||
try:
|
||
rating = float(m_rating.group(1))
|
||
except ValueError:
|
||
pass
|
||
|
||
poster_url: str | None = None
|
||
for rgx in (_POSTER_JSONLD_RE, _POSTER_RE, _POSTER_OG_RE):
|
||
m = rgx.search(html)
|
||
if m:
|
||
candidate = m.group(1).strip()
|
||
if candidate and "blank.gif" not in candidate and "no-poster" not in candidate:
|
||
poster_url = candidate
|
||
break
|
||
|
||
# Performers, tylko sekcja "Pornstars" ma /pornstar/<slug>/ linki, dooplay
|
||
# filtruje cast w tej sekcji. Jaccard może łapać dubel ale dedup robimy w
|
||
# resolverze (po performer_id).
|
||
performers = [
|
||
RawPerformer(
|
||
external_id=f"{source_name}:{m.group(1)}",
|
||
name=_decode_html(m.group(2).strip()),
|
||
)
|
||
for m in _PORNSTAR_RE.finditer(html)
|
||
]
|
||
|
||
tags = [
|
||
RawTag(
|
||
external_id=f"{source_name}:{m.group(1)}",
|
||
name=_decode_html(m.group(2).strip()),
|
||
slug=m.group(1),
|
||
)
|
||
for m in _GENRE_TAG_RE.finditer(html)
|
||
]
|
||
if not tags:
|
||
# streamporn.li 2026-10: chipy gatunkow bez rel="tag".
|
||
tags = [
|
||
RawTag(
|
||
external_id=f"{source_name}:{m.group(1)}",
|
||
name=_decode_html(m.group(2).strip()),
|
||
slug=m.group(1),
|
||
)
|
||
for m in _GENRE_CHIP_RE.finditer(html)
|
||
]
|
||
|
||
if page_url is None:
|
||
page_url = f"{base_url}/movies/{slug}/"
|
||
|
||
# Playback sources: każdy host (Doodstream/Lulu/RPM/...) jako osobny entry.
|
||
# Dedup po href żeby ten sam host nie wpadł 2x. Raw landing page (origin=
|
||
# source_name, bez :host) appendujemy TYLKO gdy nie ma żadnych sub-hosters,
|
||
# inaczej myli usera (otwiera WebView z reklamami zamiast video; bug-report
|
||
# 2026-05-16: "mangoporn przekierowuje do strony, reklama full screen").
|
||
playback_sources: list[RawPlaybackSource] = []
|
||
seen_hrefs: set[str] = set()
|
||
|
||
# Hostery file-download (non-streamable) + malware. Mobile player nie potrafi
|
||
# ich odtworzyć, rapidgator/nitroflare/frdl serwują .zip/.rar/.mp4 do download
|
||
# (premium login required), streamtape ma malware drive-by .reg. Skipujemy
|
||
# przy ingest żeby nie zaśmiecać UI martwym contentem (bug-report 2026-05-18).
|
||
SKIP_HOSTERS = {"rapidgator", "nitroflare", "nitro", "frdl", "streamtape"}
|
||
|
||
def _emit_host_entry(href: str, source: str | None) -> None:
|
||
href = href.strip()
|
||
if not href or href in seen_hrefs:
|
||
return
|
||
seen_hrefs.add(href)
|
||
try:
|
||
from urllib.parse import urlparse
|
||
host = urlparse(href).hostname or "unknown"
|
||
host_short = host.split(".")[-2] if host.count(".") >= 1 else host
|
||
except Exception:
|
||
host_short = "unknown"
|
||
if host_short.lower() in SKIP_HOSTERS:
|
||
return
|
||
playback_sources.append(
|
||
RawPlaybackSource(
|
||
origin=f"{source_name}:{host_short}",
|
||
page_url=href,
|
||
embed_url=source or href,
|
||
thumbnail_url=poster_url,
|
||
duration_sec=duration_sec,
|
||
)
|
||
)
|
||
|
||
# Stary `<li class="hosts-buttons-wpx">` markup (mangoporn).
|
||
for m in _PLAYER_OPTION_RE.finditer(html):
|
||
_emit_host_entry(m.group("href") or "", (m.group("source") or "").strip() or None)
|
||
|
||
# Nowy `<div class="Rtable1-cell">` markup (pandamovies od ~2026-04 + nowe
|
||
# streamporn instances). data-fl-source jest opcjonalny, capturujemy CAŁY
|
||
# opening tag w group(1), data-fl-source extract z TEGO tagu (nie z window
|
||
# lookback po HTMLu, bo to mogło pickować poprzedni cell).
|
||
for m in _PLAYER_OPTION_DIV_RE.finditer(html):
|
||
href = m.group("href") or ""
|
||
opening_tag = m.group(1)
|
||
src_match = _DATA_FL_SOURCE_RE.search(opening_tag)
|
||
source = (src_match.group(1).strip() if src_match else "") or None
|
||
_emit_host_entry(href, source)
|
||
|
||
# Motyw streamporn.li 2026-10 (host-links-manager): hostery w JSON-ie atrybutu
|
||
# data-servers, HTML-escapowanym ("). "u" = adres embed, "l" = etykieta.
|
||
for m in _DATA_SERVERS_RE.finditer(html):
|
||
try:
|
||
servers = json.loads(_decode_html(m.group(1)))
|
||
except (ValueError, TypeError):
|
||
continue
|
||
if not isinstance(servers, list):
|
||
continue
|
||
for entry in servers:
|
||
if not isinstance(entry, dict):
|
||
continue
|
||
embed = str(entry.get("u") or "").strip()
|
||
if embed.startswith("http"):
|
||
_emit_host_entry(embed, embed)
|
||
|
||
if not playback_sources:
|
||
# Brak sub-hosters znalezionych, fallback do landing page (mobile otworzy
|
||
# w WebView). Robimy to TYLKO gdy nie ma alternatyw, inaczej landing jest
|
||
# niepotrzebnym ad-pageiem.
|
||
playback_sources.append(
|
||
RawPlaybackSource(
|
||
origin=source_name,
|
||
page_url=page_url,
|
||
thumbnail_url=poster_url,
|
||
)
|
||
)
|
||
|
||
return RawMovie(
|
||
external_id=slug,
|
||
title=title,
|
||
description=description,
|
||
release_year=release_year,
|
||
release_date=release_date,
|
||
duration_sec=duration_sec,
|
||
rating=rating,
|
||
poster_url=poster_url,
|
||
url=page_url,
|
||
studio=studio,
|
||
performers=performers,
|
||
tags=tags,
|
||
playback_sources=playback_sources,
|
||
raw={"slug": slug, "html_len": len(html)},
|
||
)
|
||
|
||
|
||
# ---- per-site instances ----------------------------------------------------
|
||
|
||
class StreampornConnector(DooplayConnector):
|
||
name = "streamporn"
|
||
base_url = "https://streamporn.nl"
|
||
|
||
|
||
class StreampornVipConnector(DooplayConnector):
|
||
# streamporn.nl padł (stale od 2026-05-23); streamporn.vip to żywy sibling z tym
|
||
# samym PsyPlay theme i świeższym katalogiem (ocena 2026-07-02). Osobny source_name
|
||
# → nowe origins `streampornvip:<host>`, nie miesza z martwym .nl.
|
||
name = "streampornvip"
|
||
# 2026-10-02 streamporn.vip -> 302 streamporn.li. Konektor odrzuca linki spoza swojej
|
||
# domeny, wiec po przekierowaniu lista byla pusta (0 filmow od 2026-09-23). Id filmu to
|
||
# slug, bez domeny, wiec przeprowadzka nie robi duplikatow.
|
||
base_url = "https://streamporn.li"
|
||
|
||
def _listing_path(self, page: int) -> str:
|
||
"""Strona główna, NIE `/movies/`. Diagnoza 2026-08-03: źródło stało od
|
||
2026-07-11 (0 nowych przez 23 dni), ale strona żyła i konektor parsował
|
||
poprawnie, tyle że `/movies/` na tym motywie NIE jest sortowane po dacie i
|
||
oddaje stary katalog. Zmierzone na tych samych 8 pozycjach: `/movies/` dało
|
||
roczniki 2012-2015, strona główna wyłącznie 2026 (po 10-11 źródeł playbacku).
|
||
`?orderby=date` motyw ignoruje (odpowiedź identyczna z domyślną).
|
||
|
||
To ta sama klasa błędu co przy freshporno: właściwa strona, zły endpoint
|
||
listingu, patrz [[reference_kvs_root_rotates_use_latest_updates]].
|
||
|
||
Strony 1 i 2 współdzielą ~44 linki (sidebar/widgety), ale treść główna jest
|
||
stronicowana normalnie; dedup po URL w `fetch_movies` i tak je odsiewa.
|
||
"""
|
||
return "/" if page == 1 else f"/page/{page}/"
|
||
|
||
|
||
class PandamoviesConnector(DooplayConnector):
|
||
name = "pandamovies"
|
||
base_url = "https://pandamovies.pw"
|
||
|
||
|
||
class MangopornConnector(DooplayConnector):
|
||
name = "mangoporn"
|
||
base_url = "https://mangoporn.net"
|
||
|
||
def __init__(self, *, timeout: float = 75.0):
|
||
# Strona odpowiada z VPS po 30-60 s (2026-10-02), przy 30 s ingest konczyl sie
|
||
# na timeoutcie listy.
|
||
super().__init__(timeout=timeout)
|
||
|
||
def _listing_path(self, page: int) -> str:
|
||
"""Od 2026-10 `/movies/` daje 404, najnowsze sa na stronie glownej (`/page/N/`)."""
|
||
return "/" if page == 1 else f"/page/{page}/"
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Helpers (zduplikowane z paradisehill.py, celowo, żeby connectory były niezależne)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
_TAG_RE = re.compile(r"<[^>]+>")
|
||
|
||
|
||
def _strip_tags(s: str) -> str:
|
||
return _TAG_RE.sub("", s)
|
||
|
||
|
||
_HTML_ENTITIES = {
|
||
"&": "&", "<": "<", ">": ">", """: '"', "'": "'",
|
||
"'": "'", " ": " ", "’": "'", "‘": "'",
|
||
"”": '"', "“": '"', "…": "...", "—": "—", "–": "–",
|
||
}
|
||
|
||
|
||
def _decode_html(s: str) -> str:
|
||
for k, v in _HTML_ENTITIES.items():
|
||
s = s.replace(k, v)
|
||
s = re.sub(r"&#(\d+);", lambda m: chr(int(m.group(1))), s)
|
||
s = re.sub(r"&#x([0-9a-fA-F]+);", lambda m: chr(int(m.group(1), 16)), s)
|
||
return s
|