fix(ingest): przegladanie superporn, pornbusy, streamporn, porndoe; progi watchdoga

- superporn: feed /videos/latest/N zamiast rotujacej strony glownej
- pornbusy: karty ze strony glownej przed spoznion sitemapa
- dooplay/streamporn: nowy motyw (card__th, h1 vid__t, data-servers JSON,
  st--duration, time datetime, chipy gatunkow), odstep i ponowienia w _fetch
- porndoe: listing z sitemap_videos.xml; None gdy zadna strona sceny nie parsuje
- hentaimama: /episodes/feed/ poza regexem odcinkow
- watchdog: progi per zrodlo dla serwisow publikujacych rzadko
This commit is contained in:
goon-foss 2026-10-13 10:24:03 +02:00
parent 88f8729840
commit 11400c01cb
6 changed files with 258 additions and 17 deletions

View file

@ -40,7 +40,8 @@ _BASE = "https://hentaimama.io"
_EPISODE_HREF_RE = re.compile(r'href="(https://hentaimama\.io/episodes/[^"]+)"') _EPISODE_HREF_RE = re.compile(r'href="(https://hentaimama\.io/episodes/[^"]+)"')
# Sam listing `/episodes/` i jego paginacja też pasują do wzorca wyżej — bez tego # Sam listing `/episodes/` i jego paginacja też pasują do wzorca wyżej — bez tego
# filtra wchodziły jako „odcinek" o tytule „Recent Episodes". # filtra wchodziły jako „odcinek" o tytule „Recent Episodes".
_NOT_EPISODE_RE = re.compile(r"/episodes/(?:page/\d+/)?$") # `/episodes/feed/` (RSS) tez wpada w href-regex i konczyl jako pusty detail z None.
_NOT_EPISODE_RE = re.compile(r"/episodes/(?:page/\d+/|feed/)?$")
# `og:title` zamiast <h1>: na stronie są trzy <h1>, dwa to logo serwisu, a trzeci bywa # `og:title` zamiast <h1>: na stronie są trzy <h1>, dwa to logo serwisu, a trzeci bywa
# nagłówkiem sekcji. og:title daje czysty „Shoujo Ramune Episode 7". # nagłówkiem sekcji. og:title daje czysty „Shoujo Ramune Episode 7".
_OG_TITLE_RE = re.compile(r"property=['\"]og:title['\"][^>]*content=['\"]([^'\"]{3,150})") _OG_TITLE_RE = re.compile(r"property=['\"]og:title['\"][^>]*content=['\"]([^'\"]{3,150})")

View file

@ -44,6 +44,12 @@ _BASE = "https://pornbusy.com"
_PAGE_SIZE = 20 _PAGE_SIZE = 20
_SITEMAP_INDEX = f"{_BASE}/sitemap_index.xml" _SITEMAP_INDEX = f"{_BASE}/sitemap_index.xml"
# Karta posta na stronie glownej: `<article class="... post-750 post type-post ..."
# data-post-id="750"> <a href="https://pornbusy.com/<slug>/" title=...>`.
_HOME_CARD_RE = re.compile(
r'<article[^>]*\bpost-\d+\b[^>]*>\s*<a\s+href="(https://pornbusy\.com/[a-z0-9-]+/)"',
re.IGNORECASE,
)
_LOC_RE = re.compile(r"<loc>\s*([^<]+?)\s*</loc>") _LOC_RE = re.compile(r"<loc>\s*([^<]+?)\s*</loc>")
_URL_BLOCK_RE = re.compile(r"<url>(.*?)</url>", re.DOTALL | re.IGNORECASE) _URL_BLOCK_RE = re.compile(r"<url>(.*?)</url>", re.DOTALL | re.IGNORECASE)
_LASTMOD_RE = re.compile(r"<lastmod>\s*([^<]+?)\s*</lastmod>") _LASTMOD_RE = re.compile(r"<lastmod>\s*([^<]+?)\s*</lastmod>")
@ -139,8 +145,21 @@ class PornBusyScraper(BaseBrowseScraper):
if not entries: if not entries:
return None return None
entries.sort(key=lambda e: e[0], reverse=True) entries.sort(key=lambda e: e[0], reverse=True)
seen: set[str] = set() # 2026-10-13: sitemapa WordPressa aktualizuje sie z opoznieniem. Najnowszy lastmod
catalog: list[str] = [] # w indeksie mial 9.10, a strona glowna 14 postow z 13.10, ktorych nie bylo w zadnej
# podmapie, wiec browse przez kilka dni widzial "nic nowego". Karty ze strony
# glownej (jedyna dzialajaca strona listingu, paginacja jest zepsuta) ida na
# poczatek katalogu, sitemapa dalej daje glebie.
fresh: list[str] = []
try:
home = browser_get(f"{_BASE}/", timeout=self._timeout)
home.raise_for_status()
fresh = list(dict.fromkeys(_HOME_CARD_RE.findall(home.text)))
log.info("pornbusy: %d kart na stronie glownej", len(fresh))
except Exception as e:
log.warning("pornbusy: homepage fetch failed: %s", e)
seen: set[str] = set(fresh)
catalog: list[str] = list(fresh)
for _, url in entries: for _, url in entries:
if url in seen: if url in seen:
continue continue

View file

@ -35,6 +35,7 @@ import re
from datetime import date, datetime from datetime import date, datetime
from app.config import get_settings from app.config import get_settings
from app.extractors import browser_get
from app.connectors.base import ( from app.connectors.base import (
RawFingerprint, RawFingerprint,
RawPerformer, RawPerformer,
@ -54,6 +55,13 @@ _BASE = "https://porndoe.com"
# Scene listing, `<a href="/watch/pd7a3o0e8v2b">`. Id = `pd` + alfanum. # Scene listing, `<a href="/watch/pd7a3o0e8v2b">`. Id = `pd` + alfanum.
_SCENE_URL_RE = re.compile(r'href="(/watch/[a-z0-9]+)"', re.IGNORECASE) _SCENE_URL_RE = re.compile(r'href="(/watch/[a-z0-9]+)"', re.IGNORECASE)
# Listing z sitemapy wideo (patrz crawl_page): blok <url>, jego <loc> i data publikacji.
_SITEMAP_URL_BLOCK_RE = re.compile(r"<url>(.*?)</url>", re.DOTALL | re.IGNORECASE)
_SITEMAP_LOC_RE = re.compile(r"<loc>\s*([^<]+?)\s*</loc>", re.IGNORECASE)
_SITEMAP_PUBDATE_RE = re.compile(
r"<video:publication_date>\s*([^<]+?)\s*</video:publication_date>", re.IGNORECASE
)
_SITEMAP_PAGE_SIZE = 31
_WATCH_ID_RE = re.compile(r"/watch/([a-z0-9]+)", re.IGNORECASE) _WATCH_ID_RE = re.compile(r"/watch/([a-z0-9]+)", re.IGNORECASE)
# JSON-LD <script> bloki. # JSON-LD <script> bloki.
@ -155,13 +163,77 @@ class PornDoeScraper(BaseBrowseScraper):
self._proxy = get_settings().brightdata_proxy_url self._proxy = get_settings().brightdata_proxy_url
if not self._proxy: if not self._proxy:
log.warning("porndoe: BRIGHTDATA_PROXY_URL nieustawione, scraper wylaczony") log.warning("porndoe: BRIGHTDATA_PROXY_URL nieustawione, scraper wylaczony")
self._catalog: list[str] | None = None
# 2026-10-13: porndoe to aplikacja Angular. HTML `/videos/most-recent` (i `?page=N`,
# `/videos/newest`, nawet z naglowkiem XHR) to statyczna atrapa z tymi samymi 31
# scenami z 11.09, prawdziwa lista laduje sie skryptem. Browse mial przez to new=0
# od 11.09 (watchdog 755 h). Zrodlem listingu jest `sitemap_videos.xml`: plaski urlset
# z 1000 najnowszych /watch/ i `<video:publication_date>`, w probie 13.10 wszystkie
# 1000 nieznane w bazie. Katalog ladowany raz na instancje, stronicowany po
# _SITEMAP_PAGE_SIZE, szczegoly jak dotad z JSON-LD strony sceny (przez proxy).
def _load_catalog(self) -> list[str] | None:
if self._catalog is not None:
return self._catalog
try:
res = browser_get(f"{_BASE}/sitemap_videos.xml", timeout=90, proxy=self._proxy)
res.raise_for_status()
except Exception as e:
log.warning("porndoe: sitemap_videos fetch failed: %s", e)
return None
entries: list[tuple[str, str]] = []
for block in _SITEMAP_URL_BLOCK_RE.findall(res.text):
loc = _SITEMAP_LOC_RE.search(block)
if not loc or "/watch/" not in loc.group(1):
continue
pub = _SITEMAP_PUBDATE_RE.search(block)
entries.append((pub.group(1) if pub else "", loc.group(1).strip()))
if not entries:
log.warning("porndoe: sitemap_videos bez wpisow /watch/")
return None
entries.sort(key=lambda e: e[0], reverse=True)
catalog = list(dict.fromkeys(url for _, url in entries))
log.info("porndoe: katalog z sitemapy, %d scen", len(catalog))
self._catalog = catalog
return catalog
def crawl_page(self, page: int): def crawl_page(self, page: int):
# Bez proxy nie ma po co uderzac (gwarantowane 403). Pusta lista = "wyczerpany", # Bez proxy nie ma po co uderzac (gwarantowane 403). Pusta lista = "wyczerpany",
# dzieki czemu deep-crawl nie ponawia w kolko. # dzieki czemu deep-crawl nie ponawia w kolko.
if not self._proxy: if not self._proxy:
return [] return []
return super().crawl_page(page) catalog = self._load_catalog()
if catalog is None:
return None
chunk = catalog[(page - 1) * _SITEMAP_PAGE_SIZE: page * _SITEMAP_PAGE_SIZE]
if not chunk:
return []
out = []
for scene_url in chunk:
try:
res = browser_get(scene_url, timeout=self._timeout, proxy=self._proxy)
detail_html = res.text
except Exception as e:
log.info("porndoe detail fetch failed %s: %s", scene_url, e)
continue
try:
raw = self._parse_detail(scene_url, detail_html)
except Exception as e:
log.warning("porndoe detail parse failed %s: %s", scene_url, e)
continue
if raw is not None:
out.append(raw)
if not out:
# Wszystkie strony z tej paczki bez VideoObject. Sprawdzone 2026-10-13: dla IP
# proxy (Bright Data ISP, wyjscie w Londynie) kazdy nowy /watch/ przekierowuje
# na /videos, a listing stoi na 11.09, takze na de./fr./br. subdomenach; stare
# sceny (sprzed wrzesnia) otwieraja sie normalnie. Wyglada na ograniczenie
# katalogu dla Wielkiej Brytanii (Online Safety Act), nie na blad parsera.
# None = "przejsciowo nie da sie", zeby deep-crawl nie oznaczyl zrodla jako
# wyczerpanego. Fix wymaga wyjscia proxy spoza UK.
log.warning("porndoe: 0 z %d stron sceny sparsowanych (geo-blokada UK?)", len(chunk))
return None
return out
def _listing_url(self, page: int) -> str: def _listing_url(self, page: int) -> str:
if page <= 1: if page <= 1:

View file

@ -101,9 +101,14 @@ class SuperpornScraper(BaseBrowseScraper):
log.warning("superporn: BRIGHTDATA_PROXY_URL unset — scraper disabled") log.warning("superporn: BRIGHTDATA_PROXY_URL unset — scraper disabled")
def _listing_url(self, page: int) -> str: def _listing_url(self, page: int) -> str:
# 2026-10-13: strona glowna `/` to rotujaca karuzela (48 scen, te same na
# `?page=2`, `?sort=latest` itd.), przez co browse mial new=0 od 30.09 mimo
# swiezych publikacji. Prawdziwy feed po dacie to `/videos/latest`, stronicowany
# SCIEZKA `/videos/latest/N` (`?page=N` jest ignorowane i oddaje strone 1).
# Ta sama klasa bledu co freshporno, patrz pamiec "KVS root rotuje".
if page <= 1: if page <= 1:
return f"{_BASE}/" return f"{_BASE}/videos/latest"
return f"{_BASE}/?page={page}" return f"{_BASE}/videos/latest/{page}"
def crawl_page(self, page: int): def crawl_page(self, page: int):
# Bez proxy nie ma sensu uderzać (gwarantowany CF 403), sygnalizuj # Bez proxy nie ma sensu uderzać (gwarantowany CF 403), sygnalizuj

View file

@ -21,8 +21,10 @@ mógł wybrać czyim embedem chce odpalić scenę.
""" """
from __future__ import annotations from __future__ import annotations
import json
import logging import logging
import re import re
import time
from collections.abc import Iterator from collections.abc import Iterator
from datetime import date, datetime from datetime import date, datetime
from typing import Any from typing import Any
@ -59,10 +61,14 @@ USER_AGENT = (
# 2026-10-02 mangoporn przeszedl na `<a class="thumb" href=...>` (klasa PRZED href) i # 2026-10-02 mangoporn przeszedl na `<a class="thumb" href=...>` (klasa PRZED href) i
# wzorzec z href na poczatku nie lapal juz nic: 0 filmow od 2026-09-23. Teraz href moze # wzorzec z href na poczatku nie lapal juz nic: 0 filmow od 2026-09-23. Teraz href moze
# stac w dowolnym miejscu znacznika; linki menu do /movies/category/ odpadaja. # stac w dowolnym miejscu znacznika; linki menu do /movies/category/ odpadaja.
# 2026-10-13 streamporn.li zmienil motyw: karta to `<article class="card"><a class="card__th"
# href=...>` (klasa PRZED href, slug bez /movies/), stary wzorzec dawal 0 linkow od 2.10.
_LIST_ITEM_RE = re.compile( _LIST_ITEM_RE = re.compile(
r'<a\s+href="(?P<url>https?://[^"]+)"[^>]*\bclass="ml-mask\b[^"]*"' r'<a\s+href="(?P<url>https?://[^"]+)"[^>]*\bclass="ml-mask\b[^"]*"'
r"|" r"|"
r'<a\s+(?:[^>]*?\s)?href="(?P<url2>https?://[^"]+/movies/(?!category/)[a-z0-9-]+/)"', r'<a\s+(?:[^>]*?\s)?href="(?P<url2>https?://[^"]+/movies/(?!category/)[a-z0-9-]+/)"'
r"|"
r'<a\s+class="(?:card__th|ml-mask)[^"]*"\s+href="(?P<url3>https?://[^"]+)"',
re.IGNORECASE, re.IGNORECASE,
) )
# Tolerantny title, mangoporn (dooplay) używa <h1> w class="data", streamporn/pandamovies # Tolerantny title, mangoporn (dooplay) używa <h1> w class="data", streamporn/pandamovies
@ -155,6 +161,40 @@ _POSTER_OG_RE = re.compile(
) )
_DURATION_MINS_RE = re.compile(r"(\d+)\s*min", re.IGNORECASE) _DURATION_MINS_RE = re.compile(r"(\d+)\s*min", re.IGNORECASE)
# Motyw streamporn.li od 2026-10 (wtyczka host-links-manager). Tytul siedzi w
# `<h1 class="vid__t"><span>Watch TYTUL ROK by STUDIO Porn Movie Online Free</span></h1>`,
# pasek `st st--duration` ma MM:SS, data w `<time datetime=...>`, gatunki to
# `<a class="chip" href=".../genre/<slug>/">` bez rel="tag", a hostery nie sa linkami
# w HTML, tylko JSON-em w atrybucie `data-servers` sekcji `<section class="hlm">`:
# [{"u": "https://playmogo.com/e/..", "t": "iframe", "l": "DoodStream"}, ...].
_VID_TITLE_RE = re.compile(
r'<h1[^>]*\bclass="vid__t[^"]*"[^>]*>\s*(?:<span[^>]*>)?\s*(.+?)\s*(?:</span>)?\s*</h1>',
re.IGNORECASE | re.DOTALL,
)
_VID_TITLE_PARTS_RE = re.compile(
r"^(?:Watch\s+)?(?P<title>.+?)"
r"(?:\s+(?P<year>(?:19|20)\d{2})\s+by\s+(?P<studio>.+?))?"
r"(?:\s+Porn\s+Movie\s+Online\s+Free)?\s*$",
re.IGNORECASE | re.DOTALL,
)
_ST_DURATION_RE = re.compile(
r'class="st st--duration"[^>]*>[^<]*(?:<svg.*?</svg>)?\s*(\d{1,2}):(\d{2})(?::(\d{2}))?\s*<',
re.IGNORECASE | re.DOTALL,
)
_ST_DATE_RE = re.compile(
r'class="st st--date"[^>]*>.*?<time\s+datetime="(\d{4}-\d{2}-\d{2})', re.IGNORECASE | re.DOTALL
)
_YEAR_LOOSE_RE = re.compile(r'/(?:year|release-year)/(\d{4})/"', re.IGNORECASE)
_GENRE_CHIP_RE = re.compile(
r'<a\s+class="chip"\s+href="https?://[^/]+/genres?/([a-z0-9-]+)/"[^>]*>([^<]+)</a>',
re.IGNORECASE,
)
_DATA_SERVERS_RE = re.compile(r'data-servers="([^"]*)"', re.IGNORECASE)
def _slug(name: str) -> str:
return re.sub(r"-{2,}", "-", re.sub(r"[^a-z0-9]+", "-", name.lower())).strip("-")
class DooplayConnector(BaseMovieConnector): class DooplayConnector(BaseMovieConnector):
"""Generic dooplay scraper. Instantiated per-site via subclasses below.""" """Generic dooplay scraper. Instantiated per-site via subclasses below."""
@ -169,13 +209,23 @@ class DooplayConnector(BaseMovieConnector):
if not getattr(self, "name", None): if not getattr(self, "name", None):
raise RuntimeError(f"{type(self).__name__} requires class-level `name`") raise RuntimeError(f"{type(self).__name__} requires class-level `name`")
self._timeout = timeout self._timeout = timeout
self._last_fetch_at = 0.0
# Odstep miedzy zapytaniami do jednej strony. streamporn.li (2026-10-13) odpowiada
# na pojedyncze zapytanie w 0,2 s, ale drugie zapytanie zaraz po pierwszym wisialo
# do timeoutu (30 s, curl 28), a wyjatek curl_cffi nie jest httpx.HTTPError, wiec
# wywracal caly przebieg ingestu po pierwszym filmie.
_pace_sec = 1.0
_fetch_attempts = 3
def close(self) -> None: def close(self) -> None:
pass pass
def _fetch(self, url: str) -> str: def _fetch(self, url: str) -> str:
"""browser_get z chrome120 impersonation, psyplay sites czasem blokują """browser_get z chrome120 impersonation, psyplay sites czasem blokują
czysty httpx (Python TLS fingerprint) zwracając 500/403. curl_cffi fixuje to.""" czysty httpx (Python TLS fingerprint) zwracając 500/403. curl_cffi fixuje to.
Timeout i bledy transportu: do `_fetch_attempts` prob z rosnacym odstepem, potem
httpx.TransportError, zeby petla w fetch_movies pominela film zamiast paść."""
if not url.startswith("http"): if not url.startswith("http"):
url = self.base_url.rstrip("/") + url url = self.base_url.rstrip("/") + url
headers = { headers = {
@ -184,7 +234,23 @@ class DooplayConnector(BaseMovieConnector):
"Accept": "text/html,application/xhtml+xml", "Accept": "text/html,application/xhtml+xml",
"Referer": self.base_url + "/", "Referer": self.base_url + "/",
} }
r = browser_get(url, headers=headers, timeout=self._timeout, follow_redirects=True) last_exc: Exception | None = None
r = None
for attempt in range(self._fetch_attempts):
wait = self._pace_sec - (time.monotonic() - self._last_fetch_at)
if wait > 0:
time.sleep(wait)
try:
r = browser_get(url, headers=headers, timeout=self._timeout, follow_redirects=True)
self._last_fetch_at = time.monotonic()
break
except Exception as e: # noqa: BLE001 - curl_cffi/httpx, rozne klasy
last_exc = e
self._last_fetch_at = time.monotonic()
log.info("%s: fetch attempt %d failed for %s: %s", self.name, attempt + 1, url, e)
time.sleep(2.0 * (attempt + 1))
if r is None:
raise httpx.TransportError(f"fetch failed after {self._fetch_attempts} attempts: {url}: {last_exc}")
if r.status_code >= 400: if r.status_code >= 400:
raise httpx.HTTPStatusError( raise httpx.HTTPStatusError(
f"{r.status_code} for {url}", f"{r.status_code} for {url}",
@ -251,7 +317,7 @@ class DooplayConnector(BaseMovieConnector):
from urllib.parse import urlparse from urllib.parse import urlparse
site_host = urlparse(self.base_url).hostname site_host = urlparse(self.base_url).hostname
for m in _LIST_ITEM_RE.finditer(text): for m in _LIST_ITEM_RE.finditer(text):
url = m.group("url") or m.group("url2") url = m.group("url") or m.group("url2") or m.group("url3")
if not url: if not url:
continue continue
try: try:
@ -278,14 +344,30 @@ class DooplayConnector(BaseMovieConnector):
def _parse_dooplay_detail( def _parse_dooplay_detail(
*, slug: str, html: str, source_name: str, base_url: str, page_url: str | None = None *, slug: str, html: str, source_name: str, base_url: str, page_url: str | None = None
) -> RawMovie | None: ) -> RawMovie | None:
vid_year: int | None = None
vid_studio: str | None = None
m_title = _TITLE_RE.search(html) or _H1_TITLE_RE.search(html) m_title = _TITLE_RE.search(html) or _H1_TITLE_RE.search(html)
if not m_title: if m_title:
log.warning("%s: no title in %s", source_name, slug) title = _decode_html(next(g for g in m_title.groups() if g).strip())
return None else:
title = _decode_html(next(g for g in m_title.groups() if g).strip()) # Motyw streamporn.li 2026-10: h1 z zagniezdzonym <span>, tytul, rok i studio w
# jednym napisie ("Watch TYTUL 2026 by STUDIO Porn Movie Online Free").
m_vid = _VID_TITLE_RE.search(html)
if not m_vid:
log.warning("%s: no title in %s", source_name, slug)
return None
raw_title = _decode_html(_strip_tags(m_vid.group(1))).strip()
m_parts = _VID_TITLE_PARTS_RE.match(raw_title)
title = (m_parts.group("title") if m_parts else raw_title).strip()
if m_parts and m_parts.group("year"):
vid_year = int(m_parts.group("year"))
if m_parts and m_parts.group("studio"):
vid_studio = m_parts.group("studio").strip()
if not title:
return None
m_year = _YEAR_RE.search(html) m_year = _YEAR_RE.search(html) or _YEAR_LOOSE_RE.search(html)
release_year = int(m_year.group(1)) if m_year else None release_year = int(m_year.group(1)) if m_year else vid_year
studio: RawStudio | None = None studio: RawStudio | None = None
m_studio = _STUDIO_RE.search(html) m_studio = _STUDIO_RE.search(html)
@ -297,6 +379,14 @@ def _parse_dooplay_detail(
name=studio_name, name=studio_name,
slug=studio_slug, slug=studio_slug,
) )
elif vid_studio:
studio_slug = _slug(vid_studio)
if studio_slug:
studio = RawStudio(
external_id=f"{source_name}:{studio_slug}",
name=vid_studio,
slug=studio_slug,
)
duration_sec: int | None = None duration_sec: int | None = None
m_dur = _DURATION_SPAN_RE.search(html) or _DURATION_TEXT_RE.search(html) m_dur = _DURATION_SPAN_RE.search(html) or _DURATION_TEXT_RE.search(html)
@ -307,6 +397,15 @@ def _parse_dooplay_detail(
m_m = re.search(r"(\d+)\s*min", text, re.IGNORECASE) m_m = re.search(r"(\d+)\s*min", text, re.IGNORECASE)
if m_h or m_m: if m_h or m_m:
duration_sec = (int(m_h.group(1)) * 3600 if m_h else 0) + (int(m_m.group(1)) * 60 if m_m else 0) duration_sec = (int(m_h.group(1)) * 3600 if m_h else 0) + (int(m_m.group(1)) * 60 if m_m else 0)
if duration_sec is None:
# streamporn.li 2026-10: pasek "29:00" (MM:SS) albo "1:29:00" (H:MM:SS).
m_st = _ST_DURATION_RE.search(html)
if m_st:
a, b, c = m_st.group(1), m_st.group(2), m_st.group(3)
if c is not None:
duration_sec = int(a) * 3600 + int(b) * 60 + int(c)
else:
duration_sec = int(a) * 60 + int(b)
release_date: date | None = None release_date: date | None = None
m_rd = _RELEASE_DATE_SPAN_RE.search(html) or _RELEASE_DATE_TEXT_RE.search(html) m_rd = _RELEASE_DATE_SPAN_RE.search(html) or _RELEASE_DATE_TEXT_RE.search(html)
@ -318,6 +417,13 @@ def _parse_dooplay_detail(
break break
except ValueError: except ValueError:
continue continue
if release_date is None:
m_sd = _ST_DATE_RE.search(html)
if m_sd:
try:
release_date = datetime.strptime(m_sd.group(1), "%Y-%m-%d").date()
except ValueError:
pass
description: str | None = None description: str | None = None
m_desc = _DESCRIPTION_RE.search(html) m_desc = _DESCRIPTION_RE.search(html)
@ -360,6 +466,16 @@ def _parse_dooplay_detail(
) )
for m in _GENRE_TAG_RE.finditer(html) for m in _GENRE_TAG_RE.finditer(html)
] ]
if not tags:
# streamporn.li 2026-10: chipy gatunkow bez rel="tag".
tags = [
RawTag(
external_id=f"{source_name}:{m.group(1)}",
name=_decode_html(m.group(2).strip()),
slug=m.group(1),
)
for m in _GENRE_CHIP_RE.finditer(html)
]
if page_url is None: if page_url is None:
page_url = f"{base_url}/movies/{slug}/" page_url = f"{base_url}/movies/{slug}/"
@ -416,6 +532,22 @@ def _parse_dooplay_detail(
source = (src_match.group(1).strip() if src_match else "") or None source = (src_match.group(1).strip() if src_match else "") or None
_emit_host_entry(href, source) _emit_host_entry(href, source)
# Motyw streamporn.li 2026-10 (host-links-manager): hostery w JSON-ie atrybutu
# data-servers, HTML-escapowanym (&quot;). "u" = adres embed, "l" = etykieta.
for m in _DATA_SERVERS_RE.finditer(html):
try:
servers = json.loads(_decode_html(m.group(1)))
except (ValueError, TypeError):
continue
if not isinstance(servers, list):
continue
for entry in servers:
if not isinstance(entry, dict):
continue
embed = str(entry.get("u") or "").strip()
if embed.startswith("http"):
_emit_host_entry(embed, embed)
if not playback_sources: if not playback_sources:
# Brak sub-hosters znalezionych, fallback do landing page (mobile otworzy # Brak sub-hosters znalezionych, fallback do landing page (mobile otworzy
# w WebView). Robimy to TYLKO gdy nie ma alternatyw, inaczej landing jest # w WebView). Robimy to TYLKO gdy nie ma alternatyw, inaczej landing jest

View file

@ -59,6 +59,15 @@ def _bucket(age_h: float) -> tuple[str, str]:
return "swiezo", "warning" return "swiezo", "warning"
# Serwisy, ktore publikuja rzadziej niz co dwa dni: prog 48 h robil z nich staly alarm
# (sprawdzone 2026-10-13: xanimeporn 1 do 3 wpisow na tydzien z RSS, hentaimama wydaje
# odcinki raz w tygodniu, obie listy w calosci znane w bazie). Prog w godzinach.
_BROWSE_MAX_AGE_OVERRIDE_H: dict[str, int] = {
"xanimeporn": 168,
"hentaimama": 240,
}
def run_ingest_freshness_watchdog( def run_ingest_freshness_watchdog(
*, *,
max_age_hours: int = 48, max_age_hours: int = 48,
@ -86,7 +95,10 @@ def run_ingest_freshness_watchdog(
# (origin, tabela, próg_h, klasa) # (origin, tabela, próg_h, klasa)
checks: list[tuple[str, str, int, str]] = ( checks: list[tuple[str, str, int, str]] = (
[(f"tube:{t}", "playback_sources", max_age_hours, "browse") for t in sorted(browse_tags)] [
(f"tube:{t}", "playback_sources", _BROWSE_MAX_AGE_OVERRIDE_H.get(t, max_age_hours), "browse")
for t in sorted(browse_tags)
]
+ [(f"tube:{t}", "playback_sources", search_max_age_hours, "search") + [(f"tube:{t}", "playback_sources", search_max_age_hours, "search")
for t in sorted(search_tags)] for t in sorted(search_tags)]
) )