fix(ingest): przegladanie superporn, pornbusy, streamporn, porndoe; progi watchdoga
- superporn: feed /videos/latest/N zamiast rotujacej strony glownej - pornbusy: karty ze strony glownej przed spoznion sitemapa - dooplay/streamporn: nowy motyw (card__th, h1 vid__t, data-servers JSON, st--duration, time datetime, chipy gatunkow), odstep i ponowienia w _fetch - porndoe: listing z sitemap_videos.xml; None gdy zadna strona sceny nie parsuje - hentaimama: /episodes/feed/ poza regexem odcinkow - watchdog: progi per zrodlo dla serwisow publikujacych rzadko
This commit is contained in:
goon-foss
parent
88f8729840
commit
11400c01cb
6 changed files with 258 additions and 17 deletions
|
|
@ -40,7 +40,8 @@ _BASE = "https://hentaimama.io"
|
|||
_EPISODE_HREF_RE = re.compile(r'href="(https://hentaimama\.io/episodes/[^"]+)"')
|
||||
# Sam listing `/episodes/` i jego paginacja też pasują do wzorca wyżej — bez tego
|
||||
# filtra wchodziły jako „odcinek" o tytule „Recent Episodes".
|
||||
_NOT_EPISODE_RE = re.compile(r"/episodes/(?:page/\d+/)?$")
|
||||
# `/episodes/feed/` (RSS) tez wpada w href-regex i konczyl jako pusty detail z None.
|
||||
_NOT_EPISODE_RE = re.compile(r"/episodes/(?:page/\d+/|feed/)?$")
|
||||
# `og:title` zamiast <h1>: na stronie są trzy <h1>, dwa to logo serwisu, a trzeci bywa
|
||||
# nagłówkiem sekcji. og:title daje czysty „Shoujo Ramune Episode 7".
|
||||
_OG_TITLE_RE = re.compile(r"property=['\"]og:title['\"][^>]*content=['\"]([^'\"]{3,150})")
|
||||
|
|
|
|||
|
|
@ -44,6 +44,12 @@ _BASE = "https://pornbusy.com"
|
|||
_PAGE_SIZE = 20
|
||||
|
||||
_SITEMAP_INDEX = f"{_BASE}/sitemap_index.xml"
|
||||
# Karta posta na stronie glownej: `<article class="... post-750 post type-post ..."
|
||||
# data-post-id="750"> <a href="https://pornbusy.com/<slug>/" title=...>`.
|
||||
_HOME_CARD_RE = re.compile(
|
||||
r'<article[^>]*\bpost-\d+\b[^>]*>\s*<a\s+href="(https://pornbusy\.com/[a-z0-9-]+/)"',
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_LOC_RE = re.compile(r"<loc>\s*([^<]+?)\s*</loc>")
|
||||
_URL_BLOCK_RE = re.compile(r"<url>(.*?)</url>", re.DOTALL | re.IGNORECASE)
|
||||
_LASTMOD_RE = re.compile(r"<lastmod>\s*([^<]+?)\s*</lastmod>")
|
||||
|
|
@ -139,8 +145,21 @@ class PornBusyScraper(BaseBrowseScraper):
|
|||
if not entries:
|
||||
return None
|
||||
entries.sort(key=lambda e: e[0], reverse=True)
|
||||
seen: set[str] = set()
|
||||
catalog: list[str] = []
|
||||
# 2026-10-13: sitemapa WordPressa aktualizuje sie z opoznieniem. Najnowszy lastmod
|
||||
# w indeksie mial 9.10, a strona glowna 14 postow z 13.10, ktorych nie bylo w zadnej
|
||||
# podmapie, wiec browse przez kilka dni widzial "nic nowego". Karty ze strony
|
||||
# glownej (jedyna dzialajaca strona listingu, paginacja jest zepsuta) ida na
|
||||
# poczatek katalogu, sitemapa dalej daje glebie.
|
||||
fresh: list[str] = []
|
||||
try:
|
||||
home = browser_get(f"{_BASE}/", timeout=self._timeout)
|
||||
home.raise_for_status()
|
||||
fresh = list(dict.fromkeys(_HOME_CARD_RE.findall(home.text)))
|
||||
log.info("pornbusy: %d kart na stronie glownej", len(fresh))
|
||||
except Exception as e:
|
||||
log.warning("pornbusy: homepage fetch failed: %s", e)
|
||||
seen: set[str] = set(fresh)
|
||||
catalog: list[str] = list(fresh)
|
||||
for _, url in entries:
|
||||
if url in seen:
|
||||
continue
|
||||
|
|
|
|||
|
|
@ -35,6 +35,7 @@ import re
|
|||
from datetime import date, datetime
|
||||
|
||||
from app.config import get_settings
|
||||
from app.extractors import browser_get
|
||||
from app.connectors.base import (
|
||||
RawFingerprint,
|
||||
RawPerformer,
|
||||
|
|
@ -54,6 +55,13 @@ _BASE = "https://porndoe.com"
|
|||
|
||||
# Scene listing, `<a href="/watch/pd7a3o0e8v2b">`. Id = `pd` + alfanum.
|
||||
_SCENE_URL_RE = re.compile(r'href="(/watch/[a-z0-9]+)"', re.IGNORECASE)
|
||||
# Listing z sitemapy wideo (patrz crawl_page): blok <url>, jego <loc> i data publikacji.
|
||||
_SITEMAP_URL_BLOCK_RE = re.compile(r"<url>(.*?)</url>", re.DOTALL | re.IGNORECASE)
|
||||
_SITEMAP_LOC_RE = re.compile(r"<loc>\s*([^<]+?)\s*</loc>", re.IGNORECASE)
|
||||
_SITEMAP_PUBDATE_RE = re.compile(
|
||||
r"<video:publication_date>\s*([^<]+?)\s*</video:publication_date>", re.IGNORECASE
|
||||
)
|
||||
_SITEMAP_PAGE_SIZE = 31
|
||||
_WATCH_ID_RE = re.compile(r"/watch/([a-z0-9]+)", re.IGNORECASE)
|
||||
|
||||
# JSON-LD <script> bloki.
|
||||
|
|
@ -155,13 +163,77 @@ class PornDoeScraper(BaseBrowseScraper):
|
|||
self._proxy = get_settings().brightdata_proxy_url
|
||||
if not self._proxy:
|
||||
log.warning("porndoe: BRIGHTDATA_PROXY_URL nieustawione, scraper wylaczony")
|
||||
self._catalog: list[str] | None = None
|
||||
|
||||
# 2026-10-13: porndoe to aplikacja Angular. HTML `/videos/most-recent` (i `?page=N`,
|
||||
# `/videos/newest`, nawet z naglowkiem XHR) to statyczna atrapa z tymi samymi 31
|
||||
# scenami z 11.09, prawdziwa lista laduje sie skryptem. Browse mial przez to new=0
|
||||
# od 11.09 (watchdog 755 h). Zrodlem listingu jest `sitemap_videos.xml`: plaski urlset
|
||||
# z 1000 najnowszych /watch/ i `<video:publication_date>`, w probie 13.10 wszystkie
|
||||
# 1000 nieznane w bazie. Katalog ladowany raz na instancje, stronicowany po
|
||||
# _SITEMAP_PAGE_SIZE, szczegoly jak dotad z JSON-LD strony sceny (przez proxy).
|
||||
def _load_catalog(self) -> list[str] | None:
|
||||
if self._catalog is not None:
|
||||
return self._catalog
|
||||
try:
|
||||
res = browser_get(f"{_BASE}/sitemap_videos.xml", timeout=90, proxy=self._proxy)
|
||||
res.raise_for_status()
|
||||
except Exception as e:
|
||||
log.warning("porndoe: sitemap_videos fetch failed: %s", e)
|
||||
return None
|
||||
entries: list[tuple[str, str]] = []
|
||||
for block in _SITEMAP_URL_BLOCK_RE.findall(res.text):
|
||||
loc = _SITEMAP_LOC_RE.search(block)
|
||||
if not loc or "/watch/" not in loc.group(1):
|
||||
continue
|
||||
pub = _SITEMAP_PUBDATE_RE.search(block)
|
||||
entries.append((pub.group(1) if pub else "", loc.group(1).strip()))
|
||||
if not entries:
|
||||
log.warning("porndoe: sitemap_videos bez wpisow /watch/")
|
||||
return None
|
||||
entries.sort(key=lambda e: e[0], reverse=True)
|
||||
catalog = list(dict.fromkeys(url for _, url in entries))
|
||||
log.info("porndoe: katalog z sitemapy, %d scen", len(catalog))
|
||||
self._catalog = catalog
|
||||
return catalog
|
||||
|
||||
def crawl_page(self, page: int):
|
||||
# Bez proxy nie ma po co uderzac (gwarantowane 403). Pusta lista = "wyczerpany",
|
||||
# dzieki czemu deep-crawl nie ponawia w kolko.
|
||||
if not self._proxy:
|
||||
return []
|
||||
return super().crawl_page(page)
|
||||
catalog = self._load_catalog()
|
||||
if catalog is None:
|
||||
return None
|
||||
chunk = catalog[(page - 1) * _SITEMAP_PAGE_SIZE: page * _SITEMAP_PAGE_SIZE]
|
||||
if not chunk:
|
||||
return []
|
||||
out = []
|
||||
for scene_url in chunk:
|
||||
try:
|
||||
res = browser_get(scene_url, timeout=self._timeout, proxy=self._proxy)
|
||||
detail_html = res.text
|
||||
except Exception as e:
|
||||
log.info("porndoe detail fetch failed %s: %s", scene_url, e)
|
||||
continue
|
||||
try:
|
||||
raw = self._parse_detail(scene_url, detail_html)
|
||||
except Exception as e:
|
||||
log.warning("porndoe detail parse failed %s: %s", scene_url, e)
|
||||
continue
|
||||
if raw is not None:
|
||||
out.append(raw)
|
||||
if not out:
|
||||
# Wszystkie strony z tej paczki bez VideoObject. Sprawdzone 2026-10-13: dla IP
|
||||
# proxy (Bright Data ISP, wyjscie w Londynie) kazdy nowy /watch/ przekierowuje
|
||||
# na /videos, a listing stoi na 11.09, takze na de./fr./br. subdomenach; stare
|
||||
# sceny (sprzed wrzesnia) otwieraja sie normalnie. Wyglada na ograniczenie
|
||||
# katalogu dla Wielkiej Brytanii (Online Safety Act), nie na blad parsera.
|
||||
# None = "przejsciowo nie da sie", zeby deep-crawl nie oznaczyl zrodla jako
|
||||
# wyczerpanego. Fix wymaga wyjscia proxy spoza UK.
|
||||
log.warning("porndoe: 0 z %d stron sceny sparsowanych (geo-blokada UK?)", len(chunk))
|
||||
return None
|
||||
return out
|
||||
|
||||
def _listing_url(self, page: int) -> str:
|
||||
if page <= 1:
|
||||
|
|
|
|||
|
|
@ -101,9 +101,14 @@ class SuperpornScraper(BaseBrowseScraper):
|
|||
log.warning("superporn: BRIGHTDATA_PROXY_URL unset — scraper disabled")
|
||||
|
||||
def _listing_url(self, page: int) -> str:
|
||||
# 2026-10-13: strona glowna `/` to rotujaca karuzela (48 scen, te same na
|
||||
# `?page=2`, `?sort=latest` itd.), przez co browse mial new=0 od 30.09 mimo
|
||||
# swiezych publikacji. Prawdziwy feed po dacie to `/videos/latest`, stronicowany
|
||||
# SCIEZKA `/videos/latest/N` (`?page=N` jest ignorowane i oddaje strone 1).
|
||||
# Ta sama klasa bledu co freshporno, patrz pamiec "KVS root rotuje".
|
||||
if page <= 1:
|
||||
return f"{_BASE}/"
|
||||
return f"{_BASE}/?page={page}"
|
||||
return f"{_BASE}/videos/latest"
|
||||
return f"{_BASE}/videos/latest/{page}"
|
||||
|
||||
def crawl_page(self, page: int):
|
||||
# Bez proxy nie ma sensu uderzać (gwarantowany CF 403), sygnalizuj
|
||||
|
|
|
|||
|
|
@ -21,8 +21,10 @@ mógł wybrać czyim embedem chce odpalić scenę.
|
|||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
from collections.abc import Iterator
|
||||
from datetime import date, datetime
|
||||
from typing import Any
|
||||
|
|
@ -59,10 +61,14 @@ USER_AGENT = (
|
|||
# 2026-10-02 mangoporn przeszedl na `<a class="thumb" href=...>` (klasa PRZED href) i
|
||||
# wzorzec z href na poczatku nie lapal juz nic: 0 filmow od 2026-09-23. Teraz href moze
|
||||
# stac w dowolnym miejscu znacznika; linki menu do /movies/category/ odpadaja.
|
||||
# 2026-10-13 streamporn.li zmienil motyw: karta to `<article class="card"><a class="card__th"
|
||||
# href=...>` (klasa PRZED href, slug bez /movies/), stary wzorzec dawal 0 linkow od 2.10.
|
||||
_LIST_ITEM_RE = re.compile(
|
||||
r'<a\s+href="(?P<url>https?://[^"]+)"[^>]*\bclass="ml-mask\b[^"]*"'
|
||||
r"|"
|
||||
r'<a\s+(?:[^>]*?\s)?href="(?P<url2>https?://[^"]+/movies/(?!category/)[a-z0-9-]+/)"',
|
||||
r'<a\s+(?:[^>]*?\s)?href="(?P<url2>https?://[^"]+/movies/(?!category/)[a-z0-9-]+/)"'
|
||||
r"|"
|
||||
r'<a\s+class="(?:card__th|ml-mask)[^"]*"\s+href="(?P<url3>https?://[^"]+)"',
|
||||
re.IGNORECASE,
|
||||
)
|
||||
# Tolerantny title, mangoporn (dooplay) używa <h1> w class="data", streamporn/pandamovies
|
||||
|
|
@ -155,6 +161,40 @@ _POSTER_OG_RE = re.compile(
|
|||
)
|
||||
_DURATION_MINS_RE = re.compile(r"(\d+)\s*min", re.IGNORECASE)
|
||||
|
||||
# Motyw streamporn.li od 2026-10 (wtyczka host-links-manager). Tytul siedzi w
|
||||
# `<h1 class="vid__t"><span>Watch TYTUL ROK by STUDIO Porn Movie Online Free</span></h1>`,
|
||||
# pasek `st st--duration` ma MM:SS, data w `<time datetime=...>`, gatunki to
|
||||
# `<a class="chip" href=".../genre/<slug>/">` bez rel="tag", a hostery nie sa linkami
|
||||
# w HTML, tylko JSON-em w atrybucie `data-servers` sekcji `<section class="hlm">`:
|
||||
# [{"u": "https://playmogo.com/e/..", "t": "iframe", "l": "DoodStream"}, ...].
|
||||
_VID_TITLE_RE = re.compile(
|
||||
r'<h1[^>]*\bclass="vid__t[^"]*"[^>]*>\s*(?:<span[^>]*>)?\s*(.+?)\s*(?:</span>)?\s*</h1>',
|
||||
re.IGNORECASE | re.DOTALL,
|
||||
)
|
||||
_VID_TITLE_PARTS_RE = re.compile(
|
||||
r"^(?:Watch\s+)?(?P<title>.+?)"
|
||||
r"(?:\s+(?P<year>(?:19|20)\d{2})\s+by\s+(?P<studio>.+?))?"
|
||||
r"(?:\s+Porn\s+Movie\s+Online\s+Free)?\s*$",
|
||||
re.IGNORECASE | re.DOTALL,
|
||||
)
|
||||
_ST_DURATION_RE = re.compile(
|
||||
r'class="st st--duration"[^>]*>[^<]*(?:<svg.*?</svg>)?\s*(\d{1,2}):(\d{2})(?::(\d{2}))?\s*<',
|
||||
re.IGNORECASE | re.DOTALL,
|
||||
)
|
||||
_ST_DATE_RE = re.compile(
|
||||
r'class="st st--date"[^>]*>.*?<time\s+datetime="(\d{4}-\d{2}-\d{2})', re.IGNORECASE | re.DOTALL
|
||||
)
|
||||
_YEAR_LOOSE_RE = re.compile(r'/(?:year|release-year)/(\d{4})/"', re.IGNORECASE)
|
||||
_GENRE_CHIP_RE = re.compile(
|
||||
r'<a\s+class="chip"\s+href="https?://[^/]+/genres?/([a-z0-9-]+)/"[^>]*>([^<]+)</a>',
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_DATA_SERVERS_RE = re.compile(r'data-servers="([^"]*)"', re.IGNORECASE)
|
||||
|
||||
|
||||
def _slug(name: str) -> str:
|
||||
return re.sub(r"-{2,}", "-", re.sub(r"[^a-z0-9]+", "-", name.lower())).strip("-")
|
||||
|
||||
|
||||
class DooplayConnector(BaseMovieConnector):
|
||||
"""Generic dooplay scraper. Instantiated per-site via subclasses below."""
|
||||
|
|
@ -169,13 +209,23 @@ class DooplayConnector(BaseMovieConnector):
|
|||
if not getattr(self, "name", None):
|
||||
raise RuntimeError(f"{type(self).__name__} requires class-level `name`")
|
||||
self._timeout = timeout
|
||||
self._last_fetch_at = 0.0
|
||||
|
||||
# Odstep miedzy zapytaniami do jednej strony. streamporn.li (2026-10-13) odpowiada
|
||||
# na pojedyncze zapytanie w 0,2 s, ale drugie zapytanie zaraz po pierwszym wisialo
|
||||
# do timeoutu (30 s, curl 28), a wyjatek curl_cffi nie jest httpx.HTTPError, wiec
|
||||
# wywracal caly przebieg ingestu po pierwszym filmie.
|
||||
_pace_sec = 1.0
|
||||
_fetch_attempts = 3
|
||||
|
||||
def close(self) -> None:
|
||||
pass
|
||||
|
||||
def _fetch(self, url: str) -> str:
|
||||
"""browser_get z chrome120 impersonation, psyplay sites czasem blokują
|
||||
czysty httpx (Python TLS fingerprint) zwracając 500/403. curl_cffi fixuje to."""
|
||||
czysty httpx (Python TLS fingerprint) zwracając 500/403. curl_cffi fixuje to.
|
||||
Timeout i bledy transportu: do `_fetch_attempts` prob z rosnacym odstepem, potem
|
||||
httpx.TransportError, zeby petla w fetch_movies pominela film zamiast paść."""
|
||||
if not url.startswith("http"):
|
||||
url = self.base_url.rstrip("/") + url
|
||||
headers = {
|
||||
|
|
@ -184,7 +234,23 @@ class DooplayConnector(BaseMovieConnector):
|
|||
"Accept": "text/html,application/xhtml+xml",
|
||||
"Referer": self.base_url + "/",
|
||||
}
|
||||
r = browser_get(url, headers=headers, timeout=self._timeout, follow_redirects=True)
|
||||
last_exc: Exception | None = None
|
||||
r = None
|
||||
for attempt in range(self._fetch_attempts):
|
||||
wait = self._pace_sec - (time.monotonic() - self._last_fetch_at)
|
||||
if wait > 0:
|
||||
time.sleep(wait)
|
||||
try:
|
||||
r = browser_get(url, headers=headers, timeout=self._timeout, follow_redirects=True)
|
||||
self._last_fetch_at = time.monotonic()
|
||||
break
|
||||
except Exception as e: # noqa: BLE001 - curl_cffi/httpx, rozne klasy
|
||||
last_exc = e
|
||||
self._last_fetch_at = time.monotonic()
|
||||
log.info("%s: fetch attempt %d failed for %s: %s", self.name, attempt + 1, url, e)
|
||||
time.sleep(2.0 * (attempt + 1))
|
||||
if r is None:
|
||||
raise httpx.TransportError(f"fetch failed after {self._fetch_attempts} attempts: {url}: {last_exc}")
|
||||
if r.status_code >= 400:
|
||||
raise httpx.HTTPStatusError(
|
||||
f"{r.status_code} for {url}",
|
||||
|
|
@ -251,7 +317,7 @@ class DooplayConnector(BaseMovieConnector):
|
|||
from urllib.parse import urlparse
|
||||
site_host = urlparse(self.base_url).hostname
|
||||
for m in _LIST_ITEM_RE.finditer(text):
|
||||
url = m.group("url") or m.group("url2")
|
||||
url = m.group("url") or m.group("url2") or m.group("url3")
|
||||
if not url:
|
||||
continue
|
||||
try:
|
||||
|
|
@ -278,14 +344,30 @@ class DooplayConnector(BaseMovieConnector):
|
|||
def _parse_dooplay_detail(
|
||||
*, slug: str, html: str, source_name: str, base_url: str, page_url: str | None = None
|
||||
) -> RawMovie | None:
|
||||
vid_year: int | None = None
|
||||
vid_studio: str | None = None
|
||||
m_title = _TITLE_RE.search(html) or _H1_TITLE_RE.search(html)
|
||||
if not m_title:
|
||||
log.warning("%s: no title in %s", source_name, slug)
|
||||
return None
|
||||
title = _decode_html(next(g for g in m_title.groups() if g).strip())
|
||||
if m_title:
|
||||
title = _decode_html(next(g for g in m_title.groups() if g).strip())
|
||||
else:
|
||||
# Motyw streamporn.li 2026-10: h1 z zagniezdzonym <span>, tytul, rok i studio w
|
||||
# jednym napisie ("Watch TYTUL 2026 by STUDIO Porn Movie Online Free").
|
||||
m_vid = _VID_TITLE_RE.search(html)
|
||||
if not m_vid:
|
||||
log.warning("%s: no title in %s", source_name, slug)
|
||||
return None
|
||||
raw_title = _decode_html(_strip_tags(m_vid.group(1))).strip()
|
||||
m_parts = _VID_TITLE_PARTS_RE.match(raw_title)
|
||||
title = (m_parts.group("title") if m_parts else raw_title).strip()
|
||||
if m_parts and m_parts.group("year"):
|
||||
vid_year = int(m_parts.group("year"))
|
||||
if m_parts and m_parts.group("studio"):
|
||||
vid_studio = m_parts.group("studio").strip()
|
||||
if not title:
|
||||
return None
|
||||
|
||||
m_year = _YEAR_RE.search(html)
|
||||
release_year = int(m_year.group(1)) if m_year else None
|
||||
m_year = _YEAR_RE.search(html) or _YEAR_LOOSE_RE.search(html)
|
||||
release_year = int(m_year.group(1)) if m_year else vid_year
|
||||
|
||||
studio: RawStudio | None = None
|
||||
m_studio = _STUDIO_RE.search(html)
|
||||
|
|
@ -297,6 +379,14 @@ def _parse_dooplay_detail(
|
|||
name=studio_name,
|
||||
slug=studio_slug,
|
||||
)
|
||||
elif vid_studio:
|
||||
studio_slug = _slug(vid_studio)
|
||||
if studio_slug:
|
||||
studio = RawStudio(
|
||||
external_id=f"{source_name}:{studio_slug}",
|
||||
name=vid_studio,
|
||||
slug=studio_slug,
|
||||
)
|
||||
|
||||
duration_sec: int | None = None
|
||||
m_dur = _DURATION_SPAN_RE.search(html) or _DURATION_TEXT_RE.search(html)
|
||||
|
|
@ -307,6 +397,15 @@ def _parse_dooplay_detail(
|
|||
m_m = re.search(r"(\d+)\s*min", text, re.IGNORECASE)
|
||||
if m_h or m_m:
|
||||
duration_sec = (int(m_h.group(1)) * 3600 if m_h else 0) + (int(m_m.group(1)) * 60 if m_m else 0)
|
||||
if duration_sec is None:
|
||||
# streamporn.li 2026-10: pasek "29:00" (MM:SS) albo "1:29:00" (H:MM:SS).
|
||||
m_st = _ST_DURATION_RE.search(html)
|
||||
if m_st:
|
||||
a, b, c = m_st.group(1), m_st.group(2), m_st.group(3)
|
||||
if c is not None:
|
||||
duration_sec = int(a) * 3600 + int(b) * 60 + int(c)
|
||||
else:
|
||||
duration_sec = int(a) * 60 + int(b)
|
||||
|
||||
release_date: date | None = None
|
||||
m_rd = _RELEASE_DATE_SPAN_RE.search(html) or _RELEASE_DATE_TEXT_RE.search(html)
|
||||
|
|
@ -318,6 +417,13 @@ def _parse_dooplay_detail(
|
|||
break
|
||||
except ValueError:
|
||||
continue
|
||||
if release_date is None:
|
||||
m_sd = _ST_DATE_RE.search(html)
|
||||
if m_sd:
|
||||
try:
|
||||
release_date = datetime.strptime(m_sd.group(1), "%Y-%m-%d").date()
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
description: str | None = None
|
||||
m_desc = _DESCRIPTION_RE.search(html)
|
||||
|
|
@ -360,6 +466,16 @@ def _parse_dooplay_detail(
|
|||
)
|
||||
for m in _GENRE_TAG_RE.finditer(html)
|
||||
]
|
||||
if not tags:
|
||||
# streamporn.li 2026-10: chipy gatunkow bez rel="tag".
|
||||
tags = [
|
||||
RawTag(
|
||||
external_id=f"{source_name}:{m.group(1)}",
|
||||
name=_decode_html(m.group(2).strip()),
|
||||
slug=m.group(1),
|
||||
)
|
||||
for m in _GENRE_CHIP_RE.finditer(html)
|
||||
]
|
||||
|
||||
if page_url is None:
|
||||
page_url = f"{base_url}/movies/{slug}/"
|
||||
|
|
@ -416,6 +532,22 @@ def _parse_dooplay_detail(
|
|||
source = (src_match.group(1).strip() if src_match else "") or None
|
||||
_emit_host_entry(href, source)
|
||||
|
||||
# Motyw streamporn.li 2026-10 (host-links-manager): hostery w JSON-ie atrybutu
|
||||
# data-servers, HTML-escapowanym ("). "u" = adres embed, "l" = etykieta.
|
||||
for m in _DATA_SERVERS_RE.finditer(html):
|
||||
try:
|
||||
servers = json.loads(_decode_html(m.group(1)))
|
||||
except (ValueError, TypeError):
|
||||
continue
|
||||
if not isinstance(servers, list):
|
||||
continue
|
||||
for entry in servers:
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
embed = str(entry.get("u") or "").strip()
|
||||
if embed.startswith("http"):
|
||||
_emit_host_entry(embed, embed)
|
||||
|
||||
if not playback_sources:
|
||||
# Brak sub-hosters znalezionych, fallback do landing page (mobile otworzy
|
||||
# w WebView). Robimy to TYLKO gdy nie ma alternatyw, inaczej landing jest
|
||||
|
|
|
|||
|
|
@ -59,6 +59,15 @@ def _bucket(age_h: float) -> tuple[str, str]:
|
|||
return "swiezo", "warning"
|
||||
|
||||
|
||||
# Serwisy, ktore publikuja rzadziej niz co dwa dni: prog 48 h robil z nich staly alarm
|
||||
# (sprawdzone 2026-10-13: xanimeporn 1 do 3 wpisow na tydzien z RSS, hentaimama wydaje
|
||||
# odcinki raz w tygodniu, obie listy w calosci znane w bazie). Prog w godzinach.
|
||||
_BROWSE_MAX_AGE_OVERRIDE_H: dict[str, int] = {
|
||||
"xanimeporn": 168,
|
||||
"hentaimama": 240,
|
||||
}
|
||||
|
||||
|
||||
def run_ingest_freshness_watchdog(
|
||||
*,
|
||||
max_age_hours: int = 48,
|
||||
|
|
@ -86,7 +95,10 @@ def run_ingest_freshness_watchdog(
|
|||
|
||||
# (origin, tabela, próg_h, klasa)
|
||||
checks: list[tuple[str, str, int, str]] = (
|
||||
[(f"tube:{t}", "playback_sources", max_age_hours, "browse") for t in sorted(browse_tags)]
|
||||
[
|
||||
(f"tube:{t}", "playback_sources", _BROWSE_MAX_AGE_OVERRIDE_H.get(t, max_age_hours), "browse")
|
||||
for t in sorted(browse_tags)
|
||||
]
|
||||
+ [(f"tube:{t}", "playback_sources", search_max_age_hours, "search")
|
||||
for t in sorted(search_tags)]
|
||||
)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue