diff --git a/app/connectors/direct_scrapers/hentaimama.py b/app/connectors/direct_scrapers/hentaimama.py index 202e9a8..0034e30 100644 --- a/app/connectors/direct_scrapers/hentaimama.py +++ b/app/connectors/direct_scrapers/hentaimama.py @@ -40,7 +40,8 @@ _BASE = "https://hentaimama.io" _EPISODE_HREF_RE = re.compile(r'href="(https://hentaimama\.io/episodes/[^"]+)"') # Sam listing `/episodes/` i jego paginacja też pasują do wzorca wyżej — bez tego # filtra wchodziły jako „odcinek" o tytule „Recent Episodes". -_NOT_EPISODE_RE = re.compile(r"/episodes/(?:page/\d+/)?$") +# `/episodes/feed/` (RSS) tez wpada w href-regex i konczyl jako pusty detail z None. +_NOT_EPISODE_RE = re.compile(r"/episodes/(?:page/\d+/|feed/)?$") # `og:title` zamiast

: na stronie są trzy

, dwa to logo serwisu, a trzeci bywa # nagłówkiem sekcji. og:title daje czysty „Shoujo Ramune Episode 7". _OG_TITLE_RE = re.compile(r"property=['\"]og:title['\"][^>]*content=['\"]([^'\"]{3,150})") diff --git a/app/connectors/direct_scrapers/pornbusy.py b/app/connectors/direct_scrapers/pornbusy.py index 9d00db9..addc635 100644 --- a/app/connectors/direct_scrapers/pornbusy.py +++ b/app/connectors/direct_scrapers/pornbusy.py @@ -44,6 +44,12 @@ _BASE = "https://pornbusy.com" _PAGE_SIZE = 20 _SITEMAP_INDEX = f"{_BASE}/sitemap_index.xml" +# Karta posta na stronie glownej: `
`. +_HOME_CARD_RE = re.compile( + r']*\bpost-\d+\b[^>]*>\s*\s*([^<]+?)\s*") _URL_BLOCK_RE = re.compile(r"(.*?)", re.DOTALL | re.IGNORECASE) _LASTMOD_RE = re.compile(r"\s*([^<]+?)\s*") @@ -139,8 +145,21 @@ class PornBusyScraper(BaseBrowseScraper): if not entries: return None entries.sort(key=lambda e: e[0], reverse=True) - seen: set[str] = set() - catalog: list[str] = [] + # 2026-10-13: sitemapa WordPressa aktualizuje sie z opoznieniem. Najnowszy lastmod + # w indeksie mial 9.10, a strona glowna 14 postow z 13.10, ktorych nie bylo w zadnej + # podmapie, wiec browse przez kilka dni widzial "nic nowego". Karty ze strony + # glownej (jedyna dzialajaca strona listingu, paginacja jest zepsuta) ida na + # poczatek katalogu, sitemapa dalej daje glebie. + fresh: list[str] = [] + try: + home = browser_get(f"{_BASE}/", timeout=self._timeout) + home.raise_for_status() + fresh = list(dict.fromkeys(_HOME_CARD_RE.findall(home.text))) + log.info("pornbusy: %d kart na stronie glownej", len(fresh)) + except Exception as e: + log.warning("pornbusy: homepage fetch failed: %s", e) + seen: set[str] = set(fresh) + catalog: list[str] = list(fresh) for _, url in entries: if url in seen: continue diff --git a/app/connectors/direct_scrapers/porndoe.py b/app/connectors/direct_scrapers/porndoe.py index 6a78329..74a347f 100644 --- a/app/connectors/direct_scrapers/porndoe.py +++ b/app/connectors/direct_scrapers/porndoe.py @@ -35,6 +35,7 @@ import re from datetime import date, datetime from app.config import get_settings +from app.extractors import browser_get from app.connectors.base import ( RawFingerprint, RawPerformer, @@ -54,6 +55,13 @@ _BASE = "https://porndoe.com" # Scene listing, ``. Id = `pd` + alfanum. _SCENE_URL_RE = re.compile(r'href="(/watch/[a-z0-9]+)"', re.IGNORECASE) +# Listing z sitemapy wideo (patrz crawl_page): blok , jego i data publikacji. +_SITEMAP_URL_BLOCK_RE = re.compile(r"(.*?)", re.DOTALL | re.IGNORECASE) +_SITEMAP_LOC_RE = re.compile(r"\s*([^<]+?)\s*", re.IGNORECASE) +_SITEMAP_PUBDATE_RE = re.compile( + r"\s*([^<]+?)\s*", re.IGNORECASE +) +_SITEMAP_PAGE_SIZE = 31 _WATCH_ID_RE = re.compile(r"/watch/([a-z0-9]+)", re.IGNORECASE) # JSON-LD