From 0536b1b61ced861763318e8e69da86bab351ead0 Mon Sep 17 00:00:00 2001 From: goon-foss Date: Thu, 15 Oct 2026 14:16:05 +0200 Subject: [PATCH] fix(ingest): padniete pobrania szczegolow to nie pusta strona, pandamovies na .org crawl_page oddaje None, gdy padly wszystkie pobrania szczegolow (dlawienie serwisu), zamiast [] liczonego jako pusta strona; naughtymachinima przez to skonczyl na stronie 21 z ~1800. pandamovies.pw przekierowuje na .org, a konektor odrzucal linki spoza swojej domeny (0 filmow od 20.08). Co-Authored-By: Claude Opus 5.5 --- app/connectors/direct_scrapers/_browse_base.py | 13 +++++++++++++ app/connectors/dooplay.py | 5 ++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/app/connectors/direct_scrapers/_browse_base.py b/app/connectors/direct_scrapers/_browse_base.py index 3728a83..bc565cf 100644 --- a/app/connectors/direct_scrapers/_browse_base.py +++ b/app/connectors/direct_scrapers/_browse_base.py @@ -95,12 +95,22 @@ class BaseBrowseScraper(BaseDirectTubeScraper, abc.ABC): log.info("%s browse page %d: %d scene URLs", self.sitetag, page, len(urls)) out: list[RawScene] = [] + # Pobrania szczegolow, ktore padly (wyjatek albo HTTP >= 400). Gdy padly WSZYSTKIE, + # to dlawienie serwisu, a nie pusta strona: zwracamy None (sprobuj pozniej), inaczej + # deep-crawl liczy to do pustych stron pod rzad i oznacza koniec katalogu. + # naughtymachinima: exhausted na stronie 21 z ~1800 (2026-08-21, wykryte 10-15). + failed = 0 for scene_url in urls: try: res = browser_get(scene_url, timeout=self._timeout, proxy=self._proxy) detail_html = res.text if hasattr(res, "text") else res except Exception as e: log.info("%s detail fetch failed %s: %s", self.sitetag, scene_url, e) + failed += 1 + continue + if getattr(res, "status_code", 200) >= 400: + log.info("%s detail HTTP %s %s", self.sitetag, res.status_code, scene_url) + failed += 1 continue try: raw = self._parse_detail(scene_url, detail_html) @@ -109,6 +119,9 @@ class BaseBrowseScraper(BaseDirectTubeScraper, abc.ABC): continue if raw is not None: out.append(raw) + if not out and failed == len(urls): + log.warning("%s browse page %d: wszystkie %d pobrania padly", self.sitetag, page, failed) + return None return out def latest_scenes(self, *, max_pages: int = 5) -> Iterator[RawScene]: diff --git a/app/connectors/dooplay.py b/app/connectors/dooplay.py index f33a532..ec3f1e6 100644 --- a/app/connectors/dooplay.py +++ b/app/connectors/dooplay.py @@ -625,7 +625,10 @@ class StreampornVipConnector(DooplayConnector): class PandamoviesConnector(DooplayConnector): name = "pandamovies" - base_url = "https://pandamovies.pw" + # 2026-08 pandamovies.pw -> 302 pandamovies.org; konektor odrzuca linki spoza swojej + # domeny, wiec od 20.08 ingest dawal 0 filmow. Id filmu to slug, przeprowadzka nie robi + # duplikatow (to samo co streamporn.vip -> .li). + base_url = "https://pandamovies.org" class MangopornConnector(DooplayConnector):