diff --git a/app/connectors/direct_scrapers/_browse_base.py b/app/connectors/direct_scrapers/_browse_base.py index 3728a83..bc565cf 100644 --- a/app/connectors/direct_scrapers/_browse_base.py +++ b/app/connectors/direct_scrapers/_browse_base.py @@ -95,12 +95,22 @@ class BaseBrowseScraper(BaseDirectTubeScraper, abc.ABC): log.info("%s browse page %d: %d scene URLs", self.sitetag, page, len(urls)) out: list[RawScene] = [] + # Pobrania szczegolow, ktore padly (wyjatek albo HTTP >= 400). Gdy padly WSZYSTKIE, + # to dlawienie serwisu, a nie pusta strona: zwracamy None (sprobuj pozniej), inaczej + # deep-crawl liczy to do pustych stron pod rzad i oznacza koniec katalogu. + # naughtymachinima: exhausted na stronie 21 z ~1800 (2026-08-21, wykryte 10-15). + failed = 0 for scene_url in urls: try: res = browser_get(scene_url, timeout=self._timeout, proxy=self._proxy) detail_html = res.text if hasattr(res, "text") else res except Exception as e: log.info("%s detail fetch failed %s: %s", self.sitetag, scene_url, e) + failed += 1 + continue + if getattr(res, "status_code", 200) >= 400: + log.info("%s detail HTTP %s %s", self.sitetag, res.status_code, scene_url) + failed += 1 continue try: raw = self._parse_detail(scene_url, detail_html) @@ -109,6 +119,9 @@ class BaseBrowseScraper(BaseDirectTubeScraper, abc.ABC): continue if raw is not None: out.append(raw) + if not out and failed == len(urls): + log.warning("%s browse page %d: wszystkie %d pobrania padly", self.sitetag, page, failed) + return None return out def latest_scenes(self, *, max_pages: int = 5) -> Iterator[RawScene]: diff --git a/app/connectors/dooplay.py b/app/connectors/dooplay.py index f33a532..ec3f1e6 100644 --- a/app/connectors/dooplay.py +++ b/app/connectors/dooplay.py @@ -625,7 +625,10 @@ class StreampornVipConnector(DooplayConnector): class PandamoviesConnector(DooplayConnector): name = "pandamovies" - base_url = "https://pandamovies.pw" + # 2026-08 pandamovies.pw -> 302 pandamovies.org; konektor odrzuca linki spoza swojej + # domeny, wiec od 20.08 ingest dawal 0 filmow. Id filmu to slug, przeprowadzka nie robi + # duplikatow (to samo co streamporn.vip -> .li). + base_url = "https://pandamovies.org" class MangopornConnector(DooplayConnector):