goon-foss/goon
1
0
Fork 0
Code Issues Pull requests Projects Releases 2 Packages Wiki Activity Actions 2

fix(ingest): padniete pobrania szczegolow to nie pusta strona, pandamovies na .org

Browse source 
crawl_page oddaje None, gdy padly wszystkie pobrania szczegolow (dlawienie
serwisu), zamiast [] liczonego jako pusta strona; naughtymachinima przez to
skonczyl na stronie 21 z ~1800. pandamovies.pw przekierowuje na .org, a
konektor odrzucal linki spoza swojej domeny (0 filmow od 20.08).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
goon-foss 2026-10-15 14:16:05 +02:00
parent b19d041f09
commit 0536b1b61c
2 changed files with 17 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
app/connectors/direct_scrapers/_browse_base.py
View file

@ -95,12 +95,22 @@ class BaseBrowseScraper(BaseDirectTubeScraper, abc.ABC):
log.info("%s browse page %d: %d scene URLs", self.sitetag, page, len(urls))
out: list[RawScene] = []
# Pobrania szczegolow, ktore padly (wyjatek albo HTTP >= 400). Gdy padly WSZYSTKIE,
# to dlawienie serwisu, a nie pusta strona: zwracamy None (sprobuj pozniej), inaczej
# deep-crawl liczy to do pustych stron pod rzad i oznacza koniec katalogu.
# naughtymachinima: exhausted na stronie 21 z ~1800 (2026-08-21, wykryte 10-15).
failed = 0
for scene_url in urls:
try:
res = browser_get(scene_url, timeout=self._timeout, proxy=self._proxy)
detail_html = res.text if hasattr(res, "text") else res
except Exception as e:
log.info("%s detail fetch failed %s: %s", self.sitetag, scene_url, e)
failed += 1
continue
if getattr(res, "status_code", 200) >= 400:
log.info("%s detail HTTP %s %s", self.sitetag, res.status_code, scene_url)
failed += 1
continue
try:
raw = self._parse_detail(scene_url, detail_html)
@ -109,6 +119,9 @@ class BaseBrowseScraper(BaseDirectTubeScraper, abc.ABC):
continue
if raw is not None:
out.append(raw)
if not out and failed == len(urls):
log.warning("%s browse page %d: wszystkie %d pobrania padly", self.sitetag, page, failed)
return None
return out
def latest_scenes(self, *, max_pages: int = 5) -> Iterator[RawScene]:

5
app/connectors/dooplay.py
View file

@ -625,7 +625,10 @@ class StreampornVipConnector(DooplayConnector):
class PandamoviesConnector(DooplayConnector):
name = "pandamovies"
base_url = "https://pandamovies.pw"
# 2026-08 pandamovies.pw -> 302 pandamovies.org; konektor odrzuca linki spoza swojej
# domeny, wiec od 20.08 ingest dawal 0 filmow. Id filmu to slug, przeprowadzka nie robi
# duplikatow (to samo co streamporn.vip -> .li).
base_url = "https://pandamovies.org"
class MangopornConnector(DooplayConnector):