goon-foss/goon
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(deep-crawl): soft per-run time budget so a slow tube can't hit the hard kill

Browse source 
run_deep_crawl picks one tube/run and crawls 60 pages under _job_deep_crawl's 3600s
hard timeout. A detail-fetch scraper on a slow patch (per-scene page fetch, e.g. via
proxy) could exceed it → the run is killed mid-page, the cursor is never saved (orphan
thread), and that tube makes zero progress — recurring Sentry GOON-V. Added a 3000s
in-run budget that breaks after a completed page, saves the cursor, and returns cleanly;
the next run continues. budget_hit surfaced in the summary log to spot the slow tube.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
goon-foss 2026-06-29 11:21:41 +02:00
parent ac17350a67
commit 449b7e30d3
1 changed files with 25 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

29
app/scheduler/deep_crawl.py
View file

@ -39,6 +39,13 @@ _PAGE_CAP: dict[str, int] = {
"xvideoscom": 1800,
}
# Miękki budżet czasu na run (s). Detail-fetch scrapery (per-scena fetch strony, np.
# przez wolne proxy) potrafią przekroczyć hard-timeout 3600s z _job_deep_crawl → run
# ubijany w locie, kursor NIE zapisany (orphan thread), tube zero postępu + alert
# GOON-V. Budżet < hard-timeout: przerywamy PO skończonej stronie, zapisujemy kursor,
# wracamy czysto — następny run kontynuuje. Margines 600s na dokończenie strony w toku.
_RUN_BUDGET_SEC = 3000
def _state_path() -> Path:
return Path(getattr(get_settings(), "deepcrawl_state_path", None) or _DEFAULT_STATE)
@ -119,6 +126,7 @@ def run_deep_crawl(*, pages_per_run: int = 60, sitetags: list[str] | None = None
t0 = time.time()
last_done = start - 1
exhausted = False
budget_hit = False
if cap is not None and start > cap:
# kursor osiągnął per-tube cap → traktuj jak koniec katalogu (reset re-sweepuje od 1)
@ -141,7 +149,17 @@ def run_deep_crawl(*, pages_per_run: int = 60, sitetags: list[str] | None = None
except Exception:
counters["errors"] += 1
last_done = page
if cap is not None and last_done >= cap:
# Miękki budżet: stop po skończonej stronie (kursor=last_done zapisany niżej),
# zanim hard-timeout ubije run mid-page (orphan thread, kursor zgubiony — GOON-V).
if time.time() - t0 > _RUN_BUDGET_SEC:
budget_hit = True
log.warning(
"deep-crawl %s: run budget %ds hit at page %d (%d/%d stron) — stop czysto, "
"kursor zapisany, kontynuacja w następnym runie",
sitetag, _RUN_BUDGET_SEC, page, page - start + 1, pages_per_run,
)
break
if not budget_hit and cap is not None and last_done >= cap:
log.info("deep-crawl %s: reached page cap %d (exhausted)", sitetag, cap)
exhausted = True
@ -152,7 +170,10 @@ def run_deep_crawl(*, pages_per_run: int = 60, sitetags: list[str] | None = None
_save_state(state)
log.info(
"deep-crawl %s pages %d-%d: %s exhausted=%s (%.0fs)",
sitetag, start, last_done, counters, exhausted, time.time() - t0,
"deep-crawl %s pages %d-%d: %s exhausted=%s budget_hit=%s (%.0fs)",
sitetag, start, last_done, counters, exhausted, budget_hit, time.time() - t0,
)
return {"sitetag": sitetag, "start": start, "end": last_done, "exhausted": exhausted, **counters}
return {
"sitetag": sitetag, "start": start, "end": last_done,
"exhausted": exhausted, "budget_hit": budget_hit, **counters,
}