09f389100a
Some checks are pending
Backend tests / test (push) Waiting to run
- browse_runs (migracja 0056): jeden wiersz per scraper i przebieg browse-latest - quality_watchdog: listing pusty albo bez nowych, skok lub zanik obsady, sceny bez dlugosci i miniatur, spadek udanych odtworzen per origin; fingerprint per zrodlo i rodzaj - ingest_watchdog: prog swiezosci z mediany odstepow zrodla (3x, max 14 d) zamiast stalej - duration backfill: resolvery perverzija (JSON-LD) i mypornerleak (itemprop), paczka po rowno z originow, 404/410 zamyka wiersz od razu - parsery: galaxyporn data z JSON-LD, hentaimama dlugosc, javflix aktorki, fullmovies i hdporngg data z uploadDate
97 lines
3.9 KiB
Python
97 lines
3.9 KiB
Python
"""Browse-latest strategy, scrap newest scenes from rich-metadata tubes.
|
||
|
||
Pattern (vs performer-driven):
|
||
- performer-driven: backfill scen dla znanych performerów (TPDB/StashDB → tube).
|
||
Wymaga już znanego performera w DB.
|
||
- browse-latest: forward-fill świeżymi scenami z tube'ów (~100 najnowszych /
|
||
tube / dzień). Łapie sceny których performer może być new dla nas, później
|
||
canonical ingest dorobi metadata.
|
||
|
||
Każdy `BaseBrowseScraper.latest_scenes(max_pages=5)` yielduje RawScene z bogatą
|
||
metadata (studio + performers + duration + tags + description). Composite fuzzy
|
||
w resolverze ma więc dobre sygnały dla canonical match (vs orphan-only tubes
|
||
typu pornditt, gdzie był sam title + krótki opis).
|
||
|
||
Schedulowane przez `jobs.py` co 6h / 4×dobę (`sched_browse_latest_hours=6`).
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import logging
|
||
import time
|
||
from dataclasses import dataclass
|
||
|
||
from app.connectors.direct_scrapers import ALL_BROWSE_SCRAPERS, SCRAPER_SOURCE_NAME
|
||
from app.db import session_scope
|
||
from app.models.browse_run import BrowseRun
|
||
from app.models.source import SourceKind
|
||
from app.scheduler.performer_driven import _ingest_iter_into_run
|
||
|
||
log = logging.getLogger(__name__)
|
||
|
||
|
||
def _record_run(sitetag: str, counters: dict | None, elapsed: float, failed: bool, note: str | None) -> None:
|
||
"""Jeden wiersz browse_runs per scraper i przebieg; czyta go watchdog listingu.
|
||
Blad zapisu nie moze zatrzymac browse (log i dalej)."""
|
||
c = counters or {}
|
||
try:
|
||
with session_scope() as s:
|
||
s.add(
|
||
BrowseRun(
|
||
sitetag=sitetag,
|
||
elapsed_s=round(elapsed, 1),
|
||
seen=int(c.get("seen", 0)),
|
||
new=int(c.get("new", 0)),
|
||
updated=int(c.get("updated", 0)),
|
||
skipped=int(c.get("skipped", 0)),
|
||
errors=int(c.get("errors", 0)),
|
||
failed=failed,
|
||
note=(note or None) and note[:500],
|
||
)
|
||
)
|
||
except Exception: # pragma: no cover - tabela moze nie istniec przed migracja 0056
|
||
log.exception("browse-latest: zapis browse_runs dla %s nie powiodl sie", sitetag)
|
||
|
||
|
||
@dataclass
|
||
class BrowseCounters:
|
||
scrapers_run: int = 0
|
||
total_seen: int = 0
|
||
total_new: int = 0
|
||
total_updated: int = 0
|
||
total_errors: int = 0
|
||
|
||
|
||
def run_browse_latest(*, max_pages: int = 5) -> BrowseCounters:
|
||
"""Iteruj wszystkie zarejestrowane browse scrapers, scrap latest N pages każdy."""
|
||
counters = BrowseCounters()
|
||
for scraper_cls in ALL_BROWSE_SCRAPERS:
|
||
scraper = scraper_cls()
|
||
t0 = time.time()
|
||
log.info("browse-latest: %s starting (max_pages=%d)", scraper.sitetag, max_pages)
|
||
try:
|
||
c = _ingest_iter_into_run(
|
||
source_kind=SourceKind.scraper,
|
||
source_name=SCRAPER_SOURCE_NAME,
|
||
run_label=f"browse-latest:{scraper.sitetag}",
|
||
iterator_factory=lambda s=scraper, mp=max_pages: s.latest_scenes(
|
||
max_pages=mp
|
||
),
|
||
)
|
||
counters.scrapers_run += 1
|
||
counters.total_seen += c.get("seen", 0)
|
||
counters.total_new += c.get("new", 0)
|
||
counters.total_updated += c.get("updated", 0)
|
||
counters.total_errors += c.get("errors", 0)
|
||
elapsed = time.time() - t0
|
||
log.info(
|
||
"browse-latest: %s done in %.1fs counters=%s",
|
||
scraper.sitetag, elapsed, c,
|
||
)
|
||
_record_run(scraper.sitetag, c, elapsed, failed=False, note=None)
|
||
except Exception as e:
|
||
counters.total_errors += 1
|
||
log.exception("browse-latest: %s failed: %s", scraper.sitetag, e)
|
||
_record_run(scraper.sitetag, None, time.time() - t0, failed=True, note=f"{type(e).__name__}: {e}")
|
||
|
||
log.info("browse-latest done: %s", counters)
|
||
return counters
|