diff --git a/app/scheduler/clips_ingest.py b/app/scheduler/clips_ingest.py index dafac74..9348870 100644 --- a/app/scheduler/clips_ingest.py +++ b/app/scheduler/clips_ingest.py @@ -200,7 +200,7 @@ def check_clips_now(clip_ids: list) -> int: rows = session.execute( select(Clip.id, Clip.hd_url, Clip.sd_url).where(Clip.id.in_(clip_ids), Clip.dead_at.is_(None)) ).all() - marked = 0 + marked_ids: list = [] with httpx.Client(headers=_REAP_UA, timeout=15, follow_redirects=True) as http: for cid, hd, sd in rows: url = hd or sd @@ -212,12 +212,82 @@ def check_clips_now(clip_ids: list) -> int: text("UPDATE clips SET dead_at = now(), dead_reason = :r WHERE id = :id AND dead_at IS NULL"), {"r": f"{reason} (apka)", "id": cid}, ) - marked += 1 + marked_ids.append(cid) + marked = len(marked_ids) if marked: log.info("clips: %d zgloszonych z apki oznaczone jako martwe", marked) + sweep_creators(marked_ids) return marked +#: Ile klipow jednego przebiegu zamiatania i jak dawno sprawdzony klip bierzemy ponownie. +_SWEEP_CAP = 600 +_SWEEP_RECHECK_HOURS = 6 + + +def sweep_creators(dead_clip_ids: list, *, cap: int = _SWEEP_CAP, sleep: float = 0.1) -> int: + """Po znalezieniu skasowanego klipu sprawdza od razu pozostale zywe klipy jego tworcy. + + Tworcy kasuja zwykle cale konto naraz, a przemial przy 1000 klipow na godzine robi + okrazenie katalogu w okolo tydzien. 2026-10-06 @thenaughtywoman mial 115 martwych + i 120 "zywych", z ktorych kazdy oddawal 403, wiec strona tworcy pokazywala puste + kafelki, a feed podsuwal czarne klipy. Kazdy klip sprawdzamy osobno (HEAD), nic nie + oznaczamy w ciemno. Sprawdzone zywe dostaja swiezy updated_at, wiec ten sam tworca + zgloszony z kilku telefonow naraz nie jest mielony w kolko. Zwraca ile oznaczono. + """ + import time as _time + from datetime import UTC, datetime, timedelta + + import httpx + from sqlalchemy import select + + if not dead_clip_ids: + return 0 + recheck_before = datetime.now(UTC) - timedelta(hours=_SWEEP_RECHECK_HOURS) + with session_scope() as session: + creators = [ + c for (c,) in session.execute( + select(Clip.creator).where(Clip.id.in_(dead_clip_ids), Clip.creator.is_not(None)).distinct() + ).all() + ] + if not creators: + return 0 + rows = session.execute( + select(Clip.id, Clip.hd_url, Clip.sd_url) + .where( + Clip.creator.in_(creators), + Clip.dead_at.is_(None), + Clip.updated_at < recheck_before, + ) + .order_by(Clip.updated_at.asc()) + .limit(cap) + ).all() + dead: list[tuple] = [] + alive: list = [] + with httpx.Client(headers=_REAP_UA, timeout=15, follow_redirects=True) as http: + for cid, hd, sd in rows: + url = hd or sd + if not url: + continue + reason = _head_dead(http, url) + if reason: + dead.append((cid, reason)) + else: + alive.append(cid) + _time.sleep(sleep) + with session_scope() as session: + for cid, reason in dead: + session.execute( + text("UPDATE clips SET dead_at = now(), dead_reason = :r WHERE id = :id AND dead_at IS NULL"), + {"r": f"{reason} (tworca)", "id": cid}, + ) + if alive: + session.execute(text("UPDATE clips SET updated_at = now() WHERE id = ANY(:ids)"), {"ids": alive}) + if rows: + log.info("clips: zamiatanie %d tworcow, sprawdzone %d, martwe %d", len(creators), len(rows), len(dead)) + return len(dead) + + def reap_dead(*, limit: int = 2000, apply: bool = True, sleep: float = 0.2) -> dict[str, int]: """Reaper martwych klipow: tworcy kasuja klipy, RedGifs oddaje wtedy 404/410 na pliku. @@ -267,6 +337,7 @@ def reap_dead(*, limit: int = 2000, apply: bool = True, sleep: float = 0.2) -> d alive = [cid for cid, _hd, _sd in rows if cid not in dead_set] if alive: session.execute(text("UPDATE clips SET updated_at = now() WHERE id = ANY(:ids)"), {"ids": alive}) + dead += sweep_creators(list(dead_set)) out = {"checked": checked, "dead": dead, "errors": errors, "applied": int(apply)} log.info("clips reaper: %s", out) return out