"""Reaper martwych klipów: twórcy kasują klipy, RedGifs oddaje wtedy 404/410 na pliku. Sprawdza HEAD `hd_url` (albo `sd_url`) dla próbki żywych klipów: najpierw te najdawniej sprawdzane (`updated_at`), bo ingest `latest` i tak odświeża świeże. 404/410/403 → `dead_at` + `dead_reason`; błędy sieci nie oznaczają śmierci (patrz notatka o dwóch kontrolach przy przemiale źródeł). Klip, który źródło znów poda w listingu, upsert odżywia (zdejmuje `dead_at`). Uruchomienie (kontener worker): python -m scripts.clips_reap_dead --limit 500 # dry-run python -m scripts.clips_reap_dead --limit 500 --yes # zapisz """ from __future__ import annotations import argparse import logging import time import httpx from sqlalchemy import select, text from app.db import session_scope from app.models.clip import Clip log = logging.getLogger(__name__) _UA = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/152.0.0.0 Safari/537.36"} _DEAD_CODES = {403, 404, 410} def run(*, limit: int, apply: bool) -> dict[str, int]: checked = dead = errors = 0 dead_ids: list = [] with session_scope() as session: rows = session.execute( select(Clip.id, Clip.hd_url, Clip.sd_url) .where(Clip.dead_at.is_(None)) .order_by(Clip.updated_at.asc()) .limit(limit) ).all() with httpx.Client(headers=_UA, timeout=20, follow_redirects=True) as http: for cid, hd, sd in rows: url = hd or sd if not url: continue checked += 1 try: r = http.head(url) except httpx.HTTPError: errors += 1 continue if r.status_code in _DEAD_CODES: dead += 1 dead_ids.append((cid, f"http {r.status_code}")) time.sleep(0.2) if apply: with session_scope() as session: for cid, reason in dead_ids: session.execute( text("UPDATE clips SET dead_at = now(), dead_reason = :r WHERE id = :id AND dead_at IS NULL"), {"r": reason, "id": cid}, ) # Sprawdzone i zywe: odswiez updated_at, zeby kolejny run bral inne. alive = [cid for cid, _hd, _sd in rows if cid not in {d[0] for d in dead_ids}] if alive: session.execute( text("UPDATE clips SET updated_at = now() WHERE id = ANY(:ids)"), {"ids": alive} ) out = {"checked": checked, "dead": dead, "errors": errors, "applied": int(apply)} print(out) return out def main() -> None: logging.basicConfig(level=logging.INFO, format="%(message)s") ap = argparse.ArgumentParser() ap.add_argument("--limit", type=int, default=500) ap.add_argument("--yes", action="store_true", help="zapisz dead_at (bez tego dry-run)") a = ap.parse_args() run(limit=a.limit, apply=a.yes) if __name__ == "__main__": main()