fix(clips): skasowany klip uruchamia sprawdzenie reszty klipow tworcy

Tworcy kasuja zwykle cale konto naraz, a przemial przy 1000 klipow na
godzine robi okrazenie katalogu w okolo tydzien. Zgloszenie d343a9b6:
strona tworcy z pustymi kafelkami, 120 "zywych" klipow, kazdy 403.
Po oznaczeniu martwego klipu (przemial albo zgloszenie z apki) od razu
HEAD na pozostalych zywych klipach tego tworcy niesprawdzanych od 6 h,
do 600 na przebieg. Sprawdzone na produkcji: 120 z 120 oznaczone.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
goon-foss 2026-10-06 15:38:54 +02:00
parent e84e49a3a9
commit 8971d81ef9

View file

@ -200,7 +200,7 @@ def check_clips_now(clip_ids: list) -> int:
rows = session.execute( rows = session.execute(
select(Clip.id, Clip.hd_url, Clip.sd_url).where(Clip.id.in_(clip_ids), Clip.dead_at.is_(None)) select(Clip.id, Clip.hd_url, Clip.sd_url).where(Clip.id.in_(clip_ids), Clip.dead_at.is_(None))
).all() ).all()
marked = 0 marked_ids: list = []
with httpx.Client(headers=_REAP_UA, timeout=15, follow_redirects=True) as http: with httpx.Client(headers=_REAP_UA, timeout=15, follow_redirects=True) as http:
for cid, hd, sd in rows: for cid, hd, sd in rows:
url = hd or sd url = hd or sd
@ -212,12 +212,82 @@ def check_clips_now(clip_ids: list) -> int:
text("UPDATE clips SET dead_at = now(), dead_reason = :r WHERE id = :id AND dead_at IS NULL"), text("UPDATE clips SET dead_at = now(), dead_reason = :r WHERE id = :id AND dead_at IS NULL"),
{"r": f"{reason} (apka)", "id": cid}, {"r": f"{reason} (apka)", "id": cid},
) )
marked += 1 marked_ids.append(cid)
marked = len(marked_ids)
if marked: if marked:
log.info("clips: %d zgloszonych z apki oznaczone jako martwe", marked) log.info("clips: %d zgloszonych z apki oznaczone jako martwe", marked)
sweep_creators(marked_ids)
return marked return marked
#: Ile klipow jednego przebiegu zamiatania i jak dawno sprawdzony klip bierzemy ponownie.
_SWEEP_CAP = 600
_SWEEP_RECHECK_HOURS = 6
def sweep_creators(dead_clip_ids: list, *, cap: int = _SWEEP_CAP, sleep: float = 0.1) -> int:
"""Po znalezieniu skasowanego klipu sprawdza od razu pozostale zywe klipy jego tworcy.
Tworcy kasuja zwykle cale konto naraz, a przemial przy 1000 klipow na godzine robi
okrazenie katalogu w okolo tydzien. 2026-10-06 @thenaughtywoman mial 115 martwych
i 120 "zywych", z ktorych kazdy oddawal 403, wiec strona tworcy pokazywala puste
kafelki, a feed podsuwal czarne klipy. Kazdy klip sprawdzamy osobno (HEAD), nic nie
oznaczamy w ciemno. Sprawdzone zywe dostaja swiezy updated_at, wiec ten sam tworca
zgloszony z kilku telefonow naraz nie jest mielony w kolko. Zwraca ile oznaczono.
"""
import time as _time
from datetime import UTC, datetime, timedelta
import httpx
from sqlalchemy import select
if not dead_clip_ids:
return 0
recheck_before = datetime.now(UTC) - timedelta(hours=_SWEEP_RECHECK_HOURS)
with session_scope() as session:
creators = [
c for (c,) in session.execute(
select(Clip.creator).where(Clip.id.in_(dead_clip_ids), Clip.creator.is_not(None)).distinct()
).all()
]
if not creators:
return 0
rows = session.execute(
select(Clip.id, Clip.hd_url, Clip.sd_url)
.where(
Clip.creator.in_(creators),
Clip.dead_at.is_(None),
Clip.updated_at < recheck_before,
)
.order_by(Clip.updated_at.asc())
.limit(cap)
).all()
dead: list[tuple] = []
alive: list = []
with httpx.Client(headers=_REAP_UA, timeout=15, follow_redirects=True) as http:
for cid, hd, sd in rows:
url = hd or sd
if not url:
continue
reason = _head_dead(http, url)
if reason:
dead.append((cid, reason))
else:
alive.append(cid)
_time.sleep(sleep)
with session_scope() as session:
for cid, reason in dead:
session.execute(
text("UPDATE clips SET dead_at = now(), dead_reason = :r WHERE id = :id AND dead_at IS NULL"),
{"r": f"{reason} (tworca)", "id": cid},
)
if alive:
session.execute(text("UPDATE clips SET updated_at = now() WHERE id = ANY(:ids)"), {"ids": alive})
if rows:
log.info("clips: zamiatanie %d tworcow, sprawdzone %d, martwe %d", len(creators), len(rows), len(dead))
return len(dead)
def reap_dead(*, limit: int = 2000, apply: bool = True, sleep: float = 0.2) -> dict[str, int]: def reap_dead(*, limit: int = 2000, apply: bool = True, sleep: float = 0.2) -> dict[str, int]:
"""Reaper martwych klipow: tworcy kasuja klipy, RedGifs oddaje wtedy 404/410 na pliku. """Reaper martwych klipow: tworcy kasuja klipy, RedGifs oddaje wtedy 404/410 na pliku.
@ -267,6 +337,7 @@ def reap_dead(*, limit: int = 2000, apply: bool = True, sleep: float = 0.2) -> d
alive = [cid for cid, _hd, _sd in rows if cid not in dead_set] alive = [cid for cid, _hd, _sd in rows if cid not in dead_set]
if alive: if alive:
session.execute(text("UPDATE clips SET updated_at = now() WHERE id = ANY(:ids)"), {"ids": alive}) session.execute(text("UPDATE clips SET updated_at = now() WHERE id = ANY(:ids)"), {"ids": alive})
dead += sweep_creators(list(dead_set))
out = {"checked": checked, "dead": dead, "errors": errors, "applied": int(apply)} out = {"checked": checked, "dead": dead, "errors": errors, "applied": int(apply)}
log.info("clips reaper: %s", out) log.info("clips reaper: %s", out)
return out return out