fix(clips): skasowany klip uruchamia sprawdzenie reszty klipow tworcy
Tworcy kasuja zwykle cale konto naraz, a przemial przy 1000 klipow na godzine robi okrazenie katalogu w okolo tydzien. Zgloszenie d343a9b6: strona tworcy z pustymi kafelkami, 120 "zywych" klipow, kazdy 403. Po oznaczeniu martwego klipu (przemial albo zgloszenie z apki) od razu HEAD na pozostalych zywych klipach tego tworcy niesprawdzanych od 6 h, do 600 na przebieg. Sprawdzone na produkcji: 120 z 120 oznaczone. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
e84e49a3a9
commit
8971d81ef9
1 changed files with 73 additions and 2 deletions
|
|
@ -200,7 +200,7 @@ def check_clips_now(clip_ids: list) -> int:
|
|||
rows = session.execute(
|
||||
select(Clip.id, Clip.hd_url, Clip.sd_url).where(Clip.id.in_(clip_ids), Clip.dead_at.is_(None))
|
||||
).all()
|
||||
marked = 0
|
||||
marked_ids: list = []
|
||||
with httpx.Client(headers=_REAP_UA, timeout=15, follow_redirects=True) as http:
|
||||
for cid, hd, sd in rows:
|
||||
url = hd or sd
|
||||
|
|
@ -212,12 +212,82 @@ def check_clips_now(clip_ids: list) -> int:
|
|||
text("UPDATE clips SET dead_at = now(), dead_reason = :r WHERE id = :id AND dead_at IS NULL"),
|
||||
{"r": f"{reason} (apka)", "id": cid},
|
||||
)
|
||||
marked += 1
|
||||
marked_ids.append(cid)
|
||||
marked = len(marked_ids)
|
||||
if marked:
|
||||
log.info("clips: %d zgloszonych z apki oznaczone jako martwe", marked)
|
||||
sweep_creators(marked_ids)
|
||||
return marked
|
||||
|
||||
|
||||
#: Ile klipow jednego przebiegu zamiatania i jak dawno sprawdzony klip bierzemy ponownie.
|
||||
_SWEEP_CAP = 600
|
||||
_SWEEP_RECHECK_HOURS = 6
|
||||
|
||||
|
||||
def sweep_creators(dead_clip_ids: list, *, cap: int = _SWEEP_CAP, sleep: float = 0.1) -> int:
|
||||
"""Po znalezieniu skasowanego klipu sprawdza od razu pozostale zywe klipy jego tworcy.
|
||||
|
||||
Tworcy kasuja zwykle cale konto naraz, a przemial przy 1000 klipow na godzine robi
|
||||
okrazenie katalogu w okolo tydzien. 2026-10-06 @thenaughtywoman mial 115 martwych
|
||||
i 120 "zywych", z ktorych kazdy oddawal 403, wiec strona tworcy pokazywala puste
|
||||
kafelki, a feed podsuwal czarne klipy. Kazdy klip sprawdzamy osobno (HEAD), nic nie
|
||||
oznaczamy w ciemno. Sprawdzone zywe dostaja swiezy updated_at, wiec ten sam tworca
|
||||
zgloszony z kilku telefonow naraz nie jest mielony w kolko. Zwraca ile oznaczono.
|
||||
"""
|
||||
import time as _time
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
import httpx
|
||||
from sqlalchemy import select
|
||||
|
||||
if not dead_clip_ids:
|
||||
return 0
|
||||
recheck_before = datetime.now(UTC) - timedelta(hours=_SWEEP_RECHECK_HOURS)
|
||||
with session_scope() as session:
|
||||
creators = [
|
||||
c for (c,) in session.execute(
|
||||
select(Clip.creator).where(Clip.id.in_(dead_clip_ids), Clip.creator.is_not(None)).distinct()
|
||||
).all()
|
||||
]
|
||||
if not creators:
|
||||
return 0
|
||||
rows = session.execute(
|
||||
select(Clip.id, Clip.hd_url, Clip.sd_url)
|
||||
.where(
|
||||
Clip.creator.in_(creators),
|
||||
Clip.dead_at.is_(None),
|
||||
Clip.updated_at < recheck_before,
|
||||
)
|
||||
.order_by(Clip.updated_at.asc())
|
||||
.limit(cap)
|
||||
).all()
|
||||
dead: list[tuple] = []
|
||||
alive: list = []
|
||||
with httpx.Client(headers=_REAP_UA, timeout=15, follow_redirects=True) as http:
|
||||
for cid, hd, sd in rows:
|
||||
url = hd or sd
|
||||
if not url:
|
||||
continue
|
||||
reason = _head_dead(http, url)
|
||||
if reason:
|
||||
dead.append((cid, reason))
|
||||
else:
|
||||
alive.append(cid)
|
||||
_time.sleep(sleep)
|
||||
with session_scope() as session:
|
||||
for cid, reason in dead:
|
||||
session.execute(
|
||||
text("UPDATE clips SET dead_at = now(), dead_reason = :r WHERE id = :id AND dead_at IS NULL"),
|
||||
{"r": f"{reason} (tworca)", "id": cid},
|
||||
)
|
||||
if alive:
|
||||
session.execute(text("UPDATE clips SET updated_at = now() WHERE id = ANY(:ids)"), {"ids": alive})
|
||||
if rows:
|
||||
log.info("clips: zamiatanie %d tworcow, sprawdzone %d, martwe %d", len(creators), len(rows), len(dead))
|
||||
return len(dead)
|
||||
|
||||
|
||||
def reap_dead(*, limit: int = 2000, apply: bool = True, sleep: float = 0.2) -> dict[str, int]:
|
||||
"""Reaper martwych klipow: tworcy kasuja klipy, RedGifs oddaje wtedy 404/410 na pliku.
|
||||
|
||||
|
|
@ -267,6 +337,7 @@ def reap_dead(*, limit: int = 2000, apply: bool = True, sleep: float = 0.2) -> d
|
|||
alive = [cid for cid, _hd, _sd in rows if cid not in dead_set]
|
||||
if alive:
|
||||
session.execute(text("UPDATE clips SET updated_at = now() WHERE id = ANY(:ids)"), {"ids": alive})
|
||||
dead += sweep_creators(list(dead_set))
|
||||
out = {"checked": checked, "dead": dead, "errors": errors, "applied": int(apply)}
|
||||
log.info("clips reaper: %s", out)
|
||||
return out
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue