fix(clips): skasowany klip uruchamia sprawdzenie reszty klipow tworcy
Tworcy kasuja zwykle cale konto naraz, a przemial przy 1000 klipow na godzine robi okrazenie katalogu w okolo tydzien. Zgloszenie d343a9b6: strona tworcy z pustymi kafelkami, 120 "zywych" klipow, kazdy 403. Po oznaczeniu martwego klipu (przemial albo zgloszenie z apki) od razu HEAD na pozostalych zywych klipach tego tworcy niesprawdzanych od 6 h, do 600 na przebieg. Sprawdzone na produkcji: 120 z 120 oznaczone. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
e84e49a3a9
commit
8971d81ef9
1 changed files with 73 additions and 2 deletions
|
|
@ -200,7 +200,7 @@ def check_clips_now(clip_ids: list) -> int:
|
||||||
rows = session.execute(
|
rows = session.execute(
|
||||||
select(Clip.id, Clip.hd_url, Clip.sd_url).where(Clip.id.in_(clip_ids), Clip.dead_at.is_(None))
|
select(Clip.id, Clip.hd_url, Clip.sd_url).where(Clip.id.in_(clip_ids), Clip.dead_at.is_(None))
|
||||||
).all()
|
).all()
|
||||||
marked = 0
|
marked_ids: list = []
|
||||||
with httpx.Client(headers=_REAP_UA, timeout=15, follow_redirects=True) as http:
|
with httpx.Client(headers=_REAP_UA, timeout=15, follow_redirects=True) as http:
|
||||||
for cid, hd, sd in rows:
|
for cid, hd, sd in rows:
|
||||||
url = hd or sd
|
url = hd or sd
|
||||||
|
|
@ -212,12 +212,82 @@ def check_clips_now(clip_ids: list) -> int:
|
||||||
text("UPDATE clips SET dead_at = now(), dead_reason = :r WHERE id = :id AND dead_at IS NULL"),
|
text("UPDATE clips SET dead_at = now(), dead_reason = :r WHERE id = :id AND dead_at IS NULL"),
|
||||||
{"r": f"{reason} (apka)", "id": cid},
|
{"r": f"{reason} (apka)", "id": cid},
|
||||||
)
|
)
|
||||||
marked += 1
|
marked_ids.append(cid)
|
||||||
|
marked = len(marked_ids)
|
||||||
if marked:
|
if marked:
|
||||||
log.info("clips: %d zgloszonych z apki oznaczone jako martwe", marked)
|
log.info("clips: %d zgloszonych z apki oznaczone jako martwe", marked)
|
||||||
|
sweep_creators(marked_ids)
|
||||||
return marked
|
return marked
|
||||||
|
|
||||||
|
|
||||||
|
#: Ile klipow jednego przebiegu zamiatania i jak dawno sprawdzony klip bierzemy ponownie.
|
||||||
|
_SWEEP_CAP = 600
|
||||||
|
_SWEEP_RECHECK_HOURS = 6
|
||||||
|
|
||||||
|
|
||||||
|
def sweep_creators(dead_clip_ids: list, *, cap: int = _SWEEP_CAP, sleep: float = 0.1) -> int:
|
||||||
|
"""Po znalezieniu skasowanego klipu sprawdza od razu pozostale zywe klipy jego tworcy.
|
||||||
|
|
||||||
|
Tworcy kasuja zwykle cale konto naraz, a przemial przy 1000 klipow na godzine robi
|
||||||
|
okrazenie katalogu w okolo tydzien. 2026-10-06 @thenaughtywoman mial 115 martwych
|
||||||
|
i 120 "zywych", z ktorych kazdy oddawal 403, wiec strona tworcy pokazywala puste
|
||||||
|
kafelki, a feed podsuwal czarne klipy. Kazdy klip sprawdzamy osobno (HEAD), nic nie
|
||||||
|
oznaczamy w ciemno. Sprawdzone zywe dostaja swiezy updated_at, wiec ten sam tworca
|
||||||
|
zgloszony z kilku telefonow naraz nie jest mielony w kolko. Zwraca ile oznaczono.
|
||||||
|
"""
|
||||||
|
import time as _time
|
||||||
|
from datetime import UTC, datetime, timedelta
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
from sqlalchemy import select
|
||||||
|
|
||||||
|
if not dead_clip_ids:
|
||||||
|
return 0
|
||||||
|
recheck_before = datetime.now(UTC) - timedelta(hours=_SWEEP_RECHECK_HOURS)
|
||||||
|
with session_scope() as session:
|
||||||
|
creators = [
|
||||||
|
c for (c,) in session.execute(
|
||||||
|
select(Clip.creator).where(Clip.id.in_(dead_clip_ids), Clip.creator.is_not(None)).distinct()
|
||||||
|
).all()
|
||||||
|
]
|
||||||
|
if not creators:
|
||||||
|
return 0
|
||||||
|
rows = session.execute(
|
||||||
|
select(Clip.id, Clip.hd_url, Clip.sd_url)
|
||||||
|
.where(
|
||||||
|
Clip.creator.in_(creators),
|
||||||
|
Clip.dead_at.is_(None),
|
||||||
|
Clip.updated_at < recheck_before,
|
||||||
|
)
|
||||||
|
.order_by(Clip.updated_at.asc())
|
||||||
|
.limit(cap)
|
||||||
|
).all()
|
||||||
|
dead: list[tuple] = []
|
||||||
|
alive: list = []
|
||||||
|
with httpx.Client(headers=_REAP_UA, timeout=15, follow_redirects=True) as http:
|
||||||
|
for cid, hd, sd in rows:
|
||||||
|
url = hd or sd
|
||||||
|
if not url:
|
||||||
|
continue
|
||||||
|
reason = _head_dead(http, url)
|
||||||
|
if reason:
|
||||||
|
dead.append((cid, reason))
|
||||||
|
else:
|
||||||
|
alive.append(cid)
|
||||||
|
_time.sleep(sleep)
|
||||||
|
with session_scope() as session:
|
||||||
|
for cid, reason in dead:
|
||||||
|
session.execute(
|
||||||
|
text("UPDATE clips SET dead_at = now(), dead_reason = :r WHERE id = :id AND dead_at IS NULL"),
|
||||||
|
{"r": f"{reason} (tworca)", "id": cid},
|
||||||
|
)
|
||||||
|
if alive:
|
||||||
|
session.execute(text("UPDATE clips SET updated_at = now() WHERE id = ANY(:ids)"), {"ids": alive})
|
||||||
|
if rows:
|
||||||
|
log.info("clips: zamiatanie %d tworcow, sprawdzone %d, martwe %d", len(creators), len(rows), len(dead))
|
||||||
|
return len(dead)
|
||||||
|
|
||||||
|
|
||||||
def reap_dead(*, limit: int = 2000, apply: bool = True, sleep: float = 0.2) -> dict[str, int]:
|
def reap_dead(*, limit: int = 2000, apply: bool = True, sleep: float = 0.2) -> dict[str, int]:
|
||||||
"""Reaper martwych klipow: tworcy kasuja klipy, RedGifs oddaje wtedy 404/410 na pliku.
|
"""Reaper martwych klipow: tworcy kasuja klipy, RedGifs oddaje wtedy 404/410 na pliku.
|
||||||
|
|
||||||
|
|
@ -267,6 +337,7 @@ def reap_dead(*, limit: int = 2000, apply: bool = True, sleep: float = 0.2) -> d
|
||||||
alive = [cid for cid, _hd, _sd in rows if cid not in dead_set]
|
alive = [cid for cid, _hd, _sd in rows if cid not in dead_set]
|
||||||
if alive:
|
if alive:
|
||||||
session.execute(text("UPDATE clips SET updated_at = now() WHERE id = ANY(:ids)"), {"ids": alive})
|
session.execute(text("UPDATE clips SET updated_at = now() WHERE id = ANY(:ids)"), {"ids": alive})
|
||||||
|
dead += sweep_creators(list(dead_set))
|
||||||
out = {"checked": checked, "dead": dead, "errors": errors, "applied": int(apply)}
|
out = {"checked": checked, "dead": dead, "errors": errors, "applied": int(apply)}
|
||||||
log.info("clips reaper: %s", out)
|
log.info("clips reaper: %s", out)
|
||||||
return out
|
return out
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue