2dbb9ba65f
resolve_kvs zwracal None na 404, API dawalo 503 bez konca i zrodlo nigdy nie znikalo; ok. polowa yespornvip to usuniete strony. Teraz TubePageError, wiec resolve oznacza je martwe. Skrypt reap_kvs_404 czysci zaleglosci (dwa 404 w odstepie i brak odtwarzacza). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
98 lines
3.4 KiB
Python
98 lines
3.4 KiB
Python
"""Oznacza martwe zrodla KVS (yespornvip i inne) ze strona sceny 404/410 (2026-10-15).
|
|
|
|
resolve_kvs do 15.10 zamienial 404 na "chwilowy blad", wiec usuniete filmy wisialy jako
|
|
zywe (yespornvip: ok. polowa z 39 tys. zrodel, zgloszenie 5e81eb76). Od 15.10 resolve sam
|
|
oznacza 404 przy kliknieciu, ten skrypt czysci zaleglosci bez czekania na klikniecia.
|
|
|
|
Dwie kontrole (patrz notatka o przemiale zrodel): strona daje 404/410 DWA razy, w odstepie
|
|
co najmniej --recheck-after sekund, i w tresci nie ma odtwarzacza (license_code). Przekierowanie
|
|
na nowy adres z odtwarzaczem to zywe zrodlo i zostaje.
|
|
|
|
python scripts/reap_kvs_404.py --origin tube:yespornvip --limit 100 # podglad
|
|
python scripts/reap_kvs_404.py --origin tube:yespornvip --limit 50000 --apply
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import logging
|
|
import time
|
|
from datetime import UTC, datetime
|
|
|
|
import sqlalchemy as sa
|
|
|
|
from app.db import session_scope
|
|
from app.extractors._fetch import browser_get
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
def _status(url: str) -> tuple[int, bool]:
|
|
try:
|
|
r = browser_get(url, timeout=20.0, follow_redirects=True)
|
|
except Exception:
|
|
return 0, False
|
|
return r.status_code, "license_code" in (r.text or "")
|
|
|
|
|
|
def run(origin: str, limit: int, apply: bool, rps: float, recheck_after: float) -> dict[str, int]:
|
|
c = {"checked": 0, "alive": 0, "first_404": 0, "dead": 0, "other": 0}
|
|
with session_scope() as s:
|
|
rows = s.execute(
|
|
sa.text(
|
|
"SELECT id, page_url FROM playback_sources WHERE origin = :o AND dead_at IS NULL "
|
|
"ORDER BY id LIMIT :lim"
|
|
),
|
|
{"o": origin, "lim": limit},
|
|
).all()
|
|
suspects: list[tuple] = []
|
|
for pb_id, url in rows:
|
|
c["checked"] += 1
|
|
st, player = _status(url)
|
|
if st in (404, 410) and not player:
|
|
c["first_404"] += 1
|
|
suspects.append((pb_id, url, time.time()))
|
|
elif st and st < 400:
|
|
c["alive"] += 1
|
|
else:
|
|
c["other"] += 1
|
|
if c["checked"] % 500 == 0:
|
|
log.info("kvs reap pass1 %s", c)
|
|
time.sleep(1.0 / rps)
|
|
dead: list = []
|
|
for pb_id, url, t0 in suspects:
|
|
wait = recheck_after - (time.time() - t0)
|
|
if wait > 0:
|
|
time.sleep(wait)
|
|
st, player = _status(url)
|
|
if st in (404, 410) and not player:
|
|
dead.append(pb_id)
|
|
time.sleep(1.0 / rps)
|
|
c["dead"] = len(dead)
|
|
print(c)
|
|
if apply and dead:
|
|
reason = f"tube page 404 (reap_kvs_404 {datetime.now(UTC):%Y-%m-%d})"
|
|
for i in range(0, len(dead), 500):
|
|
with session_scope() as s:
|
|
s.execute(
|
|
sa.text("UPDATE playback_sources SET dead_at = now(), dead_reason = :r WHERE id = ANY(:ids)"),
|
|
{"r": reason, "ids": dead[i:i + 500]},
|
|
)
|
|
s.commit()
|
|
return c
|
|
|
|
|
|
def main() -> int:
|
|
logging.basicConfig(level=logging.INFO)
|
|
p = argparse.ArgumentParser()
|
|
p.add_argument("--origin", required=True)
|
|
p.add_argument("--limit", type=int, default=100)
|
|
p.add_argument("--apply", action="store_true")
|
|
p.add_argument("--rps", type=float, default=3.0)
|
|
p.add_argument("--recheck-after", type=float, default=60.0)
|
|
args = p.parse_args()
|
|
run(args.origin, args.limit, args.apply, args.rps, args.recheck_after)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|