From 2dbb9ba65f45fc45414af6d818fecc9871b2e25f Mon Sep 17 00:00:00 2001 From: goon-foss Date: Thu, 15 Oct 2026 14:38:05 +0200 Subject: [PATCH] fix(kvs): strona 404/410 to usuniety film, nie chwilowy blad resolve_kvs zwracal None na 404, API dawalo 503 bez konca i zrodlo nigdy nie znikalo; ok. polowa yespornvip to usuniete strony. Teraz TubePageError, wiec resolve oznacza je martwe. Skrypt reap_kvs_404 czysci zaleglosci (dwa 404 w odstepie i brak odtwarzacza). Co-Authored-By: Claude Opus 5.5 --- app/extractors/tubes/_kvs.py | 7 ++- scripts/reap_kvs_404.py | 98 ++++++++++++++++++++++++++++++++++++ 2 files changed, 104 insertions(+), 1 deletion(-) create mode 100644 scripts/reap_kvs_404.py diff --git a/app/extractors/tubes/_kvs.py b/app/extractors/tubes/_kvs.py index 89cc44c..b87a805 100644 --- a/app/extractors/tubes/_kvs.py +++ b/app/extractors/tubes/_kvs.py @@ -24,7 +24,7 @@ import time import urllib.parse as _up from app.extractors._fetch import _DEFAULT_IMPERSONATE, _DEFAULT_UA, _HAS_CURL_CFFI -from app.extractors._models import StreamSource +from app.extractors._models import StreamSource, TubePageError log = logging.getLogger(__name__) @@ -134,6 +134,11 @@ def resolve_kvs(page_url: str, *, base_url: str, timeout: float = 60.0) -> list[ except Exception as e: log.info("kvs: page fetch failed %s: %s", page_url, e) return None + if resp.status_code in (404, 410): + # Usuniety film: TubePageError -> API oznacza zrodlo martwe (410). Wczesniej + # zwracalismy None = "chwilowy blad" (503) bez konca: ok. polowa z 39 tys. zrodel + # yespornvip to strony 404, pokazywane jako zywe (zgloszenie 5e81eb76, 2026-10-15). + raise TubePageError(resp.status_code, page_url) if not html: log.info("kvs: empty page %s", page_url) return None diff --git a/scripts/reap_kvs_404.py b/scripts/reap_kvs_404.py new file mode 100644 index 0000000..a3b7321 --- /dev/null +++ b/scripts/reap_kvs_404.py @@ -0,0 +1,98 @@ +"""Oznacza martwe zrodla KVS (yespornvip i inne) ze strona sceny 404/410 (2026-10-15). + +resolve_kvs do 15.10 zamienial 404 na "chwilowy blad", wiec usuniete filmy wisialy jako +zywe (yespornvip: ok. polowa z 39 tys. zrodel, zgloszenie 5e81eb76). Od 15.10 resolve sam +oznacza 404 przy kliknieciu, ten skrypt czysci zaleglosci bez czekania na klikniecia. + +Dwie kontrole (patrz notatka o przemiale zrodel): strona daje 404/410 DWA razy, w odstepie +co najmniej --recheck-after sekund, i w tresci nie ma odtwarzacza (license_code). Przekierowanie +na nowy adres z odtwarzaczem to zywe zrodlo i zostaje. + + python scripts/reap_kvs_404.py --origin tube:yespornvip --limit 100 # podglad + python scripts/reap_kvs_404.py --origin tube:yespornvip --limit 50000 --apply +""" +from __future__ import annotations + +import argparse +import logging +import time +from datetime import UTC, datetime + +import sqlalchemy as sa + +from app.db import session_scope +from app.extractors._fetch import browser_get + +log = logging.getLogger(__name__) + + +def _status(url: str) -> tuple[int, bool]: + try: + r = browser_get(url, timeout=20.0, follow_redirects=True) + except Exception: + return 0, False + return r.status_code, "license_code" in (r.text or "") + + +def run(origin: str, limit: int, apply: bool, rps: float, recheck_after: float) -> dict[str, int]: + c = {"checked": 0, "alive": 0, "first_404": 0, "dead": 0, "other": 0} + with session_scope() as s: + rows = s.execute( + sa.text( + "SELECT id, page_url FROM playback_sources WHERE origin = :o AND dead_at IS NULL " + "ORDER BY id LIMIT :lim" + ), + {"o": origin, "lim": limit}, + ).all() + suspects: list[tuple] = [] + for pb_id, url in rows: + c["checked"] += 1 + st, player = _status(url) + if st in (404, 410) and not player: + c["first_404"] += 1 + suspects.append((pb_id, url, time.time())) + elif st and st < 400: + c["alive"] += 1 + else: + c["other"] += 1 + if c["checked"] % 500 == 0: + log.info("kvs reap pass1 %s", c) + time.sleep(1.0 / rps) + dead: list = [] + for pb_id, url, t0 in suspects: + wait = recheck_after - (time.time() - t0) + if wait > 0: + time.sleep(wait) + st, player = _status(url) + if st in (404, 410) and not player: + dead.append(pb_id) + time.sleep(1.0 / rps) + c["dead"] = len(dead) + print(c) + if apply and dead: + reason = f"tube page 404 (reap_kvs_404 {datetime.now(UTC):%Y-%m-%d})" + for i in range(0, len(dead), 500): + with session_scope() as s: + s.execute( + sa.text("UPDATE playback_sources SET dead_at = now(), dead_reason = :r WHERE id = ANY(:ids)"), + {"r": reason, "ids": dead[i:i + 500]}, + ) + s.commit() + return c + + +def main() -> int: + logging.basicConfig(level=logging.INFO) + p = argparse.ArgumentParser() + p.add_argument("--origin", required=True) + p.add_argument("--limit", type=int, default=100) + p.add_argument("--apply", action="store_true") + p.add_argument("--rps", type=float, default=3.0) + p.add_argument("--recheck-after", type=float, default=60.0) + args = p.parse_args() + run(args.origin, args.limit, args.apply, args.rps, args.recheck_after) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())