From 97bca096e6404616967de32896e1810c62327469 Mon Sep 17 00:00:00 2001 From: goon-foss Date: Thu, 15 Oct 2026 12:41:11 +0200 Subject: [PATCH] feat: alarm na serwis bez udanych odtworzen, miniatury latestpornvideo z plakatu lulu Watchdog odtwarzania: origin z >= 300 zywymi zrodlami i zerem udanych odtworzen w 7 dni (hentaimama i xanime padly tak, ze nic nie alarmowalo). latestpornvideo: link podgladu pixhost zamieniany na obrazek, usuniety obrazek i brak miniatury -> plakat odtwarzacza lulu; skrypt dla starych. Co-Authored-By: Claude Opus 5.5 --- .../direct_scrapers/latestpornvideo.py | 50 ++++++++++ app/scheduler/quality_watchdog.py | 56 ++++++++++- scripts/fix_latestpornvideo_thumbs.py | 94 +++++++++++++++++++ 3 files changed, 198 insertions(+), 2 deletions(-) create mode 100644 scripts/fix_latestpornvideo_thumbs.py diff --git a/app/connectors/direct_scrapers/latestpornvideo.py b/app/connectors/direct_scrapers/latestpornvideo.py index c208d0f..0421575 100644 --- a/app/connectors/direct_scrapers/latestpornvideo.py +++ b/app/connectors/direct_scrapers/latestpornvideo.py @@ -26,6 +26,8 @@ import logging import re from datetime import date +import httpx + from app.connectors.base import ( RawPlaybackSource, RawScene, @@ -58,6 +60,51 @@ _TITLE_DATE_RE = re.compile(r"^(.*?)\s*\b(\d{2})\s+(\d{2})\s+(\d{2})\b") _CAT_SKIP = {"latest-porn-videos", "uncategorized", "", "mixed-porn", "porn-movie"} +_PIXHOST_SHOW_RE = re.compile(r"^https?://(?:www\.)?pixhost\.to/show/\d+/[^/?#]+$", re.IGNORECASE) +_OG_IMAGE_RE = re.compile(r"""property=["']og:image["'][^>]*content=["']([^"']+)""", re.IGNORECASE) +_LULU_EMBED_RE = re.compile(r"https?://(?:www\.)?lulu[a-z0-9]*\.[a-z]+/(?:e|d|v)/([a-z0-9]+)", re.IGNORECASE) + + +def _pixhost_removed(url: str) -> bool: + """pixhost przekierowuje usuniety obrazek na baner /image/removed.png.""" + try: + r = httpx.head(url, headers={"User-Agent": "Mozilla/5.0"}, timeout=10.0, follow_redirects=False) + except Exception: + return False + return r.status_code in (301, 302, 303, 307, 308) and "removed" in (r.headers.get("location") or "") + + +def _lulu_poster(page_url: str, timeout: float) -> str | None: + """Plakat z odtwarzacza lulu osadzonego we wpisie (img.lulucdn.com/.jpg).""" + try: + page = browser_get(page_url, timeout=timeout).text + except Exception: + return None + m = _LULU_EMBED_RE.search(page) + return f"https://img.lulucdn.com/{m.group(1)}.jpg" if m else None + + +def better_thumb(thumb: str | None, page_url: str, *, timeout: float = 30.0) -> str | None: + """Miniatura, ktora da sie pokazac (zgloszenia eb20f1c0/4d8d5daf, 2026-10-12). + + - `pixhost.to/show/...` to strona podgladu (HTML), nie obrazek: bierzemy jej og:image. + - Usuniety obrazek pixhost i brak miniatury: plakat z odtwarzacza lulu z wpisu. + Dodatkowe zapytania tylko w tych przypadkach (ok. 10% wpisow). + """ + if thumb and _PIXHOST_SHOW_RE.match(thumb): + try: + show = httpx.get(thumb, headers={"User-Agent": "Mozilla/5.0"}, timeout=10.0).text + m = _OG_IMAGE_RE.search(show) + thumb = html.unescape(m.group(1)) if m else None + except Exception: + thumb = None + if thumb and "pixhost." in thumb and _pixhost_removed(thumb): + thumb = None + if not thumb: + return _lulu_poster(page_url, timeout) + return thumb + + def _name_from_slug(slug: str) -> str: return " ".join(w.capitalize() for w in slug.split("-") if w) @@ -155,6 +202,9 @@ class LatestPornVideoScraper(BaseBrowseScraper): seen_tag.add(sl) tags.append(RawTag(external_id=f"{self.sitetag}:tag:{sl}", name=_name_from_slug(sl), slug=sl)) + # Po odczycie studia i daty z nazwy pliku, bo zapas (plakat lulu) tej nazwy nie ma. + thumb = better_thumb(thumb, scene_url, timeout=self._timeout) + fingerprints = [] if thumb: ph = compute_thumbnail_phash(thumb, referer=_BASE + "/") diff --git a/app/scheduler/quality_watchdog.py b/app/scheduler/quality_watchdog.py index e913f3a..fda3d2c 100644 --- a/app/scheduler/quality_watchdog.py +++ b/app/scheduler/quality_watchdog.py @@ -16,7 +16,8 @@ bez spamu co 6 h): - **parser** (playback_sources + scenes z 24 h wobec 14 dni wczesniej): skok lub zanik performerow na scene, udzial scen bez dlugosci, bez miniatury. Lapie zmiane motywu strony w pierwszej dobie, bez czekania na zgloszenie z apki. -- **playback** (playback_events 24 h wobec 7 dni): spadek udanych odtworzen per origin. +- **playback** (playback_events 24 h wobec 7 dni): spadek udanych odtworzen per origin, + plus serwis z >= 300 zywymi zrodlami i zerem udanych odtworzen w 7 dni (cicha awaria). `to_sentry=False` daje sam odczyt (reczne sprawdzenie bez zasmiecania issue). """ @@ -43,6 +44,7 @@ _PARSER_MIN_BASE = 30 # playback _PLAYBACK_MIN_ATTEMPTS = 30 _PLAYBACK_MIN_BASE_ATTEMPTS = 50 +_SILENT_MIN_LIVE = 300 def _cadence_gap_days(session, origin: str, lookback_days: int = 60) -> float | None: @@ -214,6 +216,56 @@ def check_playback_health(session) -> list[dict[str, Any]]: return out +def check_silent_origins(session) -> list[dict[str, Any]]: + """Serwis z wieloma zywymi zrodlami i ZERO udanych odtworzen w 7 dni. + + check_playback_health liczy tylko originy ze zdarzeniami (min. 30 w 24 h). hentaimama + i xanimeporn padly tak, ze apka nie dochodzila do odtwarzacza (503 z resolve, pobieranie + pliku w WebView), wiec przez miesiac nie mialy ani jednego wiersza i nic nie alarmowalo + (2026-10-15). Od tego dnia resolve zapisuje tez bledy serwera, ale serwis, ktorego + nikt nie odtwarza z innego powodu, i tak bylby niewidoczny. Prog 300 zrodel: w dniu + wdrozenia lapalo dokladnie te dwa serwisy. + """ + rows = session.execute( + text( + """ + WITH big AS ( + SELECT origin, count(*) AS live + FROM playback_sources + WHERE dead_at IS NULL + GROUP BY origin + HAVING count(*) >= :min_live + ), ev AS ( + SELECT origin, + count(*) AS att, + count(*) FILTER (WHERE status = 'success') AS ok + FROM playback_events + WHERE created_at > now() - interval '7 days' + GROUP BY origin + ) + SELECT big.origin, big.live, coalesce(ev.att, 0) AS att + FROM big LEFT JOIN ev ON ev.origin = big.origin + WHERE coalesce(ev.ok, 0) = 0 + """ + ), + {"min_live": _SILENT_MIN_LIVE}, + ).all() + return [ + { + "origin": r.origin, + "bucket": "zero7d", + "live": int(r.live), + "att7d": int(r.att), + "level": "error", + "message": ( + f"playback-watchdog: {r.origin} 0 udanych odtworzen w 7 dni " + f"({int(r.att)} prob, {int(r.live)} zywych zrodel)" + ), + } + for r in rows + ] + + def _emit(items: list[dict[str, Any]], *, kind: str, key: str, sub: str) -> None: try: import sentry_sdk @@ -238,7 +290,7 @@ def run_quality_watchdogs(*, to_sentry: bool = True) -> dict[str, Any]: with session_scope() as s: listing = check_listing_health(s) parser = check_parser_quality(s) - playback = check_playback_health(s) + playback = check_playback_health(s) + check_silent_origins(s) for group, name in ((listing, "listing"), (parser, "parser"), (playback, "playback")): for x in group: log.warning("quality-watchdog [%s]: %s", name, x["message"]) diff --git a/scripts/fix_latestpornvideo_thumbs.py b/scripts/fix_latestpornvideo_thumbs.py new file mode 100644 index 0000000..82c0860 --- /dev/null +++ b/scripts/fix_latestpornvideo_thumbs.py @@ -0,0 +1,94 @@ +"""latestpornvideo: odzyskanie miniatur, ktore nie dzialaja (2026-10-15). + +Scraper od dzis sam poprawia nowe wpisy (`better_thumb`), tu poprawiamy istniejace: +- brak miniatury, +- `pixhost.to/show/...` (strona podgladu zamiast obrazka), +- obrazek pixhost usuniety (przekierowanie na /image/removed.png), +- z `--imgds`: miniatury imgds.net (dawne ticdn.net), ktore daja 404. + +W kazdym z tych przypadkow zapasem jest plakat odtwarzacza lulu z wpisu. + + python scripts/fix_latestpornvideo_thumbs.py # podglad 20 wierszy + python scripts/fix_latestpornvideo_thumbs.py --apply --limit 20000 [--imgds] + +Dlugie, na prodzie przez `docker compose run -d --rm worker python3 /srv/scripts/...`. +""" +from __future__ import annotations + +import argparse +import logging +import time + +import httpx +import sqlalchemy as sa + +from app.connectors.direct_scrapers.latestpornvideo import better_thumb +from app.db import session_scope + +log = logging.getLogger(__name__) + + +def _imgds_missing(url: str) -> bool: + try: + r = httpx.head(url, headers={"User-Agent": "Mozilla/5.0"}, timeout=10.0, follow_redirects=True) + except Exception: + return False + return r.status_code == 404 + + +def run(apply: bool, limit: int, imgds: bool, sleep: float) -> dict[str, int]: + c = {"checked": 0, "changed": 0, "unchanged": 0, "lost": 0} + cond = ( + "thumbnail_url IS NULL OR thumbnail_url LIKE '%pixhost%'" + + (" OR thumbnail_url LIKE 'https://imgds.net/snaps/%'" if imgds else "") + ) + with session_scope() as s: + rows = s.execute( + sa.text( + "SELECT id, page_url, thumbnail_url FROM playback_sources " + "WHERE origin='tube:latestpornvideocom' AND dead_at IS NULL " + f"AND ({cond}) ORDER BY id LIMIT :lim" + ), + {"lim": limit}, + ).all() + for pb_id, page_url, thumb in rows: + c["checked"] += 1 + candidate = thumb + if thumb and thumb.startswith("https://imgds.net/snaps/") and _imgds_missing(thumb): + candidate = None + new = better_thumb(candidate, page_url, timeout=30.0) + if new == thumb: + c["unchanged"] += 1 + elif not new: + c["lost"] += 1 + else: + c["changed"] += 1 + if c["changed"] <= 5: + print(f"{page_url}: {thumb} -> {new}") + if apply: + with session_scope() as s: + s.execute( + sa.text("UPDATE playback_sources SET thumbnail_url=:u WHERE id=:id"), + {"u": new[:2048], "id": pb_id}, + ) + s.commit() + if c["checked"] % 200 == 0: + log.info("lpv thumbs %s", c) + time.sleep(sleep) + return c + + +def main() -> int: + logging.basicConfig(level=logging.INFO) + p = argparse.ArgumentParser() + p.add_argument("--apply", action="store_true") + p.add_argument("--limit", type=int, default=20) + p.add_argument("--imgds", action="store_true") + p.add_argument("--sleep", type=float, default=0.2) + args = p.parse_args() + print(run(args.apply, args.limit, args.imgds, args.sleep)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())