feat: alarm na serwis bez udanych odtworzen, miniatury latestpornvideo z plakatu lulu
Watchdog odtwarzania: origin z >= 300 zywymi zrodlami i zerem udanych odtworzen w 7 dni (hentaimama i xanime padly tak, ze nic nie alarmowalo). latestpornvideo: link podgladu pixhost zamieniany na obrazek, usuniety obrazek i brak miniatury -> plakat odtwarzacza lulu; skrypt dla starych. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
goon-foss
parent
5eb72aeb7f
commit
97bca096e6
3 changed files with 198 additions and 2 deletions
|
|
@ -26,6 +26,8 @@ import logging
|
|||
import re
|
||||
from datetime import date
|
||||
|
||||
import httpx
|
||||
|
||||
from app.connectors.base import (
|
||||
RawPlaybackSource,
|
||||
RawScene,
|
||||
|
|
@ -58,6 +60,51 @@ _TITLE_DATE_RE = re.compile(r"^(.*?)\s*\b(\d{2})\s+(\d{2})\s+(\d{2})\b")
|
|||
_CAT_SKIP = {"latest-porn-videos", "uncategorized", "", "mixed-porn", "porn-movie"}
|
||||
|
||||
|
||||
_PIXHOST_SHOW_RE = re.compile(r"^https?://(?:www\.)?pixhost\.to/show/\d+/[^/?#]+$", re.IGNORECASE)
|
||||
_OG_IMAGE_RE = re.compile(r"""property=["']og:image["'][^>]*content=["']([^"']+)""", re.IGNORECASE)
|
||||
_LULU_EMBED_RE = re.compile(r"https?://(?:www\.)?lulu[a-z0-9]*\.[a-z]+/(?:e|d|v)/([a-z0-9]+)", re.IGNORECASE)
|
||||
|
||||
|
||||
def _pixhost_removed(url: str) -> bool:
|
||||
"""pixhost przekierowuje usuniety obrazek na baner /image/removed.png."""
|
||||
try:
|
||||
r = httpx.head(url, headers={"User-Agent": "Mozilla/5.0"}, timeout=10.0, follow_redirects=False)
|
||||
except Exception:
|
||||
return False
|
||||
return r.status_code in (301, 302, 303, 307, 308) and "removed" in (r.headers.get("location") or "")
|
||||
|
||||
|
||||
def _lulu_poster(page_url: str, timeout: float) -> str | None:
|
||||
"""Plakat z odtwarzacza lulu osadzonego we wpisie (img.lulucdn.com/<id>.jpg)."""
|
||||
try:
|
||||
page = browser_get(page_url, timeout=timeout).text
|
||||
except Exception:
|
||||
return None
|
||||
m = _LULU_EMBED_RE.search(page)
|
||||
return f"https://img.lulucdn.com/{m.group(1)}.jpg" if m else None
|
||||
|
||||
|
||||
def better_thumb(thumb: str | None, page_url: str, *, timeout: float = 30.0) -> str | None:
|
||||
"""Miniatura, ktora da sie pokazac (zgloszenia eb20f1c0/4d8d5daf, 2026-10-12).
|
||||
|
||||
- `pixhost.to/show/...` to strona podgladu (HTML), nie obrazek: bierzemy jej og:image.
|
||||
- Usuniety obrazek pixhost i brak miniatury: plakat z odtwarzacza lulu z wpisu.
|
||||
Dodatkowe zapytania tylko w tych przypadkach (ok. 10% wpisow).
|
||||
"""
|
||||
if thumb and _PIXHOST_SHOW_RE.match(thumb):
|
||||
try:
|
||||
show = httpx.get(thumb, headers={"User-Agent": "Mozilla/5.0"}, timeout=10.0).text
|
||||
m = _OG_IMAGE_RE.search(show)
|
||||
thumb = html.unescape(m.group(1)) if m else None
|
||||
except Exception:
|
||||
thumb = None
|
||||
if thumb and "pixhost." in thumb and _pixhost_removed(thumb):
|
||||
thumb = None
|
||||
if not thumb:
|
||||
return _lulu_poster(page_url, timeout)
|
||||
return thumb
|
||||
|
||||
|
||||
def _name_from_slug(slug: str) -> str:
|
||||
return " ".join(w.capitalize() for w in slug.split("-") if w)
|
||||
|
||||
|
|
@ -155,6 +202,9 @@ class LatestPornVideoScraper(BaseBrowseScraper):
|
|||
seen_tag.add(sl)
|
||||
tags.append(RawTag(external_id=f"{self.sitetag}:tag:{sl}", name=_name_from_slug(sl), slug=sl))
|
||||
|
||||
# Po odczycie studia i daty z nazwy pliku, bo zapas (plakat lulu) tej nazwy nie ma.
|
||||
thumb = better_thumb(thumb, scene_url, timeout=self._timeout)
|
||||
|
||||
fingerprints = []
|
||||
if thumb:
|
||||
ph = compute_thumbnail_phash(thumb, referer=_BASE + "/")
|
||||
|
|
|
|||
|
|
@ -16,7 +16,8 @@ bez spamu co 6 h):
|
|||
- **parser** (playback_sources + scenes z 24 h wobec 14 dni wczesniej): skok lub zanik
|
||||
performerow na scene, udzial scen bez dlugosci, bez miniatury. Lapie zmiane motywu
|
||||
strony w pierwszej dobie, bez czekania na zgloszenie z apki.
|
||||
- **playback** (playback_events 24 h wobec 7 dni): spadek udanych odtworzen per origin.
|
||||
- **playback** (playback_events 24 h wobec 7 dni): spadek udanych odtworzen per origin,
|
||||
plus serwis z >= 300 zywymi zrodlami i zerem udanych odtworzen w 7 dni (cicha awaria).
|
||||
|
||||
`to_sentry=False` daje sam odczyt (reczne sprawdzenie bez zasmiecania issue).
|
||||
"""
|
||||
|
|
@ -43,6 +44,7 @@ _PARSER_MIN_BASE = 30
|
|||
# playback
|
||||
_PLAYBACK_MIN_ATTEMPTS = 30
|
||||
_PLAYBACK_MIN_BASE_ATTEMPTS = 50
|
||||
_SILENT_MIN_LIVE = 300
|
||||
|
||||
|
||||
def _cadence_gap_days(session, origin: str, lookback_days: int = 60) -> float | None:
|
||||
|
|
@ -214,6 +216,56 @@ def check_playback_health(session) -> list[dict[str, Any]]:
|
|||
return out
|
||||
|
||||
|
||||
def check_silent_origins(session) -> list[dict[str, Any]]:
|
||||
"""Serwis z wieloma zywymi zrodlami i ZERO udanych odtworzen w 7 dni.
|
||||
|
||||
check_playback_health liczy tylko originy ze zdarzeniami (min. 30 w 24 h). hentaimama
|
||||
i xanimeporn padly tak, ze apka nie dochodzila do odtwarzacza (503 z resolve, pobieranie
|
||||
pliku w WebView), wiec przez miesiac nie mialy ani jednego wiersza i nic nie alarmowalo
|
||||
(2026-10-15). Od tego dnia resolve zapisuje tez bledy serwera, ale serwis, ktorego
|
||||
nikt nie odtwarza z innego powodu, i tak bylby niewidoczny. Prog 300 zrodel: w dniu
|
||||
wdrozenia lapalo dokladnie te dwa serwisy.
|
||||
"""
|
||||
rows = session.execute(
|
||||
text(
|
||||
"""
|
||||
WITH big AS (
|
||||
SELECT origin, count(*) AS live
|
||||
FROM playback_sources
|
||||
WHERE dead_at IS NULL
|
||||
GROUP BY origin
|
||||
HAVING count(*) >= :min_live
|
||||
), ev AS (
|
||||
SELECT origin,
|
||||
count(*) AS att,
|
||||
count(*) FILTER (WHERE status = 'success') AS ok
|
||||
FROM playback_events
|
||||
WHERE created_at > now() - interval '7 days'
|
||||
GROUP BY origin
|
||||
)
|
||||
SELECT big.origin, big.live, coalesce(ev.att, 0) AS att
|
||||
FROM big LEFT JOIN ev ON ev.origin = big.origin
|
||||
WHERE coalesce(ev.ok, 0) = 0
|
||||
"""
|
||||
),
|
||||
{"min_live": _SILENT_MIN_LIVE},
|
||||
).all()
|
||||
return [
|
||||
{
|
||||
"origin": r.origin,
|
||||
"bucket": "zero7d",
|
||||
"live": int(r.live),
|
||||
"att7d": int(r.att),
|
||||
"level": "error",
|
||||
"message": (
|
||||
f"playback-watchdog: {r.origin} 0 udanych odtworzen w 7 dni "
|
||||
f"({int(r.att)} prob, {int(r.live)} zywych zrodel)"
|
||||
),
|
||||
}
|
||||
for r in rows
|
||||
]
|
||||
|
||||
|
||||
def _emit(items: list[dict[str, Any]], *, kind: str, key: str, sub: str) -> None:
|
||||
try:
|
||||
import sentry_sdk
|
||||
|
|
@ -238,7 +290,7 @@ def run_quality_watchdogs(*, to_sentry: bool = True) -> dict[str, Any]:
|
|||
with session_scope() as s:
|
||||
listing = check_listing_health(s)
|
||||
parser = check_parser_quality(s)
|
||||
playback = check_playback_health(s)
|
||||
playback = check_playback_health(s) + check_silent_origins(s)
|
||||
for group, name in ((listing, "listing"), (parser, "parser"), (playback, "playback")):
|
||||
for x in group:
|
||||
log.warning("quality-watchdog [%s]: %s", name, x["message"])
|
||||
|
|
|
|||
94
scripts/fix_latestpornvideo_thumbs.py
Normal file
94
scripts/fix_latestpornvideo_thumbs.py
Normal file
|
|
@ -0,0 +1,94 @@
|
|||
"""latestpornvideo: odzyskanie miniatur, ktore nie dzialaja (2026-10-15).
|
||||
|
||||
Scraper od dzis sam poprawia nowe wpisy (`better_thumb`), tu poprawiamy istniejace:
|
||||
- brak miniatury,
|
||||
- `pixhost.to/show/...` (strona podgladu zamiast obrazka),
|
||||
- obrazek pixhost usuniety (przekierowanie na /image/removed.png),
|
||||
- z `--imgds`: miniatury imgds.net (dawne ticdn.net), ktore daja 404.
|
||||
|
||||
W kazdym z tych przypadkow zapasem jest plakat odtwarzacza lulu z wpisu.
|
||||
|
||||
python scripts/fix_latestpornvideo_thumbs.py # podglad 20 wierszy
|
||||
python scripts/fix_latestpornvideo_thumbs.py --apply --limit 20000 [--imgds]
|
||||
|
||||
Dlugie, na prodzie przez `docker compose run -d --rm worker python3 /srv/scripts/...`.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import time
|
||||
|
||||
import httpx
|
||||
import sqlalchemy as sa
|
||||
|
||||
from app.connectors.direct_scrapers.latestpornvideo import better_thumb
|
||||
from app.db import session_scope
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _imgds_missing(url: str) -> bool:
|
||||
try:
|
||||
r = httpx.head(url, headers={"User-Agent": "Mozilla/5.0"}, timeout=10.0, follow_redirects=True)
|
||||
except Exception:
|
||||
return False
|
||||
return r.status_code == 404
|
||||
|
||||
|
||||
def run(apply: bool, limit: int, imgds: bool, sleep: float) -> dict[str, int]:
|
||||
c = {"checked": 0, "changed": 0, "unchanged": 0, "lost": 0}
|
||||
cond = (
|
||||
"thumbnail_url IS NULL OR thumbnail_url LIKE '%pixhost%'"
|
||||
+ (" OR thumbnail_url LIKE 'https://imgds.net/snaps/%'" if imgds else "")
|
||||
)
|
||||
with session_scope() as s:
|
||||
rows = s.execute(
|
||||
sa.text(
|
||||
"SELECT id, page_url, thumbnail_url FROM playback_sources "
|
||||
"WHERE origin='tube:latestpornvideocom' AND dead_at IS NULL "
|
||||
f"AND ({cond}) ORDER BY id LIMIT :lim"
|
||||
),
|
||||
{"lim": limit},
|
||||
).all()
|
||||
for pb_id, page_url, thumb in rows:
|
||||
c["checked"] += 1
|
||||
candidate = thumb
|
||||
if thumb and thumb.startswith("https://imgds.net/snaps/") and _imgds_missing(thumb):
|
||||
candidate = None
|
||||
new = better_thumb(candidate, page_url, timeout=30.0)
|
||||
if new == thumb:
|
||||
c["unchanged"] += 1
|
||||
elif not new:
|
||||
c["lost"] += 1
|
||||
else:
|
||||
c["changed"] += 1
|
||||
if c["changed"] <= 5:
|
||||
print(f"{page_url}: {thumb} -> {new}")
|
||||
if apply:
|
||||
with session_scope() as s:
|
||||
s.execute(
|
||||
sa.text("UPDATE playback_sources SET thumbnail_url=:u WHERE id=:id"),
|
||||
{"u": new[:2048], "id": pb_id},
|
||||
)
|
||||
s.commit()
|
||||
if c["checked"] % 200 == 0:
|
||||
log.info("lpv thumbs %s", c)
|
||||
time.sleep(sleep)
|
||||
return c
|
||||
|
||||
|
||||
def main() -> int:
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--apply", action="store_true")
|
||||
p.add_argument("--limit", type=int, default=20)
|
||||
p.add_argument("--imgds", action="store_true")
|
||||
p.add_argument("--sleep", type=float, default=0.2)
|
||||
args = p.parse_args()
|
||||
print(run(args.apply, args.limit, args.imgds, args.sleep))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Loading…
Add table
Reference in a new issue