fix(thumbs): xhamster miniatury z JSON strony wynikow, mypornerleak z src i og:image posta, backfill brakujacych
- _search_base: hak _thumbnail_map (mapa url -> miniatura z calej strony) przed szukaniem <img> w oknie tekstu; xhamster mial 54% zrodel bez miniatury - mypornerleak search: karty maja teraz src zamiast data-src; browse (REST) bez featured media -> og:image ze strony posta - scripts/backfill_thumbs.py: uzupelnia miniatury ze strony sceny, od najnowszych Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
goon-foss
parent
320d928927
commit
c55596dd6f
5 changed files with 144 additions and 3 deletions
|
|
@ -107,6 +107,15 @@ class BaseSearchScraper(BaseDirectTubeScraper):
|
|||
"""
|
||||
return urllib.parse.quote_plus(query.strip())
|
||||
|
||||
def _thumbnail_map(self, listing_html: str) -> dict[str, str]:
|
||||
"""Opcjonalny hak: mapa scene_url -> thumbnail_url z calej strony wynikow.
|
||||
|
||||
Domyslnie pusta i wtedy dziala szukanie `<img>` w oknie wokol linku. Tuby,
|
||||
ktore renderuja karty z JSON-a (xhamster: `window.initials` z `pageURL` i
|
||||
`thumbURL`), maja miniature dla KAZDEJ karty w jednym miejscu, a okno tekstu
|
||||
gubilo polowe (54% zrodel xhamster bez miniatury, zgloszenie 8ec9dc96)."""
|
||||
return {}
|
||||
|
||||
def _fetch_scene_metadata(
|
||||
self, scene_url: str
|
||||
) -> tuple[RawStudio | None, list[RawPerformer], list[RawTag]] | None:
|
||||
|
|
@ -169,6 +178,11 @@ class BaseSearchScraper(BaseDirectTubeScraper):
|
|||
}
|
||||
|
||||
seen: set[str] = set()
|
||||
try:
|
||||
thumb_map = self._thumbnail_map(r.text)
|
||||
except Exception as e: # noqa: BLE001
|
||||
log.debug("%s thumbnail map failed: %s", self.sitetag, e)
|
||||
thumb_map = {}
|
||||
yielded = 0
|
||||
for m in self._scene_url_re.finditer(r.text):
|
||||
scene_url = self._scene_url_from_match(m).strip()
|
||||
|
|
@ -213,8 +227,8 @@ class BaseSearchScraper(BaseDirectTubeScraper):
|
|||
window_start = max(0, m.start() - self._thumbnail_window)
|
||||
window_end = min(len(r.text), m.end() + self._thumbnail_window)
|
||||
window_html = r.text[window_start:window_end]
|
||||
thumb_url: str | None = None
|
||||
img_m = _IMG_SRC_RE.search(window_html)
|
||||
thumb_url: str | None = thumb_map.get(scene_url)
|
||||
img_m = None if thumb_url else _IMG_SRC_RE.search(window_html)
|
||||
if img_m:
|
||||
thumb_url = img_m.group(1).strip()
|
||||
if thumb_url.startswith("//"):
|
||||
|
|
|
|||
|
|
@ -33,7 +33,9 @@ _BASE = "https://mypornerleak.com"
|
|||
_ARTICLE_RE = re.compile(r'<article[^>]*\bclass="([^"]+)"', re.IGNORECASE)
|
||||
_LINK_RE = re.compile(r'<a\s+href="([^"]+)"\s+title="([^"]+)"', re.IGNORECASE)
|
||||
_DURATION_RE = re.compile(r'class="duration">(?:<[^>]+>)*\s*(\d{1,2}:\d{2}(?::\d{2})?)', re.IGNORECASE)
|
||||
_THUMB_RE = re.compile(r'data-src="([^"]+)"', re.IGNORECASE)
|
||||
# Do 2026-09 karty mialy lazy `data-src`, teraz zwykle `src` (class video-main-thumb);
|
||||
# stary wzorzec zostawial 59% zrodel bez miniatury (zgloszenie 94e1da85).
|
||||
_THUMB_RE = re.compile(r'<img[^>]+(?:data-src|src)="(https?://[^"]+\.(?:jpg|jpeg|png|webp)[^"]*)"', re.IGNORECASE)
|
||||
_WN_HOST_RE = re.compile(r"https?://w\d+\.mypornerleak\.com", re.IGNORECASE)
|
||||
_CLASS_ACTOR_RE = re.compile(r"\bactors-([a-z0-9-]+)")
|
||||
_CLASS_CAT_RE = re.compile(r"\bcategory-([a-z0-9-]+)")
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ from app.connectors.direct_scrapers._browse_base import (
|
|||
compute_thumbnail_phash,
|
||||
)
|
||||
from app.extractors import browser_get
|
||||
from app.extractors.thumb_extract import extract_thumbnail_url
|
||||
from app.normalize.text import slugify
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
|
@ -85,6 +86,15 @@ class MyPornerLeakBrowseScraper(BaseBrowseScraper):
|
|||
emb = p.get("_embedded") or {}
|
||||
fm = emb.get("wp:featuredmedia") or []
|
||||
thumb = (fm[0].get("source_url") if fm and isinstance(fm[0], dict) else None) or None
|
||||
if not thumb:
|
||||
# Od 2026-09 REST nie ma featured media (featured_media=0, brak img w
|
||||
# tresci); strona posta ma og:image (zgloszenie 94e1da85).
|
||||
try:
|
||||
pr = browser_get(link, timeout=15.0)
|
||||
if pr.status_code == 200:
|
||||
thumb = extract_thumbnail_url(pr.text)
|
||||
except Exception as e: # noqa: BLE001
|
||||
log.debug("mypornerleak og:image fetch failed for %s: %s", link, e)
|
||||
|
||||
studio: RawStudio | None = None
|
||||
tags: list[RawTag] = []
|
||||
|
|
|
|||
|
|
@ -7,6 +7,13 @@ from __future__ import annotations
|
|||
|
||||
import re
|
||||
|
||||
# Karty sa renderowane z JSON-a w HTML (`window.initials`): kazda ma `pageURL` i
|
||||
# `thumbURL` (526x298 webp). Szukanie `<img>` w oknie tekstu gubilo polowe scen, bo
|
||||
# ten sam link wystepuje dwa razy (miniatura i tytul), a img jest tylko przy pierwszym.
|
||||
_JSON_CARD_RE = re.compile(
|
||||
r'"pageURL":"(https:\\/\\/xhamster\.com\\/videos\\/[^"]+)","thumbURL":"([^"]+)"'
|
||||
)
|
||||
|
||||
from app.connectors.direct_scrapers._search_base import BaseSearchScraper
|
||||
|
||||
|
||||
|
|
@ -17,3 +24,9 @@ class XHamsterScraper(BaseSearchScraper):
|
|||
r'href="(?P<url>https://xhamster\.com/videos/(?P<slug>[a-z0-9_\-]+))"',
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
def _thumbnail_map(self, listing_html: str) -> dict[str, str]:
|
||||
out: dict[str, str] = {}
|
||||
for page_url, thumb in _JSON_CARD_RE.findall(listing_html):
|
||||
out[page_url.replace("\\/", "/")] = thumb.replace("\\/", "/")
|
||||
return out
|
||||
|
|
|
|||
102
scripts/backfill_thumbs.py
Normal file
102
scripts/backfill_thumbs.py
Normal file
|
|
@ -0,0 +1,102 @@
|
|||
"""Uzupelnia brakujace miniatury scen z jednego tube'a ze strony sceny (og:image itd.).
|
||||
|
||||
Po co: scraper xhamster gubil miniatury polowy scen (szukal <img> w oknie tekstu, a
|
||||
karty sa w JSON-ie), a mypornerleak przestal wystawiac obrazek w REST. Ingest jest
|
||||
juz naprawiony (2026-09-14), ten skrypt sprzata to, co weszlo bez miniatury: wchodzi
|
||||
na strone sceny jak przycisk „Refresh thumbnail" w apce i zapisuje miniature na
|
||||
wszystkich zrodlach sceny bez miniatury.
|
||||
|
||||
Kolejnosc: od najnowszych scen, bo te sa na gorze list. Limit czasu przez --days.
|
||||
|
||||
docker compose run -d --rm --name goon-thumbs-xh worker python3 -u /srv/scripts/backfill_thumbs.py --origin tube:xhamstercom --days 30 --rps 3
|
||||
docker compose run -d --rm --name goon-thumbs-mpl worker python3 -u /srv/scripts/backfill_thumbs.py --origin tube:mypornerleakcom --days 30 --rps 2
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import sys
|
||||
import time
|
||||
|
||||
from sqlalchemy import text
|
||||
|
||||
from app.db import session_scope
|
||||
from app.extractors._fetch import browser_get
|
||||
from app.extractors.thumb_extract import extract_thumbnail_url
|
||||
|
||||
log = logging.getLogger("backfill_thumbs")
|
||||
|
||||
_SQL_KANDYDACI = text(
|
||||
"""
|
||||
SELECT DISTINCT ON (s.id) s.id AS scene_id, ps.page_url, s.created_at
|
||||
FROM scenes s
|
||||
JOIN playback_sources ps ON ps.scene_id = s.id AND ps.dead_at IS NULL AND ps.origin = :origin
|
||||
WHERE ps.thumbnail_url IS NULL
|
||||
AND s.created_at > now() - make_interval(days => :days)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM playback_sources q
|
||||
WHERE q.scene_id = s.id AND q.dead_at IS NULL AND q.thumbnail_url IS NOT NULL
|
||||
)
|
||||
ORDER BY s.id, ps.created_at
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--origin", required=True, help="np. tube:xhamstercom")
|
||||
ap.add_argument("--days", type=int, default=30)
|
||||
ap.add_argument("--limit", type=int, default=0, help="0 = bez limitu")
|
||||
ap.add_argument("--rps", type=float, default=3.0, help="zadan na sekunde")
|
||||
ap.add_argument("--dry-run", action="store_true")
|
||||
args = ap.parse_args()
|
||||
|
||||
with session_scope() as s:
|
||||
rows = s.execute(_SQL_KANDYDACI, {"origin": args.origin, "days": args.days}).all()
|
||||
rows.sort(key=lambda r: r.created_at, reverse=True)
|
||||
if args.limit:
|
||||
rows = rows[: args.limit]
|
||||
print(f"kandydatow: {len(rows)} ({args.origin}, {args.days} dni){' [dry-run]' if args.dry_run else ''}")
|
||||
if args.dry_run:
|
||||
return 0
|
||||
|
||||
ok = miss = err = 0
|
||||
okres = 1.0 / max(args.rps, 0.1)
|
||||
t_last = 0.0
|
||||
for i, row in enumerate(rows, 1):
|
||||
czekaj = okres - (time.monotonic() - t_last)
|
||||
if czekaj > 0:
|
||||
time.sleep(czekaj)
|
||||
t_last = time.monotonic()
|
||||
try:
|
||||
r = browser_get(row.page_url, timeout=15.0)
|
||||
if r.status_code != 200:
|
||||
err += 1
|
||||
continue
|
||||
thumb = extract_thumbnail_url(r.text)
|
||||
except Exception as e: # noqa: BLE001
|
||||
log.debug("fetch %s: %s", row.page_url, e)
|
||||
err += 1
|
||||
continue
|
||||
if not thumb:
|
||||
miss += 1
|
||||
continue
|
||||
with session_scope() as s:
|
||||
s.execute(
|
||||
text(
|
||||
"UPDATE playback_sources SET thumbnail_url = :t, updated_at = now() "
|
||||
"WHERE scene_id = :sid AND dead_at IS NULL AND thumbnail_url IS NULL"
|
||||
),
|
||||
{"t": thumb, "sid": row.scene_id},
|
||||
)
|
||||
s.commit()
|
||||
ok += 1
|
||||
if i % 200 == 0:
|
||||
print(f"{i}/{len(rows)} ok={ok} brak={miss} blad={err}", flush=True)
|
||||
print(f"KONIEC {args.origin}: ok={ok} brak={miss} blad={err} z {len(rows)}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Loading…
Add table
Reference in a new issue