c55596dd6f
- _search_base: hak _thumbnail_map (mapa url -> miniatura z calej strony) przed szukaniem <img> w oknie tekstu; xhamster mial 54% zrodel bez miniatury - mypornerleak search: karty maja teraz src zamiast data-src; browse (REST) bez featured media -> og:image ze strony posta - scripts/backfill_thumbs.py: uzupelnia miniatury ze strony sceny, od najnowszych Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
102 lines
3.8 KiB
Python
102 lines
3.8 KiB
Python
"""Uzupelnia brakujace miniatury scen z jednego tube'a ze strony sceny (og:image itd.).
|
|
|
|
Po co: scraper xhamster gubil miniatury polowy scen (szukal <img> w oknie tekstu, a
|
|
karty sa w JSON-ie), a mypornerleak przestal wystawiac obrazek w REST. Ingest jest
|
|
juz naprawiony (2026-09-14), ten skrypt sprzata to, co weszlo bez miniatury: wchodzi
|
|
na strone sceny jak przycisk „Refresh thumbnail" w apce i zapisuje miniature na
|
|
wszystkich zrodlach sceny bez miniatury.
|
|
|
|
Kolejnosc: od najnowszych scen, bo te sa na gorze list. Limit czasu przez --days.
|
|
|
|
docker compose run -d --rm --name goon-thumbs-xh worker python3 -u /srv/scripts/backfill_thumbs.py --origin tube:xhamstercom --days 30 --rps 3
|
|
docker compose run -d --rm --name goon-thumbs-mpl worker python3 -u /srv/scripts/backfill_thumbs.py --origin tube:mypornerleakcom --days 30 --rps 2
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import logging
|
|
import sys
|
|
import time
|
|
|
|
from sqlalchemy import text
|
|
|
|
from app.db import session_scope
|
|
from app.extractors._fetch import browser_get
|
|
from app.extractors.thumb_extract import extract_thumbnail_url
|
|
|
|
log = logging.getLogger("backfill_thumbs")
|
|
|
|
_SQL_KANDYDACI = text(
|
|
"""
|
|
SELECT DISTINCT ON (s.id) s.id AS scene_id, ps.page_url, s.created_at
|
|
FROM scenes s
|
|
JOIN playback_sources ps ON ps.scene_id = s.id AND ps.dead_at IS NULL AND ps.origin = :origin
|
|
WHERE ps.thumbnail_url IS NULL
|
|
AND s.created_at > now() - make_interval(days => :days)
|
|
AND NOT EXISTS (
|
|
SELECT 1 FROM playback_sources q
|
|
WHERE q.scene_id = s.id AND q.dead_at IS NULL AND q.thumbnail_url IS NOT NULL
|
|
)
|
|
ORDER BY s.id, ps.created_at
|
|
"""
|
|
)
|
|
|
|
|
|
def main() -> int:
|
|
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--origin", required=True, help="np. tube:xhamstercom")
|
|
ap.add_argument("--days", type=int, default=30)
|
|
ap.add_argument("--limit", type=int, default=0, help="0 = bez limitu")
|
|
ap.add_argument("--rps", type=float, default=3.0, help="zadan na sekunde")
|
|
ap.add_argument("--dry-run", action="store_true")
|
|
args = ap.parse_args()
|
|
|
|
with session_scope() as s:
|
|
rows = s.execute(_SQL_KANDYDACI, {"origin": args.origin, "days": args.days}).all()
|
|
rows.sort(key=lambda r: r.created_at, reverse=True)
|
|
if args.limit:
|
|
rows = rows[: args.limit]
|
|
print(f"kandydatow: {len(rows)} ({args.origin}, {args.days} dni){' [dry-run]' if args.dry_run else ''}")
|
|
if args.dry_run:
|
|
return 0
|
|
|
|
ok = miss = err = 0
|
|
okres = 1.0 / max(args.rps, 0.1)
|
|
t_last = 0.0
|
|
for i, row in enumerate(rows, 1):
|
|
czekaj = okres - (time.monotonic() - t_last)
|
|
if czekaj > 0:
|
|
time.sleep(czekaj)
|
|
t_last = time.monotonic()
|
|
try:
|
|
r = browser_get(row.page_url, timeout=15.0)
|
|
if r.status_code != 200:
|
|
err += 1
|
|
continue
|
|
thumb = extract_thumbnail_url(r.text)
|
|
except Exception as e: # noqa: BLE001
|
|
log.debug("fetch %s: %s", row.page_url, e)
|
|
err += 1
|
|
continue
|
|
if not thumb:
|
|
miss += 1
|
|
continue
|
|
with session_scope() as s:
|
|
s.execute(
|
|
text(
|
|
"UPDATE playback_sources SET thumbnail_url = :t, updated_at = now() "
|
|
"WHERE scene_id = :sid AND dead_at IS NULL AND thumbnail_url IS NULL"
|
|
),
|
|
{"t": thumb, "sid": row.scene_id},
|
|
)
|
|
s.commit()
|
|
ok += 1
|
|
if i % 200 == 0:
|
|
print(f"{i}/{len(rows)} ok={ok} brak={miss} blad={err}", flush=True)
|
|
print(f"KONIEC {args.origin}: ok={ok} brak={miss} blad={err} z {len(rows)}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|