b64ddfd742
Strona filmu hqporner nie ma wlasnej okladki, a pierwszy _main.jpg na niej to kafelek polecanego filmu. Scraper go bral, wiec ~90% zrodel dostalo cudze miniatury (62 tys. zrodel, 5,4 tys. roznych miniatur). Okladka jest tylko w kafelku listingu (cover_<id>): browse zbiera je z listingu, wyszukiwarka z wynikow wyszukiwania. scripts/hqporner_fix_thumbs.py poprawia zapisane miniatury przechodzac listing. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
104 lines
3.9 KiB
Python
104 lines
3.9 KiB
Python
"""Naprawa miniatur hqporner: okladki z kafelkow listingu zamiast cudzych z detali.
|
|
|
|
Scraper bral pierwszy `_main.jpg` ze strony filmu, a to kafelek polecanego filmu
|
|
(app/connectors/direct_scrapers/hqporner_browse.py, _COVER_RE). Skrypt przechodzi
|
|
listing `https://hqporner.com/?p=N`, zbiera {id filmu: okladka} i poprawia
|
|
playback_sources.thumbnail_url zrodel tube:hqpornercom.
|
|
|
|
Uzycie: python3 -u scripts/hqporner_fix_thumbs.py [--commit] [--max-pages N] [--start N]
|
|
Bez --commit tylko liczy. Postep co 50 stron; wznawianie przez --start.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import re
|
|
import time
|
|
|
|
from sqlalchemy import text
|
|
|
|
from app.connectors.direct_scrapers.hqporner_browse import listing_covers
|
|
from app.db import session_scope
|
|
from app.extractors import browser_get
|
|
|
|
_BASE = "https://hqporner.com"
|
|
_ID_RE = re.compile(r"/hdporn/(\d+)-")
|
|
|
|
|
|
def _load_sources() -> dict[str, list[tuple[str, str | None]]]:
|
|
"""{id filmu: [(id zrodla, obecna miniatura)]} dla zrodel hqporner."""
|
|
out: dict[str, list[tuple[str, str | None]]] = {}
|
|
with session_scope() as s:
|
|
for sid, page, thumb in s.execute(
|
|
text("SELECT id, page_url, thumbnail_url FROM playback_sources WHERE origin = 'tube:hqpornercom'")
|
|
):
|
|
m = _ID_RE.search(page or "")
|
|
if m:
|
|
out.setdefault(m.group(1), []).append((str(sid), thumb))
|
|
return out
|
|
|
|
|
|
def _apply(fixes: list[tuple[str, str]], commit: bool) -> None:
|
|
if not fixes or not commit:
|
|
return
|
|
with session_scope() as s:
|
|
s.execute(
|
|
text("UPDATE playback_sources SET thumbnail_url = :t, updated_at = now() WHERE id = CAST(:i AS uuid)"),
|
|
[{"i": i, "t": t} for i, t in fixes],
|
|
)
|
|
|
|
|
|
def main() -> None:
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--commit", action="store_true")
|
|
ap.add_argument("--max-pages", type=int, default=5000)
|
|
ap.add_argument("--start", type=int, default=1)
|
|
ap.add_argument("--pause", type=float, default=0.3)
|
|
args = ap.parse_args()
|
|
|
|
sources = _load_sources()
|
|
print(f"zrodel hqporner: {sum(len(v) for v in sources.values())}, filmow: {len(sources)}", flush=True)
|
|
seen_ids: set[str] = set()
|
|
fixed = same = empty_streak = 0
|
|
pending: list[tuple[str, str]] = []
|
|
t0 = time.time()
|
|
for page in range(args.start, args.start + args.max_pages):
|
|
url = _BASE + "/" if page <= 1 else f"{_BASE}/?p={page}"
|
|
covers: dict[str, str] = {}
|
|
for attempt in range(3):
|
|
try:
|
|
covers = listing_covers(browser_get(url, timeout=40).text)
|
|
break
|
|
except Exception as e: # noqa: BLE001
|
|
print(f" strona {page} proba {attempt + 1}: {e}", flush=True)
|
|
time.sleep(5)
|
|
new = {k: v for k, v in covers.items() if k not in seen_ids}
|
|
if not new:
|
|
empty_streak += 1
|
|
if empty_streak >= 3:
|
|
print(f"koniec listingu na stronie {page}", flush=True)
|
|
break
|
|
else:
|
|
empty_streak = 0
|
|
seen_ids.update(new)
|
|
for vid, cover in new.items():
|
|
for sid, thumb in sources.get(vid, []):
|
|
if thumb == cover:
|
|
same += 1
|
|
else:
|
|
pending.append((sid, cover))
|
|
fixed += 1
|
|
if page % 50 == 0:
|
|
_apply(pending, args.commit)
|
|
pending.clear()
|
|
print(f"strona {page}: filmow w listingu {len(seen_ids)}, poprawionych {fixed}, "
|
|
f"juz dobrych {same}, {time.time() - t0:.0f} s", flush=True)
|
|
time.sleep(args.pause)
|
|
_apply(pending, args.commit)
|
|
ours = set(sources)
|
|
print(f"KONIEC: filmow w listingu {len(seen_ids)}, naszych filmow {len(ours)}, "
|
|
f"znalezionych {len(ours & seen_ids)}, poprawionych zrodel {fixed}, juz dobrych {same}, "
|
|
f"{'ZAPISANE' if args.commit else 'bez zapisu'}, {time.time() - t0:.0f} s", flush=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|