"""HQPornerBrowseScraper — latest-vids browse dla hqporner.com. Dlaczego: hqporner był TYLKO w `ALL_DIRECT_SCRAPERS` (search po nazwisku performera), więc nowe sceny wchodziły wyłącznie wtedy, gdy worker akurat szukał pasującego performera. Efekt: 0 świeżych scen w 7 dni (audit 2026-07-26, user: "nie widzę nowych scen z hqporner"), mimo że tube publikuje codziennie. Browse dokłada ścieżkę niezależną od stanu performerów. Listing: `https://hqporner.com/` (page 1 = newest), kolejne `?p=N`, ~46-50 scen/stronę. Detail (`/hdporn/-.html`) ma komplet metadanych: - tytuł → `` (Title Case; `<h1>` jest lowercase, więc gorszy), - długość → meta description "Video duration is 46min 58sec", - performerki → `<a href="/actress/<slug>">Display Name</a>` (display, nie slug), - tagi → `/category/<slug>`, - miniaturka → `fastporndelivery.hqporner.com/imgs/.../_main.jpg`. `external_id` MUSI być identyczny jak w `HQPornerScraper` (search): `hqpornercom:<url>`. Inaczej ta sama scena z obu ścieżek zrobiłaby się dwoma wpisami. Miniaturki fastporndelivery wymagają Referera (bez niego 403) — backend i tak owija je w `/proxy/img/`, a phash liczymy z jawnym refererem. """ from __future__ import annotations import html as _html import logging import re from app.connectors.base import ( RawFingerprint, RawPerformer, RawPlaybackSource, RawScene, RawTag, ) from app.connectors.direct_scrapers._browse_base import ( BaseBrowseScraper, compute_thumbnail_phash, ) from app.normalize.text import slugify log = logging.getLogger(__name__) _BASE = "https://hqporner.com" _SCENE_HREF_RE = re.compile(r'/hdporn/(\d+)-([^"\.]+)\.html') _TITLE_RE = re.compile(r"<title>(.*?)", re.IGNORECASE | re.DOTALL) # meta description: "... Video duration is 46min 58sec. Tags related ..." _DUR_RE = re.compile(r"duration is\s*(?:(\d+)\s*min)?\s*(?:(\d+)\s*sec)?", re.IGNORECASE) _ACTRESS_RE = re.compile( r']+href=[\'"]/actress/([a-z0-9\-_]+)[\'"][^>]*>(.*?)', re.IGNORECASE | re.DOTALL, ) _CATEGORY_RE = re.compile(r'href=[\'"]/category/([a-z0-9\-_]+)[\'"]', re.IGNORECASE) _THUMB_RE = re.compile(r"fastporndelivery\.hqporner\.com/imgs/[^\"'\s]+_main\.jpg", re.IGNORECASE) _TAG_STRIP = re.compile(r"<[^>]+>") # hqporner wrzuca do /category/ także jakość ("1080p-porn", "4k-porn"). To nie jest tag # treści, tylko atrybut pliku, i zaśmieca chmurę tagów (user: "śmieciowe tagi"). _QUALITY_TAG_RE = re.compile(r"^(?:\d{3,4}p|4k|8k|hd|uhd|full-?hd)(?:-porn)?$", re.IGNORECASE) def _clean(text: str) -> str: return _html.unescape(_TAG_STRIP.sub("", text)).strip() class HQPornerBrowseScraper(BaseBrowseScraper): sitetag = "hqpornercom" def _listing_url(self, page: int) -> str: return _BASE + "/" if page <= 1 else f"{_BASE}/?p={page}" def _extract_scene_urls(self, listing_html: str) -> list[str]: seen: set[str] = set() out: list[str] = [] for m in _SCENE_HREF_RE.finditer(listing_html): url = f"{_BASE}/hdporn/{m.group(1)}-{m.group(2)}.html" if url in seen: continue seen.add(url) out.append(url) return out def _parse_detail(self, scene_url: str, detail_html: str) -> RawScene | None: tm = _TITLE_RE.search(detail_html) title = _clean(tm.group(1)) if tm else "" # " - HQporner.com" → utnij sufiks strony. title = re.sub(r"\s*-\s*HQporner\.com\s*$", "", title, flags=re.IGNORECASE).strip() if not title: return None duration_sec: int | None = None dm = _DUR_RE.search(detail_html) if dm and (dm.group(1) or dm.group(2)): duration_sec = int(dm.group(1) or 0) * 60 + int(dm.group(2) or 0) duration_sec = duration_sec or None performers: list[RawPerformer] = [] seen_perf: set[str] = set() for slug, disp in _ACTRESS_RE.findall(detail_html): name = _clean(disp) or slug.replace("-", " ").title() key = slugify(name) if not key or key in seen_perf: continue seen_perf.add(key) performers.append( RawPerformer(external_id=f"{self.sitetag}:performer:{key}", name=name) ) tags: list[RawTag] = [] seen_tag: set[str] = set() for slug in _CATEGORY_RE.findall(detail_html): slug = slug.lower() if slug in seen_tag or _QUALITY_TAG_RE.match(slug): continue seen_tag.add(slug) tags.append( RawTag( external_id=f"{self.sitetag}:tag:{slug}", name=slug.replace("-", " ").title(), slug=slug, ) ) thumbnail_url: str | None = None thm = _THUMB_RE.search(detail_html) if thm: thumbnail_url = "https://" + thm.group(0) fingerprints: list[RawFingerprint] = [] if thumbnail_url: ph = compute_thumbnail_phash(thumbnail_url, referer=_BASE + "/") if ph: fingerprints.append(RawFingerprint(kind="phash", value=ph)) return RawScene( # Ten sam format co HQPornerScraper.search — inaczej duplikat sceny. external_id=f"{self.sitetag}:{scene_url}", title=title, duration_sec=duration_sec, url=scene_url, performers=performers, tags=tags, fingerprints=fingerprints, playback_sources=[ RawPlaybackSource( origin=f"tube:{self.sitetag}", page_url=scene_url, duration_sec=duration_sec, thumbnail_url=thumbnail_url, ) ], raw={"source": "direct_scraper:hqporner_browse", "url": scene_url}, )