From 0c5148b6e802192173e1407dfb4473cf534b73f9 Mon Sep 17 00:00:00 2001 From: goon-foss Date: Sun, 26 Jul 2026 16:41:47 +0200 Subject: [PATCH] feat(ingest): browse-latest scraper dla hqporner MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit hqporner byl TYLKO w ALL_DIRECT_SCRAPERS (search po nazwisku performera), wiec swieze sceny wchodzily wylacznie gdy worker trafil na pasujacego performera → 0 nowych scen w 7 dni (audit 2026-07-26, user: 'nie widze nowych scen z hqporner'), mimo ze tube publikuje codziennie. HQPornerBrowseScraper: listing / + ?p=N (~50 scen/strone), detail /hdporn/-. Metadane z detail page: tytul z (Title Case; <h1> jest lowercase), dlugosc z meta description ('Video duration is 46min 58sec'), performerki z /actress/ (display name, nie slug), tagi z /category/, miniaturka fastporndelivery + phash (z Refererem, bo CDN bez niego 403). external_id identyczny jak w HQPornerScraper.search (hqpornercom:<url>) → obie sciezki trafiaja w te sama scene zamiast robic duplikat. Tagi jakosciowe (1080p-porn, 4k-porn) odfiltrowane, bo to atrybut pliku a nie tag tresci. Zweryfikowane na zywo: 50 scen z page 1, dur 2818s/1138s/1805s, performerzy+tagi+phash. --- app/connectors/direct_scrapers/__init__.py | 7 + .../direct_scrapers/hqporner_browse.py | 154 ++++++++++++++++++ 2 files changed, 161 insertions(+) create mode 100644 app/connectors/direct_scrapers/hqporner_browse.py diff --git a/app/connectors/direct_scrapers/__init__.py b/app/connectors/direct_scrapers/__init__.py index e6b8c48..807e12b 100644 --- a/app/connectors/direct_scrapers/__init__.py +++ b/app/connectors/direct_scrapers/__init__.py @@ -135,6 +135,7 @@ from app.connectors.direct_scrapers.yesporn import YesPornVipScraper # noqa: E4 from app.connectors.direct_scrapers.fullmovies import FullmoviesScraper # noqa: E402 from app.connectors.direct_scrapers.hdporngg import HDPornGGScraper # noqa: E402 from app.connectors.direct_scrapers.hqfap import HQFapScraper # noqa: E402 +from app.connectors.direct_scrapers.hqporner_browse import HQPornerBrowseScraper # noqa: E402 from app.connectors.direct_scrapers.javflix import JavflixScraper # noqa: E402 from app.connectors.direct_scrapers.vjav import VjavScraper # noqa: E402 from app.connectors.direct_scrapers.supjav import SupjavScraper # noqa: E402 @@ -220,6 +221,12 @@ ALL_BROWSE_SCRAPERS: list[type[BaseBrowseScraper]] = [ # biblioteka CDN serwowała `/upload/videos/video_down.mp4` stub; PRZYWRÓCONY 2026-07-07 # bo strona wróciła na CDN vstor.top z realnymi plikami (portable cross-IP), user request. HQFapScraper, + # HQPornerBrowseScraper — dołączony 2026-07-26. hqporner był TYLKO w + # ALL_DIRECT_SCRAPERS (search po performerze), więc świeże sceny wchodziły wyłącznie + # przy trafieniu na znanego performera → 0 nowych scen w 7 dni (user-report). Browse + # (listing / + ?p=N) daje ingest niezależny od stanu performerów. external_id ten sam + # co w search (`hqpornercom:<url>`), więc obie ścieżki trafiają w tę samą scenę. + HQPornerBrowseScraper, # FourK69Scraper — USUNIĘTY 2026-06-25 (ten sam stub), NIE sprawdzany ponownie. # JavflixScraper — JAV vertical (osobna sekcja). origin tube:javflix jest w # JAV_ORIGINS → list_scenes wyklucza je z głównego feedu (tylko zakładka JAV). diff --git a/app/connectors/direct_scrapers/hqporner_browse.py b/app/connectors/direct_scrapers/hqporner_browse.py new file mode 100644 index 0000000..5324217 --- /dev/null +++ b/app/connectors/direct_scrapers/hqporner_browse.py @@ -0,0 +1,154 @@ +"""HQPornerBrowseScraper — latest-vids browse dla hqporner.com. + +Dlaczego: hqporner był TYLKO w `ALL_DIRECT_SCRAPERS` (search po nazwisku performera), +więc nowe sceny wchodziły wyłącznie wtedy, gdy worker akurat szukał pasującego +performera. Efekt: 0 świeżych scen w 7 dni (audit 2026-07-26, user: "nie widzę nowych +scen z hqporner"), mimo że tube publikuje codziennie. Browse dokłada ścieżkę +niezależną od stanu performerów. + +Listing: `https://hqporner.com/` (page 1 = newest), kolejne `?p=N`, ~46-50 scen/stronę. +Detail (`/hdporn/<id>-<slug>.html`) ma komplet metadanych: + - tytuł → `<title>` (Title Case; `<h1>` jest lowercase, więc gorszy), + - długość → meta description "Video duration is 46min 58sec", + - performerki → `<a href="/actress/<slug>">Display Name</a>` (display, nie slug), + - tagi → `/category/<slug>`, + - miniaturka → `fastporndelivery.hqporner.com/imgs/.../_main.jpg`. + +`external_id` MUSI być identyczny jak w `HQPornerScraper` (search): `hqpornercom:<url>`. +Inaczej ta sama scena z obu ścieżek zrobiłaby się dwoma wpisami. + +Miniaturki fastporndelivery wymagają Referera (bez niego 403) — backend i tak owija je +w `/proxy/img/`, a phash liczymy z jawnym refererem. +""" +from __future__ import annotations + +import html as _html +import logging +import re + +from app.connectors.base import ( + RawFingerprint, + RawPerformer, + RawPlaybackSource, + RawScene, + RawTag, +) +from app.connectors.direct_scrapers._browse_base import ( + BaseBrowseScraper, + compute_thumbnail_phash, +) +from app.normalize.text import slugify + +log = logging.getLogger(__name__) + +_BASE = "https://hqporner.com" + +_SCENE_HREF_RE = re.compile(r'/hdporn/(\d+)-([^"\.]+)\.html') +_TITLE_RE = re.compile(r"<title>(.*?)", re.IGNORECASE | re.DOTALL) +# meta description: "... Video duration is 46min 58sec. Tags related ..." +_DUR_RE = re.compile(r"duration is\s*(?:(\d+)\s*min)?\s*(?:(\d+)\s*sec)?", re.IGNORECASE) +_ACTRESS_RE = re.compile( + r']+href=[\'"]/actress/([a-z0-9\-_]+)[\'"][^>]*>(.*?)', + re.IGNORECASE | re.DOTALL, +) +_CATEGORY_RE = re.compile(r'href=[\'"]/category/([a-z0-9\-_]+)[\'"]', re.IGNORECASE) +_THUMB_RE = re.compile(r"fastporndelivery\.hqporner\.com/imgs/[^\"'\s]+_main\.jpg", re.IGNORECASE) +_TAG_STRIP = re.compile(r"<[^>]+>") + +# hqporner wrzuca do /category/ także jakość ("1080p-porn", "4k-porn"). To nie jest tag +# treści, tylko atrybut pliku, i zaśmieca chmurę tagów (user: "śmieciowe tagi"). +_QUALITY_TAG_RE = re.compile(r"^(?:\d{3,4}p|4k|8k|hd|uhd|full-?hd)(?:-porn)?$", re.IGNORECASE) + + +def _clean(text: str) -> str: + return _html.unescape(_TAG_STRIP.sub("", text)).strip() + + +class HQPornerBrowseScraper(BaseBrowseScraper): + sitetag = "hqpornercom" + + def _listing_url(self, page: int) -> str: + return _BASE + "/" if page <= 1 else f"{_BASE}/?p={page}" + + def _extract_scene_urls(self, listing_html: str) -> list[str]: + seen: set[str] = set() + out: list[str] = [] + for m in _SCENE_HREF_RE.finditer(listing_html): + url = f"{_BASE}/hdporn/{m.group(1)}-{m.group(2)}.html" + if url in seen: + continue + seen.add(url) + out.append(url) + return out + + def _parse_detail(self, scene_url: str, detail_html: str) -> RawScene | None: + tm = _TITLE_RE.search(detail_html) + title = _clean(tm.group(1)) if tm else "" + # " - HQporner.com" → utnij sufiks strony. + title = re.sub(r"\s*-\s*HQporner\.com\s*$", "", title, flags=re.IGNORECASE).strip() + if not title: + return None + + duration_sec: int | None = None + dm = _DUR_RE.search(detail_html) + if dm and (dm.group(1) or dm.group(2)): + duration_sec = int(dm.group(1) or 0) * 60 + int(dm.group(2) or 0) + duration_sec = duration_sec or None + + performers: list[RawPerformer] = [] + seen_perf: set[str] = set() + for slug, disp in _ACTRESS_RE.findall(detail_html): + name = _clean(disp) or slug.replace("-", " ").title() + key = slugify(name) + if not key or key in seen_perf: + continue + seen_perf.add(key) + performers.append( + RawPerformer(external_id=f"{self.sitetag}:performer:{key}", name=name) + ) + + tags: list[RawTag] = [] + seen_tag: set[str] = set() + for slug in _CATEGORY_RE.findall(detail_html): + slug = slug.lower() + if slug in seen_tag or _QUALITY_TAG_RE.match(slug): + continue + seen_tag.add(slug) + tags.append( + RawTag( + external_id=f"{self.sitetag}:tag:{slug}", + name=slug.replace("-", " ").title(), + slug=slug, + ) + ) + + thumbnail_url: str | None = None + thm = _THUMB_RE.search(detail_html) + if thm: + thumbnail_url = "https://" + thm.group(0) + + fingerprints: list[RawFingerprint] = [] + if thumbnail_url: + ph = compute_thumbnail_phash(thumbnail_url, referer=_BASE + "/") + if ph: + fingerprints.append(RawFingerprint(kind="phash", value=ph)) + + return RawScene( + # Ten sam format co HQPornerScraper.search — inaczej duplikat sceny. + external_id=f"{self.sitetag}:{scene_url}", + title=title, + duration_sec=duration_sec, + url=scene_url, + performers=performers, + tags=tags, + fingerprints=fingerprints, + playback_sources=[ + RawPlaybackSource( + origin=f"tube:{self.sitetag}", + page_url=scene_url, + duration_sec=duration_sec, + thumbnail_url=thumbnail_url, + ) + ], + raw={"source": "direct_scraper:hqporner_browse", "url": scene_url}, + )