97bca096e6
Watchdog odtwarzania: origin z >= 300 zywymi zrodlami i zerem udanych odtworzen w 7 dni (hentaimama i xanime padly tak, ze nic nie alarmowalo). latestpornvideo: link podgladu pixhost zamieniany na obrazek, usuniety obrazek i brak miniatury -> plakat odtwarzacza lulu; skrypt dla starych. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
237 lines
10 KiB
Python
237 lines
10 KiB
Python
"""latestpornvideo.com, latest-vids browse scraper.
|
||
|
||
Historia: dawniej performer-driven search scraper (`/actor/<slug>/`). Problem
|
||
(user-report 1da0375e): search-scraper ingestuje TYLKO sceny performerów, których
|
||
akurat szukamy → feed strony "latest" nigdy nie wpada, w apce widać stary zestaw,
|
||
a na stronie jest świeży. Przerobione na BROWSE (latest chronologicznie z
|
||
`/page/<n>/`, page 1 = `/`), 2026-06-22.
|
||
|
||
Listing card (zero detail-fetchy, detail page nie ma performerów ani duration):
|
||
<article class="... post-<id> ... category-<cat> tag-<x> tag-<y> ...">
|
||
<a href="https://latestpornvideo.com/<id>/" title="<Tytuł>">
|
||
data-main-thumb="<Studio>-YYYY-MM-DD-...-cover.jpg"
|
||
→ tytuł, miniatura, studio+release_date (z nazwy thumba albo z tytułu
|
||
"<Studio> YY MM DD ..."). Performerzy: listing ICH NIE MA czysto
|
||
(homepage karty bez `actors-*`, jak na stronach /actor/), a `tag-*` miesza
|
||
fragmenty imion z gatunkami → NIE ufamy tagom jako performerom; performera
|
||
dorabia canonical-merge po tytule+duration. Tagi bierzemy ostrożnie.
|
||
|
||
Playback: luluvid (filemoon family) iframe → extractor `latestpornvideocom`
|
||
(_embed_iframe → type='hoster'), telefon resolwuje phone-side. page_url = /<id>/.
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import html
|
||
import logging
|
||
import re
|
||
from datetime import date
|
||
|
||
import httpx
|
||
|
||
from app.connectors.base import (
|
||
RawPlaybackSource,
|
||
RawScene,
|
||
RawStudio,
|
||
RawTag,
|
||
)
|
||
from app.connectors.direct_scrapers._browse_base import (
|
||
BaseBrowseScraper,
|
||
compute_thumbnail_phash,
|
||
)
|
||
from app.extractors import browser_get
|
||
from app.normalize.text import slugify
|
||
|
||
log = logging.getLogger(__name__)
|
||
|
||
_BASE = "https://latestpornvideo.com"
|
||
_ARTICLE_RE = re.compile(r'<article[^>]*\bclass="([^"]+)"', re.IGNORECASE)
|
||
_LINK_RE = re.compile(r'<a\s+href="([^"]+)"\s+title="([^"]+)"', re.IGNORECASE)
|
||
_THUMB_RE = re.compile(r'data-main-thumb="([^"]+)"', re.IGNORECASE)
|
||
_TICDN_RE = re.compile(r"^https?://(?:[a-z0-9-]+\.)*ticdn\.net/snaps/", re.IGNORECASE)
|
||
_CLASS_TAG_RE = re.compile(r"\btag-([a-z0-9-]+)")
|
||
_CLASS_CAT_RE = re.compile(r"\bcategory-([a-z0-9-]+)")
|
||
# Nazwa thumba: `<Studio>-YYYY-MM-DD-<rest>-cover.jpg`.
|
||
_THUMB_NAME_RE = re.compile(r"/([A-Za-z0-9][A-Za-z0-9-]*?)-(\d{4})-(\d{2})-(\d{2})-", re.IGNORECASE)
|
||
# Tytuł: `<Studio> YY MM DD <rest>` (np. "MySexMobile 20 10 23 Abella Danger").
|
||
_TITLE_DATE_RE = re.compile(r"^(.*?)\s*\b(\d{2})\s+(\d{2})\s+(\d{2})\b")
|
||
# Karty homepage zawsze siedzą w kategorii "latest-porn-videos", to nie jest tag.
|
||
# "mixed-porn" i "porn-movie": od 09.2026 serwis daje KAZDEMU wpisowi tylko te kategorie,
|
||
# wiec nic nie mowia (zgloszenie 3204ee85, 2026-10-12: "tylko mixed porn").
|
||
_CAT_SKIP = {"latest-porn-videos", "uncategorized", "", "mixed-porn", "porn-movie"}
|
||
|
||
|
||
_PIXHOST_SHOW_RE = re.compile(r"^https?://(?:www\.)?pixhost\.to/show/\d+/[^/?#]+$", re.IGNORECASE)
|
||
_OG_IMAGE_RE = re.compile(r"""property=["']og:image["'][^>]*content=["']([^"']+)""", re.IGNORECASE)
|
||
_LULU_EMBED_RE = re.compile(r"https?://(?:www\.)?lulu[a-z0-9]*\.[a-z]+/(?:e|d|v)/([a-z0-9]+)", re.IGNORECASE)
|
||
|
||
|
||
def _pixhost_removed(url: str) -> bool:
|
||
"""pixhost przekierowuje usuniety obrazek na baner /image/removed.png."""
|
||
try:
|
||
r = httpx.head(url, headers={"User-Agent": "Mozilla/5.0"}, timeout=10.0, follow_redirects=False)
|
||
except Exception:
|
||
return False
|
||
return r.status_code in (301, 302, 303, 307, 308) and "removed" in (r.headers.get("location") or "")
|
||
|
||
|
||
def _lulu_poster(page_url: str, timeout: float) -> str | None:
|
||
"""Plakat z odtwarzacza lulu osadzonego we wpisie (img.lulucdn.com/<id>.jpg)."""
|
||
try:
|
||
page = browser_get(page_url, timeout=timeout).text
|
||
except Exception:
|
||
return None
|
||
m = _LULU_EMBED_RE.search(page)
|
||
return f"https://img.lulucdn.com/{m.group(1)}.jpg" if m else None
|
||
|
||
|
||
def better_thumb(thumb: str | None, page_url: str, *, timeout: float = 30.0) -> str | None:
|
||
"""Miniatura, ktora da sie pokazac (zgloszenia eb20f1c0/4d8d5daf, 2026-10-12).
|
||
|
||
- `pixhost.to/show/...` to strona podgladu (HTML), nie obrazek: bierzemy jej og:image.
|
||
- Usuniety obrazek pixhost i brak miniatury: plakat z odtwarzacza lulu z wpisu.
|
||
Dodatkowe zapytania tylko w tych przypadkach (ok. 10% wpisow).
|
||
"""
|
||
if thumb and _PIXHOST_SHOW_RE.match(thumb):
|
||
try:
|
||
show = httpx.get(thumb, headers={"User-Agent": "Mozilla/5.0"}, timeout=10.0).text
|
||
m = _OG_IMAGE_RE.search(show)
|
||
thumb = html.unescape(m.group(1)) if m else None
|
||
except Exception:
|
||
thumb = None
|
||
if thumb and "pixhost." in thumb and _pixhost_removed(thumb):
|
||
thumb = None
|
||
if not thumb:
|
||
return _lulu_poster(page_url, timeout)
|
||
return thumb
|
||
|
||
|
||
def _name_from_slug(slug: str) -> str:
|
||
return " ".join(w.capitalize() for w in slug.split("-") if w)
|
||
|
||
|
||
class LatestPornVideoScraper(BaseBrowseScraper):
|
||
sitetag = "latestpornvideocom"
|
||
|
||
def _listing_url(self, page: int) -> str:
|
||
return _BASE + "/" if page <= 1 else f"{_BASE}/page/{page}/"
|
||
|
||
# crawl_page nadpisany → poniższe abstrakcje nieużywane, ale wymagane do instancji.
|
||
def _extract_scene_urls(self, listing_html: str) -> list[str]:
|
||
return [m.group(1) for m in _LINK_RE.finditer(listing_html)]
|
||
|
||
def _parse_detail(self, scene_url: str, detail_html: str) -> RawScene | None:
|
||
return None
|
||
|
||
def crawl_page(self, page: int) -> list[RawScene] | None:
|
||
url = self._listing_url(page)
|
||
try:
|
||
res = browser_get(url, timeout=self._timeout)
|
||
text = res.text if hasattr(res, "text") else res
|
||
except Exception as e:
|
||
log.warning("latestpornvideo browse listing fetch failed (page %d): %s", page, e)
|
||
return None
|
||
|
||
anchors = list(_ARTICLE_RE.finditer(text))
|
||
out: list[RawScene] = []
|
||
seen: set[str] = set()
|
||
for idx, m in enumerate(anchors):
|
||
cls = m.group(1)
|
||
win_end = anchors[idx + 1].start() if idx + 1 < len(anchors) else m.end() + 1500
|
||
window = text[m.start():win_end]
|
||
|
||
link_m = _LINK_RE.search(window)
|
||
if not link_m:
|
||
continue
|
||
scene_url = link_m.group(1).rstrip("/") + "/"
|
||
# tylko właściwe posty scen (/<digits>/), bez nav/kategorii
|
||
if not re.fullmatch(rf"{re.escape(_BASE)}/\d+/", scene_url) or scene_url in seen:
|
||
continue
|
||
seen.add(scene_url)
|
||
title = html.unescape(link_m.group(2)).strip()
|
||
if not title:
|
||
continue
|
||
|
||
thumb_m = _THUMB_RE.search(window)
|
||
thumb = thumb_m.group(1) if thumb_m else None
|
||
if thumb:
|
||
# ticdn.net (stary serwer miniatur DoodStream) nie istnieje, te same pliki
|
||
# leza na imgds.net (2026-10-15, 8 tys. martwych kafelkow).
|
||
thumb = _TICDN_RE.sub("https://imgds.net/snaps/", thumb)
|
||
|
||
# Studio + release_date z nazwy thumba (`<Studio>-YYYY-MM-DD-`).
|
||
studio: RawStudio | None = None
|
||
release_date: date | None = None
|
||
if thumb and (tn := _THUMB_NAME_RE.search(thumb)):
|
||
studio_raw = tn.group(1).replace("-", " ").strip()
|
||
if studio_raw:
|
||
studio = RawStudio(
|
||
external_id=f"{self.sitetag}:studio:{slugify(studio_raw)}",
|
||
name=studio_raw, slug=slugify(studio_raw),
|
||
)
|
||
try:
|
||
release_date = date(int(tn.group(2)), int(tn.group(3)), int(tn.group(4)))
|
||
except ValueError:
|
||
release_date = None
|
||
|
||
# Fallback z tytułu: `<Studio> YY MM DD ...`.
|
||
if studio is None or release_date is None:
|
||
if tm2 := _TITLE_DATE_RE.search(title):
|
||
if release_date is None:
|
||
try:
|
||
release_date = date(
|
||
2000 + int(tm2.group(2)), int(tm2.group(3)), int(tm2.group(4))
|
||
)
|
||
except ValueError:
|
||
release_date = None
|
||
studio_raw = tm2.group(1).strip(" -–")
|
||
if studio is None and 2 <= len(studio_raw) <= 30:
|
||
studio = RawStudio(
|
||
external_id=f"{self.sitetag}:studio:{slugify(studio_raw)}",
|
||
name=studio_raw, slug=slugify(studio_raw),
|
||
)
|
||
|
||
# Tagi: tylko prawdziwe kategorie (category-*), bez "latest-porn-videos".
|
||
# `tag-*` POMIJAMY, to mieszanka fragmentów imion performerów i gatunków,
|
||
# bez `actors-*` (jak na /actor/) nie da się ich rozdzielić → byłby szum.
|
||
tags: list[RawTag] = []
|
||
seen_tag: set[str] = set()
|
||
for cm in _CLASS_CAT_RE.finditer(cls):
|
||
sl = cm.group(1)
|
||
if sl in _CAT_SKIP or sl in seen_tag:
|
||
continue
|
||
seen_tag.add(sl)
|
||
tags.append(RawTag(external_id=f"{self.sitetag}:tag:{sl}", name=_name_from_slug(sl), slug=sl))
|
||
|
||
# Po odczycie studia i daty z nazwy pliku, bo zapas (plakat lulu) tej nazwy nie ma.
|
||
thumb = better_thumb(thumb, scene_url, timeout=self._timeout)
|
||
|
||
fingerprints = []
|
||
if thumb:
|
||
ph = compute_thumbnail_phash(thumb, referer=_BASE + "/")
|
||
if ph:
|
||
from app.connectors.base import RawFingerprint
|
||
|
||
fingerprints.append(RawFingerprint(kind="phash", value=ph))
|
||
|
||
out.append(
|
||
RawScene(
|
||
external_id=f"{self.sitetag}:{scene_url}",
|
||
title=title,
|
||
release_date=release_date,
|
||
url=scene_url,
|
||
studio=studio,
|
||
performers=[],
|
||
tags=tags,
|
||
fingerprints=fingerprints,
|
||
playback_sources=[
|
||
RawPlaybackSource(
|
||
origin=f"tube:{self.sitetag}",
|
||
page_url=scene_url,
|
||
thumbnail_url=thumb,
|
||
)
|
||
],
|
||
)
|
||
)
|
||
|
||
log.info("latestpornvideo browse page %d: %d scenes", page, len(out))
|
||
return out
|