diff --git a/app/connectors/direct_scrapers/__init__.py b/app/connectors/direct_scrapers/__init__.py index 807e12b..1e94fc3 100644 --- a/app/connectors/direct_scrapers/__init__.py +++ b/app/connectors/direct_scrapers/__init__.py @@ -44,6 +44,7 @@ from app.connectors.direct_scrapers.siska import SiskaScraper from app.connectors.direct_scrapers.sxyland import SxyLandScraper from app.connectors.direct_scrapers.sxyprn import SxyPrnScraper from app.connectors.direct_scrapers.watchporn import WatchPornScraper +from app.connectors.direct_scrapers.youperv import YoupervScraper from app.connectors.direct_scrapers.xhamster import XHamsterScraper from app.connectors.direct_scrapers.xmoviesforyou import XMoviesForYouScraper from app.connectors.direct_scrapers.xnxx import XnxxScraper @@ -152,6 +153,11 @@ ALL_BROWSE_SCRAPERS: list[type[BaseBrowseScraper]] = [ # uploadDate, /models/ performerzy, /tags/ tagi, /categories/ studio. Playback KVS # get_file direct mp4 (extractor watchporn, VPS-side, token nie IP-bound). WatchPornScraper, + # YoupervScraper — dodany 2026-07-26 (ocena: orphan-risk LOW, najlepszy kandydat od + # hqporner). Ripy paysite, tytuły `Studio - Performer - Title` (71% próbki), performerzy + # z linków, duration + release_date ISO, ~60-70 scen/dzień. Playback: direct mp4 bez + # tokena (extractor youpervcom, VPS-side, CDN pilnuje tylko Referera). + YoupervScraper, # Browse równolegle do istniejącego search scrapera (wzorzec xvideos/eporner): # search zostaje (pokrycie back-catalogu performerów), browse gwarantuje świeżość # wprost z feedu (watchdog 48h zamiast 168h). Konwersja 2026-06-24 (user request). diff --git a/app/connectors/direct_scrapers/youperv.py b/app/connectors/direct_scrapers/youperv.py new file mode 100644 index 0000000..e262130 --- /dev/null +++ b/app/connectors/direct_scrapers/youperv.py @@ -0,0 +1,173 @@ +"""youperv.com — browse scraper (DataLife Engine). Dodany 2026-07-26. + +Ripy paysite ze studiami (Brazzers Exxtra, Blacked, Evil Angel, Deeper, Private…), +tytuły w formacie `Studio - Performer - Title` (71% próbki 132 tytułów), świeże +(~60-70 scen/dzień). Orphan-risk LOW: nazwane studio + nazwany performer + data +co do sekundy + duration = mocny sygnał do canonical match. + +Listing: homepage (newest) + `/page/N/`, 19 scen/stronę, zero overlapu między +stronami. Scene URL: `//-.html`. + +**KRYTYCZNE — scoping obsady**: performerzy MUSZĄ być czytani tylko z bloku +`
` … `Related`. Na całej stronie jest 19-26 linków +`xfsearch/pornstar/` (blok Related), w samym fmeta 1-2 realnych. Bez scopingu +powtórzylibyśmy błąd, przez który odrzuciliśmy xxxfiles (page-wide pollution +zaśmiecająca bazę performerów). + +Tytuł zostaje z prefiksem studia (jak hdporngg/porn00) — token_set_ratio i tak +złapie canonical, a prefiks niesie dodatkowy sygnał. + +Playback: direct mp4 `` na files.klubnichka-hd.com, BEZ tokena/expiry, ale +CDN ma hotlink-guard na Referer (bez nagłówka 403, z nagłówkiem 206 cross-IP). +Rozwiązuje extractor `youpervcom` (VPS-side, mobile gra direct, zero WebView/proxy). + +Głębokość: deep-crawl capowany (`_PAGE_CAP` w deep_crawl.py) — strony ~2100+ to +stara amatorka bez performerów, z martwymi linkami (HTTP 500 na CDN). +""" +from __future__ import annotations + +import html +import re +from urllib.parse import unquote + +from app.connectors.base import ( + RawPerformer, + RawPlaybackSource, + RawScene, + RawStudio, + RawTag, +) +from app.connectors.direct_scrapers._browse_base import BaseBrowseScraper, meta_content +from app.connectors.direct_scrapers._playtube import _parse_iso_date +from app.normalize.text import slugify + +_BASE = "https://youperv.com" + +_SCENE_URL_RE = re.compile( + r'href="(https://youperv\.com/[a-z0-9\-]+/\d+-[^"]+\.html)"', re.IGNORECASE +) +_H1_RE = re.compile(r']*class="items-title[^"]*"[^>]*>(.*?)', re.IGNORECASE | re.DOTALL) +_PERF_RE = re.compile(r'xfsearch/pornstar/([^/"]+)', re.IGNORECASE) +_CAT_XF_RE = re.compile(r'xfsearch/cat/([^/"]+)', re.IGNORECASE) +_TAG_LINK_RE = re.compile(r']+href="[^"]+"[^>]*>([^<]{2,40})', re.IGNORECASE) +_DUR_RE = re.compile(r"fa-clock-o[^>]*>\s*(\d{1,2}):(\d{2})(?::(\d{2}))?", re.IGNORECASE) +_DATE_RE = re.compile(r'"datePublished"\s*:\s*"([^"]+)"') +# Sufiks h1: `… Title 07.26.2026 HD` +_H1_DATE_SUFFIX_RE = re.compile(r"\s*\d{2}\.\d{2}\.\d{4}\s*$") + + +def _clean_title(raw_h1: str) -> str: + text = re.sub(r"<[^>]+>", " ", raw_h1) # HD itp. + text = html.unescape(re.sub(r"\s+", " ", text)).strip() + text = re.sub(r"\bHD\b\s*$", "", text).strip() + return _H1_DATE_SUFFIX_RE.sub("", text).strip() + + +def _perf_name(raw_slug: str) -> str: + """`carolina%20guerrero` → `Carolina Guerrero`.""" + name = unquote(raw_slug).replace("-", " ").strip() + return " ".join(w.capitalize() if w.islower() else w for w in name.split()) + + +class YoupervScraper(BaseBrowseScraper): + sitetag = "youpervcom" + + def _listing_url(self, page: int) -> str: + return f"{_BASE}/" if page <= 1 else f"{_BASE}/page/{page}/" + + def _extract_scene_urls(self, listing_html: str) -> list[str]: + seen: set[str] = set() + out: list[str] = [] + for m in _SCENE_URL_RE.finditer(listing_html): + url = m.group(1) + if url not in seen: # każdy link jest 2× w karcie (thumb + tytuł) + seen.add(url) + out.append(url) + return out + + def _parse_detail(self, scene_url: str, detail_html: str) -> RawScene | None: + h1 = _H1_RE.search(detail_html) + title = _clean_title(h1.group(1)) if h1 else "" + if not title: + og = meta_content(detail_html, property="og:title") or "" + title = og.split(" » ")[0].strip() + if not title: + return None + + # Blok metadanych TEJ sceny: od `class="fmeta` do sekcji Related (dalej idą + # linki powiązanych scen → performer pollution, patrz docstring). + i = detail_html.find('class="fmeta') + j = detail_html.find("Related", i + 1) if i >= 0 else -1 + fmeta = detail_html[i:j] if i >= 0 and j > i else "" + + performers: list[RawPerformer] = [] + seen_p: set[str] = set() + for m in _PERF_RE.finditer(fmeta): + name = _perf_name(m.group(1)) + sl = slugify(name) + if not sl or sl in seen_p: + continue + seen_p.add(sl) + performers.append( + RawPerformer(external_id=f"{self.sitetag}:performer:{sl}", name=name) + ) + + # Studio z prefiksu `Studio - Performer - Title` (≥3 człony). Guard: prefiks + # nie może być nazwiskiem performera (wtedy to `Performer - Title`, bez studia). + studio: RawStudio | None = None + parts = [p.strip() for p in title.split(" - ")] + if len(parts) >= 3 and 2 <= len(parts[0]) <= 40: + cand = parts[0] + if slugify(cand) not in seen_p: + studio = RawStudio( + external_id=f"{self.sitetag}:studio:{slugify(cand)}", + name=cand, + slug=slugify(cand), + ) + + tags: list[RawTag] = [] + seen_t: set[str] = set() + tag_names = [_perf_name(m.group(1)) for m in _CAT_XF_RE.finditer(fmeta)] + ti = detail_html.find("full-tags") + if ti >= 0: + block = detail_html[ti:ti + 800] + tag_names += [html.unescape(m.group(1)).strip() for m in _TAG_LINK_RE.finditer(block)] + for name in tag_names: + sl = slugify(name) + if not sl or sl in seen_t or sl in seen_p or name.lower() in ("categories", "tags"): + continue + seen_t.add(sl) + tags.append(RawTag(external_id=f"{self.sitetag}:tag:{sl}", name=name, slug=sl)) + + duration_sec: int | None = None + dm = _DUR_RE.search(fmeta or detail_html) + if dm: + h_or_m, mins, secs = dm.group(1), dm.group(2), dm.group(3) + duration_sec = ( + int(h_or_m) * 3600 + int(mins) * 60 + int(secs) + if secs + else int(h_or_m) * 60 + int(mins) + ) + + rd = _DATE_RE.search(detail_html) + release_date = _parse_iso_date(rd.group(1)) if rd else None + thumbnail_url = meta_content(detail_html, property="og:image") + + return RawScene( + external_id=f"{self.sitetag}:{scene_url}", + title=title, + release_date=release_date, + duration_sec=duration_sec, + url=scene_url, + studio=studio, + performers=performers, + tags=tags, + playback_sources=[ + RawPlaybackSource( + origin=f"tube:{self.sitetag}", + page_url=scene_url, + duration_sec=duration_sec, + thumbnail_url=thumbnail_url, + ) + ], + ) diff --git a/app/extractors/__init__.py b/app/extractors/__init__.py index 6fa7e69..57bfa37 100644 --- a/app/extractors/__init__.py +++ b/app/extractors/__init__.py @@ -45,6 +45,7 @@ from app.extractors.tubes import ( watchporn, xhamster, yespornvip, + youperv, ) log = logging.getLogger(__name__) @@ -104,6 +105,9 @@ _REGISTRY: dict[str, Callable[[str], list[StreamSource] | None]] = { # watchporn — 2026-07-02 przebudowany na KVS (DoodStream-CAPTCHA zniknął). # flashvars get_file direct mp4, same-session 302 resolve, token nie IP-bound. "watchporn": watchporn.extract, + # youperv — fluidplayer, zwykły mp4 na files.klubnichka-hd.com. Bez tokena + # i bez IP-bindingu; CDN pilnuje tylko Referera (cross-IP 206 z VPS) → mobile direct. + "youpervcom": youperv.extract, "siskavideo": _embed_iframe.extract, "porn4dayspw": _embed_iframe.extract, "porndishcom": _embed_iframe.extract, diff --git a/app/extractors/tubes/youperv.py b/app/extractors/tubes/youperv.py new file mode 100644 index 0000000..c865bd2 --- /dev/null +++ b/app/extractors/tubes/youperv.py @@ -0,0 +1,52 @@ +"""youperv.com — direct mp4 extractor (fluidplayer). Dodany 2026-07-26. + +Scene page ma zwykły `` +— BEZ tokena, bez expiry, bez KVS/DoodStream/iframe. URL zawiera spacje (tytuł w +ścieżce), więc trzeba go za-quote'ować przed podaniem playerowi. + +CDN ma hotlink-guard na **Referer**: goły Range → 403 nginx, Range + Referer +`https://youperv.com/` + browser UA → 206 `video/mp4` (zweryfikowane cross-IP z VPS). +Nie jest IP-bound, więc `mobile_direct_ok` → telefon gra prosto z CDN, zero proxy +i zero WebView. playback.py dokleja Referer+UA z `referer=` do nagłówków StreamLink. +""" +from __future__ import annotations + +import logging +import re +from urllib.parse import quote, urlsplit, urlunsplit + +from app.extractors._fetch import fetch_tube_html +from app.extractors._models import StreamSource + +log = logging.getLogger(__name__) + +_BASE = "https://youperv.com" +_SOURCE_RE = re.compile( + r']+src="([^"]+\.(?:mp4|m4v)[^"]*)"', re.IGNORECASE +) + + +def _encode_url(url: str) -> str: + """Spacje/znaki specjalne w ścieżce → percent-encoding (ExoPlayer inaczej odpada). + Query zostawiamy nietknięte.""" + parts = urlsplit(url) + return urlunsplit( + (parts.scheme, parts.netloc, quote(parts.path, safe="/%"), parts.query, parts.fragment) + ) + + +def extract(page_url: str, *, timeout: float = 60.0) -> list[StreamSource] | None: + html_text = fetch_tube_html(page_url, timeout=timeout) + m = _SOURCE_RE.search(html_text) + if not m: + log.info("youperv: no mp4 on %s", page_url) + return None + return [ + StreamSource( + link=_encode_url(m.group(1).strip()), + type="mp4", + referer=_BASE + "/", + # Token brak; CDN pilnuje tylko Referera (cross-IP 206) → mobile direct. + raw={"mobile_direct_ok": True}, + ) + ] diff --git a/app/scheduler/deep_crawl.py b/app/scheduler/deep_crawl.py index 75ea47b..13ebef2 100644 --- a/app/scheduler/deep_crawl.py +++ b/app/scheduler/deep_crawl.py @@ -37,6 +37,10 @@ _DEFAULT_STATE = Path(__file__).resolve().parent.parent / "_state" / "deepcrawl_ # capu (None) → naturalny koniec katalogu. xvideos /new/ ~27 scen/stronę → 1800 ≈ ~50k. _PAGE_CAP: dict[str, int] = { "xvideoscom": 1800, + # youperv: strony ~2100+ (≤09.2023) to stara generyczna amatorka bez performerów, + # do tego martwe pliki na CDN (HTTP 500). Świeży korpus (Studio-Performer-Title) + # kończy się ~str. 2000 ≈ 38k scen — dalej crawl tylko generuje orphany. + "youpervcom": 2000, } # Miękki budżet czasu na run (s). Detail-fetch scrapery (per-scena fetch strony, np.