fix(sxyprn): revive search via performer pages + rich metadata
sxyprn ingest was frozen since 05-07: the old ?type=videos&query= endpoint returns trending (not performer-filtered), so the strict token filter correctly dropped everything -> 0 ingest. Real "search" is the performer page /<First-Last>.html. Rewrote search() to scrape those cards: clean performer (the query, avoids sxyprn's Dallas/Rae name fragmentation), studio (channel subcat), tags (#hashtags), duration, thumbnail. Token filter now runs on the card title so only genuine matches attach the performer. Verified: Lana Rhoades/Riley Reid/Angela White return results, metadata persists in DB (studio e.g. Vixen, 10-31 tags/scene), playback mp4 206. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
jtrzupek
parent
00f4779abe
commit
e0e69189a8
1 changed files with 180 additions and 11 deletions
|
|
@ -1,24 +1,193 @@
|
||||||
"""sxyprn.com — direct HTML scrape search results.
|
"""sxyprn.com — performer-page scrape (search-based, performer-driven).
|
||||||
|
|
||||||
Sxyprn search jest oparte na `?type=videos&query=<q>` GET endpoint który zwraca
|
2026-06-16 fix (zamrożony od 05-07): sxyprn NIE ma free-text searcha. Stary endpoint
|
||||||
HTML strony z linkami. Scene URL format: `https://sxyprn.com/post/<post_id>.html`.
|
`?type=videos&query=<q>` oddawał TRENDING (0 trafień dla performera → strict token
|
||||||
|
filtr słusznie wycinał wszystko → 0 ingestu → freshness zamrożona). Realny "search"
|
||||||
|
to **strona performera** `https://sxyprn.com/<Imie-Nazwisko>.html` (spacje → myślniki),
|
||||||
|
zwraca ~30 kart scen tego performera.
|
||||||
|
|
||||||
Page'owanie sxyprn niespójne — często single-page results dla query (~24 wyników).
|
Bonus tej ścieżki: w trybie performer-driven query = czysta nazwa performera, więc
|
||||||
|
omijamy fragmentację sxyprn (post taguje "Dallas Rae" jako osobne `/Dallas.html` +
|
||||||
|
`/Rae.html` / `aria-label='Dallas,Rae'`). Performera bierzemy z query (po potwierdzeniu
|
||||||
|
token-filtrem na tytule), a NIE z połamanych tagów.
|
||||||
|
|
||||||
|
Metadane z karty (bez +1 fetch detalu):
|
||||||
|
- URL sceny: `/post/<id>.html`
|
||||||
|
- duration: `<span class='duration_small'>MM:SS|HH:MM:SS`
|
||||||
|
- studio: `<span class='post_el_small_subcat'>Channel` (sxyprn "channel"; ~⅔ kart, opcjonalny)
|
||||||
|
- title + tagi: `<a class='post_time' title='{New} ...#Tag1 #Tag2... {Watch...} url'>`
|
||||||
|
- thumbnail: `data-src='//...small.jpg'`
|
||||||
|
|
||||||
|
Playback resolwuje extractor `sxyprncom` (osobno; mp4 z trafficdeposit/lulustream).
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import html
|
||||||
|
import logging
|
||||||
import re
|
import re
|
||||||
|
|
||||||
|
from app.connectors.base import (
|
||||||
|
RawPerformer,
|
||||||
|
RawPlaybackSource,
|
||||||
|
RawScene,
|
||||||
|
RawStudio,
|
||||||
|
RawTag,
|
||||||
|
)
|
||||||
from app.connectors.direct_scrapers._search_base import BaseSearchScraper
|
from app.connectors.direct_scrapers._search_base import BaseSearchScraper
|
||||||
|
from app.extractors import browser_get
|
||||||
|
from app.normalize.text import slugify
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
_BASE = "https://sxyprn.com"
|
||||||
|
|
||||||
|
# js-pop anchor karty: /post/<id>.html(?sk=...&so=...&ss=latest na stronie performera)
|
||||||
|
# + aria-label (połamani performerzy — nieużywane).
|
||||||
|
_CARD_ANCHOR_RE = re.compile(
|
||||||
|
r"<a\s+href='/post/(?P<id>[a-f0-9]+)\.html(?:\?[^']*)?'[^>]*class='js-pop'",
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
# Uwaga: duration_small ma atrybut `title='s1->c10'` ze znakiem `>` w środku, więc
|
||||||
|
# NIE używamy `[^>]*` (łamie się na tym `>`) — leniwe `.*?` do wartości HH:MM:SS.
|
||||||
|
_DURATION_RE = re.compile(
|
||||||
|
r"duration_small.*?>\s*(\d{1,2}:\d{2}(?::\d{2})?)\s*<", re.IGNORECASE | re.DOTALL
|
||||||
|
)
|
||||||
|
_SUBCAT_RE = re.compile(r"post_el_small_subcat[^>]*>([^<]+)<", re.IGNORECASE)
|
||||||
|
_THUMB_RE = re.compile(r"data-src='(//[^']+?small\.jpg)'", re.IGNORECASE)
|
||||||
|
# post_time z tytułem — keyowane po ID, bo leży w post_control PO vid_container.
|
||||||
|
_POSTTIME_RE = re.compile(
|
||||||
|
r"post_time'[^>]*href='/post/{id}\.html(?:\?[^']*)?'[^>]*title='([^']*)'",
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
_HASHTAG_RE = re.compile(r"#(\w[\w-]*)")
|
||||||
|
_BRACE_RE = re.compile(r"\{[^}]*\}")
|
||||||
|
_URL_TAIL_RE = re.compile(r"https?://\S+")
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_duration(s: str) -> int | None:
|
||||||
|
parts = s.strip().split(":")
|
||||||
|
try:
|
||||||
|
if len(parts) == 2:
|
||||||
|
return int(parts[0]) * 60 + int(parts[1])
|
||||||
|
if len(parts) == 3:
|
||||||
|
return int(parts[0]) * 3600 + int(parts[1]) * 60 + int(parts[2])
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _clean_title(raw: str) -> tuple[str, list[str]]:
|
||||||
|
"""Zwraca (czysty tytuł, tagi). Usuwa {markery}, hashtagi i końcowy URL."""
|
||||||
|
text = html.unescape(raw)
|
||||||
|
tags = [m.group(1) for m in _HASHTAG_RE.finditer(text)]
|
||||||
|
text = _BRACE_RE.sub(" ", text) # {New}, {Watch At 1080P}, {WATCH...}
|
||||||
|
text = _URL_TAIL_RE.sub(" ", text) # końcowy link do hostera
|
||||||
|
text = _HASHTAG_RE.sub(" ", text) # #Tag
|
||||||
|
text = re.sub(r"\s+", " ", text).strip(" :-")
|
||||||
|
return text, tags
|
||||||
|
|
||||||
|
|
||||||
class SxyPrnScraper(BaseSearchScraper):
|
class SxyPrnScraper(BaseSearchScraper):
|
||||||
sitetag = "sxyprncom"
|
sitetag = "sxyprncom"
|
||||||
_search_url_template = "https://sxyprn.com/?type=videos&query={query}&page={page}"
|
# _search_url_template/_scene_url_re z bazy nieużywane — mamy custom search().
|
||||||
_scene_url_re = re.compile(
|
_card_window = 2600 # od js-pop anchora do post_control z tytułem
|
||||||
r'href="(?P<url>/post/(?P<slug>[a-z0-9]+))\.html"',
|
|
||||||
)
|
|
||||||
|
|
||||||
def _title_from_slug(self, slug: str) -> str:
|
def _performer_path(self, query: str) -> str:
|
||||||
# sxyprn post ID to nieczytelny hash — placeholder, title backfill przy resolve.
|
"""`Lana Rhoades` → `Lana-Rhoades` (strona performera sxyprn)."""
|
||||||
return f"sxyprn:{slug}"
|
cleaned = re.sub(r"[^A-Za-z0-9 ]+", "", query).strip()
|
||||||
|
return re.sub(r"\s+", "-", cleaned)
|
||||||
|
|
||||||
|
def search(self, query, *, page: int = 1, limit=None):
|
||||||
|
path = self._performer_path(query)
|
||||||
|
if not path:
|
||||||
|
return
|
||||||
|
url = f"{_BASE}/{path}.html" + (f"?page={page}" if page > 1 else "")
|
||||||
|
try:
|
||||||
|
r = browser_get(url, timeout=self._timeout)
|
||||||
|
except Exception as e:
|
||||||
|
log.warning("sxyprn search fetch failed (%s): %s", url, e)
|
||||||
|
return
|
||||||
|
if r.status_code != 200:
|
||||||
|
log.debug("sxyprn search %s status=%d", url, r.status_code)
|
||||||
|
return
|
||||||
|
|
||||||
|
text = r.text
|
||||||
|
query_tokens = {
|
||||||
|
tok for tok in query.lower().split() if len(tok) >= self._query_token_min_len
|
||||||
|
}
|
||||||
|
anchors = list(_CARD_ANCHOR_RE.finditer(text))
|
||||||
|
seen: set[str] = set()
|
||||||
|
yielded = 0
|
||||||
|
for idx, m in enumerate(anchors):
|
||||||
|
post_id = m.group("id")
|
||||||
|
if post_id in seen:
|
||||||
|
continue
|
||||||
|
seen.add(post_id)
|
||||||
|
win_end = anchors[idx + 1].start() if idx + 1 < len(anchors) else m.end() + self._card_window
|
||||||
|
window = text[m.start():win_end]
|
||||||
|
|
||||||
|
tm = re.search(_POSTTIME_RE.pattern.format(id=re.escape(post_id)), text, re.IGNORECASE)
|
||||||
|
if not tm:
|
||||||
|
continue
|
||||||
|
title, tags = _clean_title(tm.group(1))
|
||||||
|
title_l = title.lower()
|
||||||
|
# Strict: scena musi realnie dotyczyć performera (oba tokeny w tytule) —
|
||||||
|
# chroni przed śmieciem na stronie i przed over-attribution.
|
||||||
|
if query_tokens and not all(tok in title_l for tok in query_tokens):
|
||||||
|
continue
|
||||||
|
if not title:
|
||||||
|
continue
|
||||||
|
|
||||||
|
scene_url = f"{_BASE}/post/{post_id}.html"
|
||||||
|
dur_m = _DURATION_RE.search(window)
|
||||||
|
duration_sec = _parse_duration(dur_m.group(1)) if dur_m else None
|
||||||
|
thumb_m = _THUMB_RE.search(window)
|
||||||
|
thumb = thumb_m.group(1) if thumb_m else None
|
||||||
|
if thumb and thumb.startswith("//"):
|
||||||
|
thumb = "https:" + thumb
|
||||||
|
|
||||||
|
studio = None
|
||||||
|
sub_m = _SUBCAT_RE.search(window)
|
||||||
|
if sub_m:
|
||||||
|
name = html.unescape(sub_m.group(1)).strip()
|
||||||
|
if name and name.lower() not in ("all", "trending"):
|
||||||
|
studio = RawStudio(
|
||||||
|
external_id=f"{self.sitetag}:studio:{slugify(name)}",
|
||||||
|
name=name,
|
||||||
|
slug=slugify(name),
|
||||||
|
)
|
||||||
|
|
||||||
|
tag_objs: list[RawTag] = []
|
||||||
|
seen_tag: set[str] = set()
|
||||||
|
for t in tags:
|
||||||
|
sl = slugify(t)
|
||||||
|
if not sl or sl in seen_tag:
|
||||||
|
continue
|
||||||
|
seen_tag.add(sl)
|
||||||
|
tag_objs.append(RawTag(external_id=f"{self.sitetag}:tag:{sl}", name=t, slug=sl))
|
||||||
|
|
||||||
|
performer = RawPerformer(
|
||||||
|
external_id=f"{self.sitetag}:performer:{slugify(query)}",
|
||||||
|
name=query.strip(),
|
||||||
|
)
|
||||||
|
|
||||||
|
yield RawScene(
|
||||||
|
external_id=f"{self.sitetag}:{post_id}",
|
||||||
|
title=title,
|
||||||
|
duration_sec=duration_sec,
|
||||||
|
url=scene_url,
|
||||||
|
studio=studio,
|
||||||
|
performers=[performer],
|
||||||
|
tags=tag_objs,
|
||||||
|
playback_sources=[
|
||||||
|
RawPlaybackSource(
|
||||||
|
origin=f"tube:{self.sitetag}",
|
||||||
|
page_url=scene_url,
|
||||||
|
duration_sec=duration_sec,
|
||||||
|
thumbnail_url=thumb,
|
||||||
|
)
|
||||||
|
],
|
||||||
|
)
|
||||||
|
yielded += 1
|
||||||
|
if limit and yielded >= limit:
|
||||||
|
return
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue