09f389100a
Some checks are pending
Backend tests / test (push) Waiting to run
- browse_runs (migracja 0056): jeden wiersz per scraper i przebieg browse-latest - quality_watchdog: listing pusty albo bez nowych, skok lub zanik obsady, sceny bez dlugosci i miniatur, spadek udanych odtworzen per origin; fingerprint per zrodlo i rodzaj - ingest_watchdog: prog swiezosci z mediany odstepow zrodla (3x, max 14 d) zamiast stalej - duration backfill: resolvery perverzija (JSON-LD) i mypornerleak (itemprop), paczka po rowno z originow, 404/410 zamyka wiersz od razu - parsery: galaxyporn data z JSON-LD, hentaimama dlugosc, javflix aktorki, fullmovies i hdporngg data z uploadDate
236 lines
10 KiB
Python
236 lines
10 KiB
Python
"""galaxyporn.net, browse scraper (WordPress retrotube). Dodany 2026-07-26.
|
|
|
|
Ripy paysite ze studiami (BrazzersExxtra, EvilAngel, NubilesPorn, RealityKings,
|
|
MYLF, BlackedRaw…). Ocena: orphan-risk LOW z twardym pomiarem, 92% performerów
|
|
z próbki 75 ma już u nas ref tpdb/stashdb, 79% tytułów matchuje istniejące sceny.
|
|
|
|
Listing `/page/N/`, 21 scen/stronę, ~1180 stron, paginacja zdrowa (overlap p1/p2 = 0).
|
|
|
|
Metadane:
|
|
- obsada: `<div id="video-actors">`, czysta, BEZ pollution (na całej stronie sceny
|
|
są dokładnie te 2-3 linki `/actor/`, nie ma sekcji Related z obcymi performerami)
|
|
- studio + data: prefiks tytułu `Studio YY MM DD Performer - Title` albo `[Studio]`
|
|
- tagi: `<div class="tags-list">` (wycinamy slug studia, żeby nie robić tag=studio)
|
|
- thumbnail: `data-main-thumb` / og:image
|
|
|
|
**Duration, uwaga, nie ma go w HTML.** Ani listing, ani detail, ani JSON-LD, ani
|
|
payload playera nie niosą długości. NULL duration jest GROŹNY: `/scenes` filtruje
|
|
`duration_sec >= min_duration_sec`, a NULL w SQL odpada z wyniku, więc sceny byłyby
|
|
NIEWIDOCZNE w apce mimo "new: N, errors: 0" (dokładnie incydent porntrex, 6479 scen).
|
|
Dlatego liczymy ją z `thumbnail.vtt` playera (ostatni cue), 1 dodatkowy request na
|
|
scenę po odpytaniu API seekplayer. Wartość jest ~1 interwał miniatur krótsza od
|
|
realnej (2226 s vs ~2276 s na próbce, ~2%), akceptowalne, bo alternatywą jest brak.
|
|
|
|
Playback: iframe `#hash` rodziny seekplayer (galaxy.upns.online / galaxy.4meplayer.pro /
|
|
sport.seekplays.com / news.upns.pro) → extractor `galaxyporncom` → istniejący
|
|
`seekplayer_engine` (ten sam AES key/IV). HLS wymaga Referera → `/proxy/hls`.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import html
|
|
import json
|
|
import logging
|
|
import re
|
|
import time
|
|
from datetime import date
|
|
from urllib.parse import urlparse
|
|
|
|
from app.connectors.base import (
|
|
RawPerformer,
|
|
RawPlaybackSource,
|
|
RawScene,
|
|
RawStudio,
|
|
RawTag,
|
|
)
|
|
from app.connectors.direct_scrapers._browse_base import BaseBrowseScraper, meta_content
|
|
from app.extractors._fetch import _DEFAULT_UA, browser_get
|
|
from app.normalize.text import slugify
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
_BASE = "https://galaxyporn.net"
|
|
|
|
_SCENE_URL_RE = re.compile(r'<a\s+href="(https://galaxyporn\.net/[a-z0-9\-]+/)"[^>]*title=', re.IGNORECASE)
|
|
_H1_RE = re.compile(r'<h1[^>]*class="entry-title"[^>]*>(.*?)</h1>', re.IGNORECASE | re.DOTALL)
|
|
_ACTORS_BLOCK_RE = re.compile(r'<div id="video-actors">(.*?)</div>', re.IGNORECASE | re.DOTALL)
|
|
_ANCHOR_TEXT_RE = re.compile(r">([^<>]+)</a>")
|
|
_TAGS_BLOCK_RE = re.compile(r'<div class="tags-list">(.*?)</div>', re.IGNORECASE | re.DOTALL)
|
|
_THUMB_RE = re.compile(r'data-main-thumb="([^"]+)"', re.IGNORECASE)
|
|
_IFRAME_RE = re.compile(r'<iframe[^>]+src="([^"]+)"', re.IGNORECASE)
|
|
|
|
# `BrazzersExxtra 26 07 26 Kayley Gunner & Elly Clutch - Swapping Off The Pervy Maid`
|
|
_TITLE_STUDIO_DATE_RE = re.compile(r"^(?P<studio>[A-Za-z0-9][A-Za-z0-9._-]{1,30})\s+(?P<y>\d{2})\s+(?P<m>\d{2})\s+(?P<d>\d{2})\s+(?P<rest>.+)$")
|
|
# `[Onlyfans] Some Title` / `[Darkkotv] …`
|
|
_TITLE_BRACKET_RE = re.compile(r"^\[(?P<studio>[^\]]{2,30})\]\s*(?P<rest>.+)$")
|
|
# Data z JSON-LD strony, gdy tytul jej nie niesie (audyt 2026-10-14: 66% scen z 7 dni
|
|
# bez daty i bez dlugosci; data jest w `"datePublished":"2026-10-13T18:40:00+03:00"`).
|
|
_JSONLD_DATE_RE = re.compile(r'"datePublished"\s*:\s*"(\d{4})-(\d{2})-(\d{2})')
|
|
# `… [2018-09-28]`
|
|
_TITLE_ISO_DATE_RE = re.compile(r"\[(\d{4})-(\d{2})-(\d{2})\]")
|
|
|
|
_VTT_CUE_RE = re.compile(r"(\d{2}):(\d{2}):(\d{2})[.,]\d+\s*-->\s*(\d{2}):(\d{2}):(\d{2})")
|
|
|
|
|
|
def _clean_text(raw: str) -> str:
|
|
text = re.sub(r"<[^>]+>", " ", raw)
|
|
return html.unescape(re.sub(r"\s+", " ", text)).strip()
|
|
|
|
|
|
def _resolve_duration(iframe_url: str, *, timeout: float, attempts: int = 3) -> int | None:
|
|
"""Długość z `thumbnail.vtt` playera (ostatni cue). Patrz docstring modułu:
|
|
duration nie ma w HTML, a NULL ukrywa sceny pod filtrem `min_duration_sec`.
|
|
|
|
API playera throttluje serie zapytań (przy ciągłym crawlu ~40% wywołań wracało
|
|
puste), więc retry z rosnącym backoffem. Każdy błąd → None (scena i tak wejdzie,
|
|
po prostu bez długości)."""
|
|
from app.extractors.hosters.seekplayer_engine import _decrypt, matches
|
|
|
|
p = urlparse(iframe_url)
|
|
if not p.hostname or not matches(iframe_url):
|
|
return None
|
|
hash_id = p.fragment.strip()
|
|
if not hash_id:
|
|
return None
|
|
host = f"{p.scheme}://{p.hostname}"
|
|
headers = {"User-Agent": _DEFAULT_UA, "Accept": "*/*", "Referer": host + "/"}
|
|
|
|
for attempt in range(attempts):
|
|
if attempt:
|
|
time.sleep(1.5 * attempt)
|
|
try:
|
|
r = browser_get(
|
|
f"{host}/api/v1/video?id={hash_id}&w=1920&h=1080&r=",
|
|
headers=headers,
|
|
timeout=timeout,
|
|
)
|
|
if r.status_code != 200 or not r.text:
|
|
continue
|
|
data = json.loads(_decrypt(r.text))
|
|
thumb = (data.get("thumbnail") or "").strip()
|
|
if not thumb:
|
|
return None # payload OK, ale brak miniatur, retry nie pomoże
|
|
vtt_url = host + thumb if thumb.startswith("/") else thumb
|
|
vr = browser_get(vtt_url, headers=headers, timeout=timeout)
|
|
if vr.status_code != 200:
|
|
continue
|
|
cues = _VTT_CUE_RE.findall(vr.text)
|
|
if not cues:
|
|
return None
|
|
h, m, s = cues[-1][3], cues[-1][4], cues[-1][5]
|
|
return (int(h) * 3600 + int(m) * 60 + int(s)) or None
|
|
except Exception as e: # pragma: no cover - best-effort
|
|
log.info("galaxyporn: duration attempt %d failed (%s): %s", attempt + 1, iframe_url[:50], e)
|
|
return None
|
|
|
|
|
|
class GalaxyPornScraper(BaseBrowseScraper):
|
|
sitetag = "galaxyporncom"
|
|
|
|
def _listing_url(self, page: int) -> str:
|
|
return f"{_BASE}/" if page <= 1 else f"{_BASE}/page/{page}/"
|
|
|
|
def _extract_scene_urls(self, listing_html: str) -> list[str]:
|
|
seen: set[str] = set()
|
|
out: list[str] = []
|
|
for m in _SCENE_URL_RE.finditer(listing_html):
|
|
url = m.group(1)
|
|
if url in seen or url.rstrip("/") == _BASE:
|
|
continue
|
|
seen.add(url)
|
|
out.append(url)
|
|
return out
|
|
|
|
def _parse_detail(self, scene_url: str, detail_html: str) -> RawScene | None:
|
|
h1 = _H1_RE.search(detail_html)
|
|
title = _clean_text(h1.group(1)) if h1 else ""
|
|
if not title:
|
|
title = (meta_content(detail_html, property="og:title") or "").strip()
|
|
if not title:
|
|
return None
|
|
|
|
# Obsada, blok wydzielony, bez scopingu na siłę (potwierdzone: cała strona
|
|
# ma dokładnie tyle linków /actor/ ile realnych aktorów).
|
|
performers: list[RawPerformer] = []
|
|
seen_p: set[str] = set()
|
|
ab = _ACTORS_BLOCK_RE.search(detail_html)
|
|
if ab:
|
|
for m in _ANCHOR_TEXT_RE.finditer(ab.group(1)):
|
|
name = html.unescape(m.group(1)).strip()
|
|
sl = slugify(name)
|
|
if not sl or sl in seen_p:
|
|
continue
|
|
seen_p.add(sl)
|
|
performers.append(
|
|
RawPerformer(external_id=f"{self.sitetag}:performer:{sl}", name=name)
|
|
)
|
|
|
|
# Studio + data z prefiksu tytułu. `Studio YY MM DD …` albo `[Studio] …`.
|
|
studio: RawStudio | None = None
|
|
release_date: date | None = None
|
|
studio_name: str | None = None
|
|
if (m := _TITLE_STUDIO_DATE_RE.match(title)):
|
|
studio_name = m.group("studio")
|
|
try:
|
|
release_date = date(2000 + int(m.group("y")), int(m.group("m")), int(m.group("d")))
|
|
except ValueError:
|
|
release_date = None
|
|
elif (m := _TITLE_BRACKET_RE.match(title)):
|
|
studio_name = m.group("studio")
|
|
if release_date is None and (m := _TITLE_ISO_DATE_RE.search(title)):
|
|
try:
|
|
release_date = date(int(m.group(1)), int(m.group(2)), int(m.group(3)))
|
|
except ValueError:
|
|
release_date = None
|
|
if release_date is None and (m := _JSONLD_DATE_RE.search(detail_html)):
|
|
try:
|
|
release_date = date(int(m.group(1)), int(m.group(2)), int(m.group(3)))
|
|
except ValueError:
|
|
release_date = None
|
|
# Guard: prefiks nie może być nazwiskiem performera (wtedy to nie studio).
|
|
if studio_name and slugify(studio_name) not in seen_p:
|
|
studio = RawStudio(
|
|
external_id=f"{self.sitetag}:studio:{slugify(studio_name)}",
|
|
name=studio_name,
|
|
slug=slugify(studio_name),
|
|
)
|
|
|
|
# Tagi, bez slugu studia (retrotube wrzuca studio także jako zwykły tag).
|
|
tags: list[RawTag] = []
|
|
seen_t: set[str] = set()
|
|
studio_slug = slugify(studio_name) if studio_name else ""
|
|
tb = _TAGS_BLOCK_RE.search(detail_html)
|
|
if tb:
|
|
for m in _ANCHOR_TEXT_RE.finditer(tb.group(1)):
|
|
name = html.unescape(m.group(1)).strip()
|
|
sl = slugify(name)
|
|
if not sl or sl in seen_t or sl == studio_slug or sl in seen_p:
|
|
continue
|
|
seen_t.add(sl)
|
|
tags.append(RawTag(external_id=f"{self.sitetag}:tag:{sl}", name=name, slug=sl))
|
|
|
|
thumbnail_url = meta_content(detail_html, property="og:image")
|
|
if not thumbnail_url and (tm := _THUMB_RE.search(detail_html)):
|
|
thumbnail_url = tm.group(1)
|
|
|
|
duration_sec: int | None = None
|
|
if (fm := _IFRAME_RE.search(detail_html)):
|
|
duration_sec = _resolve_duration(fm.group(1).strip(), timeout=self._timeout)
|
|
|
|
return RawScene(
|
|
external_id=f"{self.sitetag}:{scene_url}",
|
|
title=title,
|
|
release_date=release_date,
|
|
duration_sec=duration_sec,
|
|
url=scene_url,
|
|
studio=studio,
|
|
performers=performers,
|
|
tags=tags,
|
|
playback_sources=[
|
|
RawPlaybackSource(
|
|
origin=f"tube:{self.sitetag}",
|
|
page_url=scene_url,
|
|
duration_sec=duration_sec,
|
|
thumbnail_url=thumbnail_url,
|
|
)
|
|
],
|
|
)
|