feat(movies): konektor xtapes.la (Full Movies przez REST API WordPressa)
Lista z /wp-json/wp/v2/posts (kategoria Full Movies, _embed=wp:term): tytul z rokiem, studio z kategorii w nawiasie, gatunki, obsada z tagow. Ze strony filmu odtwarzacze, dlugosc i okladka. Przez proxy, bo Cloudflare daje VPS-owi 403. Odtwarzacz sextapes.to rozwiazuje telefon (link zwiazany z IP), backend go nie ekstrahuje; maxcloud (posrednik) pomijany. Tryb przyrostowy po after=. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
goon-foss
parent
763d38392c
commit
1555408e20
4 changed files with 255 additions and 1 deletions
|
|
@ -113,6 +113,11 @@ _IP_BOUND_CDN_RE = re.compile(
|
|||
)
|
||||
|
||||
|
||||
#: Hostery filmow rozwiazywane wylacznie na telefonie (mobile/src/lib/packerHoster.ts).
|
||||
#: sextapes.to (xtapes.la, 2026-10-15): HLS na cdn-centaurus zwiazany z IP rozwiazujacego.
|
||||
_PHONE_RESOLVE_HOSTER_RE = re.compile(r"//(?:[a-z0-9-]+\.)*sextapes\.to/", re.IGNORECASE)
|
||||
|
||||
|
||||
class StreamLink(BaseModel):
|
||||
"""Pojedynczy variant stream URL (różne quality / kontener).
|
||||
|
||||
|
|
@ -223,7 +228,13 @@ def resolve_movie_playback(
|
|||
target = pb.embed_url or pb.page_url
|
||||
stream: str | None = None
|
||||
try:
|
||||
stream = extract_stream_from_hoster(target, referer=referer)
|
||||
# Hostery rozwiazywane na telefonie: link z VPS bylby zwiazany z IP serwera (403 na
|
||||
# telefonie, potem cale wideo przez proxy). Telefon dostaje sam embed.
|
||||
stream = (
|
||||
None
|
||||
if _PHONE_RESOLVE_HOSTER_RE.search(target)
|
||||
else extract_stream_from_hoster(target, referer=referer)
|
||||
)
|
||||
except HosterDead as e:
|
||||
# Hoster wprost mówi "video deleted", oznacz dead, NIE proponuj
|
||||
# embed fallback (mobile ExoPlayer dostałby 404 HTML page i
|
||||
|
|
|
|||
|
|
@ -39,6 +39,7 @@ def get_movie_connectors() -> list[tuple[str, type]]:
|
|||
)
|
||||
from app.connectors.freeomovie import FreeoMovieConnector
|
||||
from app.connectors.paradisehill import ParadisehillConnector
|
||||
from app.connectors.xtapes import XtapesConnector
|
||||
from app.connectors.yourdailypornmovies import YourDailyPornMoviesConnector
|
||||
|
||||
# Kolejność ingestu: paradisehill FIRST (canonical primary, mirrory się do
|
||||
|
|
@ -76,4 +77,7 @@ def get_movie_connectors() -> list[tuple[str, type]]:
|
|||
# id produktu Adult Empire. Bez długości. Playback: voe + mixdrop (oba mają
|
||||
# gotowe moduły hosterów), vidara.to nieznana, leci jako zwykły embed.
|
||||
("yourdailypornmovies", YourDailyPornMoviesConnector),
|
||||
# xtapes.la (WordPress, REST API), kategoria Full Movies: 3-5 filmow dziennie,
|
||||
# czesto w dniu premiery. Ocena 2026-10-15, szczegoly w app/connectors/xtapes.py.
|
||||
("xtapes", XtapesConnector),
|
||||
]
|
||||
|
|
|
|||
225
app/connectors/xtapes.py
Normal file
225
app/connectors/xtapes.py
Normal file
|
|
@ -0,0 +1,225 @@
|
|||
"""xtapes.la, konektor filmow (kategoria "Full Movies"). Dodany 2026-10-15.
|
||||
|
||||
WordPress z otwartym REST API, wiec liste bierzemy z `/wp-json/wp/v2/posts` z filtrem
|
||||
kategorii, a nie z HTML-a. Ocena przed dodaniem (2026-10-15): 1846 filmow, 3 do 5 nowych
|
||||
dziennie, czesto w dniu premiery; z 30 najnowszych 10 nie mielismy wcale, reszte z
|
||||
yourdailypornmovies/paradisehill/mangoporn (xtapes dokleja im kolejny link).
|
||||
|
||||
Metadane z REST (`_embed=wp:term`):
|
||||
- tytul "TYTUL (RRRR) - Full Movie" -> tytul i rok,
|
||||
- studio to kategoria w nawiasie kwadratowym ("[EvilAngel]", "[Brazzers Movies]"),
|
||||
- pozostale kategorie to gatunki (bez "Full Movies"),
|
||||
- obsada to tagi (`post_tag`),
|
||||
- opis z `content` (czesto pusty).
|
||||
Ze strony filmu: odtwarzacze (iframe), dlugosc (`itemprop="duration"`), okladka (og:image).
|
||||
|
||||
Odtwarzacze: sextapes.to (P.A.C.K.E.R. + JWPlayer, HLS typu StreamWish) i maxcloud.fit.
|
||||
maxcloud to posrednik (lista serwerow w base64, za nia kolejny serwis z blokada devtools),
|
||||
wiec go pomijamy. Link sextapes jest zwiazany z IP rozwiazujacego (z VPS 403 z innego IP),
|
||||
dlatego rozwiazuje go telefon (packerHoster.ts), a backend go nie ekstrahuje (playback.py).
|
||||
|
||||
Dostep: VPS dostaje od Cloudflare 403 na xtapes.la (REST i strony), przez Bright Data 200.
|
||||
Proxy tylko do HTML/JSON, nie do wideo. sextapes.to z VPS odpowiada bez proxy.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import html as html_mod
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from collections.abc import Iterator
|
||||
from datetime import datetime
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from app.config import get_settings
|
||||
from app.connectors.base import (
|
||||
BaseMovieConnector,
|
||||
RawMovie,
|
||||
RawPerformer,
|
||||
RawPlaybackSource,
|
||||
RawStudio,
|
||||
RawTag,
|
||||
)
|
||||
from app.extractors import browser_get
|
||||
from app.models.source import SourceKind
|
||||
from app.normalize.text import slugify
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
_BASE = "https://xtapes.la"
|
||||
#: Id kategorii "Full Movies" (sprawdzone 2026-10-15; slugi kategorii sa zaciemnione liczbami).
|
||||
_MOVIES_CATEGORY = 1766
|
||||
_PER_PAGE = 30
|
||||
|
||||
_TITLE_RE = re.compile(
|
||||
r"^(?P<title>.+?)\s*\((?P<year>(?:19|20)\d{2})\)\s*(?:[-–—]\s*Full\s+Movie)?\s*$",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_FULL_MOVIE_TAIL_RE = re.compile(r"\s*[-–—]\s*Full\s+Movie\s*$", re.IGNORECASE)
|
||||
_STUDIO_CAT_RE = re.compile(r"^\[(?P<name>.+)\]$")
|
||||
_IFRAME_RE = re.compile(r"""<iframe[^>]+src=["']((?:https?:)?//[^"']+)["']""", re.IGNORECASE)
|
||||
_DURATION_RE = re.compile(r'itemprop="duration"\s+content="P(?:\d+D)?T?(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?"')
|
||||
_OG_IMAGE_RE = re.compile(r"""property=["']og:image["']\s+content=["']([^"']+)""", re.IGNORECASE)
|
||||
_TAGS_RE = re.compile(r"<[^>]+>")
|
||||
|
||||
#: Odtwarzacze, ktorych nie zapisujemy: posrednik bez wlasnego wideo i reklamy.
|
||||
_SKIP_HOSTS = ("maxcloud", "realsrv", "magsrv", "exosrv", "juicyads", "google", "doubleclick")
|
||||
|
||||
|
||||
def _host_label(url: str) -> str | None:
|
||||
host = (urlparse(url).hostname or "").lower().removeprefix("www.")
|
||||
if not host:
|
||||
return None
|
||||
parts = host.split(".")
|
||||
return parts[-2] if len(parts) >= 2 else parts[0]
|
||||
|
||||
|
||||
def parse_title(raw: str) -> tuple[str, int | None]:
|
||||
"""'Tushy Raw V87 (2026) - Full Movie' -> ('Tushy Raw V87', 2026)."""
|
||||
t = html_mod.unescape(raw).strip()
|
||||
m = _TITLE_RE.match(t)
|
||||
if m:
|
||||
return m.group("title").strip(), int(m.group("year"))
|
||||
return _FULL_MOVIE_TAIL_RE.sub("", t).strip(), None
|
||||
|
||||
|
||||
def parse_duration(page: str) -> int | None:
|
||||
m = _DURATION_RE.search(page)
|
||||
if not m or not any(m.groups()):
|
||||
return None
|
||||
h, mi, s = (int(x) if x else 0 for x in m.groups())
|
||||
total = h * 3600 + mi * 60 + s
|
||||
return total or None
|
||||
|
||||
|
||||
class XtapesConnector(BaseMovieConnector):
|
||||
kind = SourceKind.scraper
|
||||
name = "xtapes"
|
||||
base_url = _BASE
|
||||
|
||||
_MAX_PAGES_DELTA = 3
|
||||
_MAX_PAGES_FULL = 20
|
||||
|
||||
def __init__(self, *, timeout: float = 30.0) -> None:
|
||||
self._timeout = timeout
|
||||
self._proxy = get_settings().brightdata_proxy_url
|
||||
|
||||
def close(self) -> None:
|
||||
pass
|
||||
|
||||
def _get(self, url: str):
|
||||
"""xtapes.la od razu przez proxy (Cloudflare daje VPS-owi 403), bez proxy tylko gdy
|
||||
go nie skonfigurowano (lokalnie z domowego IP strona odpowiada wprost)."""
|
||||
r = browser_get(url, timeout=self._timeout, follow_redirects=True, proxy=self._proxy)
|
||||
if r.status_code >= 400:
|
||||
raise RuntimeError(f"{r.status_code} for {url}")
|
||||
return r
|
||||
|
||||
def fetch_movies(
|
||||
self, *, since: datetime | None = None, limit: int | None = None
|
||||
) -> Iterator[RawMovie]:
|
||||
seen = 0
|
||||
max_pages = self._MAX_PAGES_DELTA if since is not None else self._MAX_PAGES_FULL
|
||||
for page in range(1, max_pages + 1):
|
||||
url = (
|
||||
f"{_BASE}/wp-json/wp/v2/posts?categories={_MOVIES_CATEGORY}"
|
||||
f"&per_page={_PER_PAGE}&page={page}&_embed=wp:term"
|
||||
"&_fields=id,date,slug,link,title,content,_links,_embedded"
|
||||
)
|
||||
if since is not None:
|
||||
# Tylko nowsze od ostatniego udanego przebiegu: strona filmu idzie przez proxy
|
||||
# (~9 s), a 3 pelne strony REST to 90 filmow, ponad limit 6 min na konektor.
|
||||
url += f"&after={since.strftime('%Y-%m-%dT%H:%M:%S')}"
|
||||
try:
|
||||
posts = json.loads(self._get(url).text)
|
||||
except Exception as e:
|
||||
# Za ostatnia strona WP oddaje 400 (rest_post_invalid_page_number).
|
||||
log.info("%s REST page=%d: %s", self.name, page, e)
|
||||
return
|
||||
if not isinstance(posts, list) or not posts:
|
||||
return
|
||||
for post in posts:
|
||||
try:
|
||||
movie = self._movie_from_post(post)
|
||||
except Exception as e:
|
||||
log.warning("%s post %s failed: %s", self.name, post.get("link"), e)
|
||||
continue
|
||||
if movie is None:
|
||||
continue
|
||||
yield movie
|
||||
seen += 1
|
||||
if limit is not None and seen >= limit:
|
||||
return
|
||||
|
||||
def _movie_from_post(self, post: dict) -> RawMovie | None:
|
||||
link = (post.get("link") or "").strip()
|
||||
title, year = parse_title((post.get("title") or {}).get("rendered", ""))
|
||||
if not link or not title:
|
||||
return None
|
||||
|
||||
studio: RawStudio | None = None
|
||||
tags: list[RawTag] = []
|
||||
performers: list[RawPerformer] = []
|
||||
for group in (post.get("_embedded") or {}).get("wp:term") or []:
|
||||
for term in group or []:
|
||||
name = html_mod.unescape(term.get("name") or "").strip()
|
||||
tax = term.get("taxonomy")
|
||||
if not name:
|
||||
continue
|
||||
if tax == "category":
|
||||
sm = _STUDIO_CAT_RE.match(name)
|
||||
if sm:
|
||||
sname = sm.group("name").strip()
|
||||
if studio is None and slugify(sname):
|
||||
studio = RawStudio(
|
||||
external_id=f"{self.name}:studio:{slugify(sname)}",
|
||||
name=sname,
|
||||
slug=slugify(sname),
|
||||
)
|
||||
elif name.lower() != "full movies" and slugify(name):
|
||||
tags.append(
|
||||
RawTag(external_id=f"{self.name}:tag:{slugify(name)}", name=name, slug=slugify(name))
|
||||
)
|
||||
elif tax == "post_tag" and slugify(name):
|
||||
performers.append(
|
||||
RawPerformer(external_id=f"{self.name}:performer:{slugify(name)}", name=name)
|
||||
)
|
||||
|
||||
page = self._get(link).text
|
||||
|
||||
# Playback najpierw: bez odtwarzacza film jest bezuzyteczny, nie tworzymy sieroty.
|
||||
playback: list[RawPlaybackSource] = []
|
||||
seen_host: set[str] = set()
|
||||
for m in _IFRAME_RE.finditer(page):
|
||||
embed = m.group(1)
|
||||
if embed.startswith("//"):
|
||||
embed = "https:" + embed
|
||||
host = _host_label(embed)
|
||||
if not host or host in seen_host or any(s in embed for s in _SKIP_HOSTS):
|
||||
continue
|
||||
seen_host.add(host)
|
||||
playback.append(
|
||||
RawPlaybackSource(origin=f"{self.name}:{host}", page_url=embed, embed_url=embed)
|
||||
)
|
||||
if not playback:
|
||||
return None
|
||||
|
||||
desc = _TAGS_RE.sub(" ", (post.get("content") or {}).get("rendered") or "")
|
||||
desc = re.sub(r"\s+", " ", html_mod.unescape(desc)).strip() or None
|
||||
om = _OG_IMAGE_RE.search(page)
|
||||
poster = html_mod.unescape(om.group(1)).strip() if om else None
|
||||
|
||||
return RawMovie(
|
||||
external_id=post.get("slug") or urlparse(link).path.strip("/").split("/")[-1],
|
||||
title=title,
|
||||
description=desc,
|
||||
release_year=year,
|
||||
duration_sec=parse_duration(page),
|
||||
url=link,
|
||||
poster_url=poster,
|
||||
studio=studio,
|
||||
performers=performers,
|
||||
tags=tags,
|
||||
playback_sources=playback,
|
||||
raw={"source": self.name, "url": link, "wp_id": post.get("id")},
|
||||
)
|
||||
14
scripts/ingest_xtapes_backfill.py
Normal file
14
scripts/ingest_xtapes_backfill.py
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
"""Pierwsze zaciagniecie filmow z xtapes.la (2026-10-15): pelny tryb, do 20 stron REST.
|
||||
|
||||
Dalej filmy dochodza codziennie przez _job_movie_ingest (tryb przyrostowy, after=).
|
||||
docker compose run -d --rm worker python3 /srv/scripts/ingest_xtapes_backfill.py [limit]
|
||||
"""
|
||||
import logging
|
||||
import sys
|
||||
|
||||
from app.connectors.xtapes import XtapesConnector
|
||||
from app.ingest import ingest_movies_from_connector
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
limit = int(sys.argv[1]) if len(sys.argv) > 1 else None
|
||||
print(ingest_movies_from_connector(XtapesConnector(), limit=limit, use_delta=False))
|
||||
Loading…
Add table
Reference in a new issue