diff --git a/app/api/playback.py b/app/api/playback.py index 1d3cb08..948e9ba 100644 --- a/app/api/playback.py +++ b/app/api/playback.py @@ -113,6 +113,11 @@ _IP_BOUND_CDN_RE = re.compile( ) +#: Hostery filmow rozwiazywane wylacznie na telefonie (mobile/src/lib/packerHoster.ts). +#: sextapes.to (xtapes.la, 2026-10-15): HLS na cdn-centaurus zwiazany z IP rozwiazujacego. +_PHONE_RESOLVE_HOSTER_RE = re.compile(r"//(?:[a-z0-9-]+\.)*sextapes\.to/", re.IGNORECASE) + + class StreamLink(BaseModel): """Pojedynczy variant stream URL (różne quality / kontener). @@ -223,7 +228,13 @@ def resolve_movie_playback( target = pb.embed_url or pb.page_url stream: str | None = None try: - stream = extract_stream_from_hoster(target, referer=referer) + # Hostery rozwiazywane na telefonie: link z VPS bylby zwiazany z IP serwera (403 na + # telefonie, potem cale wideo przez proxy). Telefon dostaje sam embed. + stream = ( + None + if _PHONE_RESOLVE_HOSTER_RE.search(target) + else extract_stream_from_hoster(target, referer=referer) + ) except HosterDead as e: # Hoster wprost mówi "video deleted", oznacz dead, NIE proponuj # embed fallback (mobile ExoPlayer dostałby 404 HTML page i diff --git a/app/connectors/__init__.py b/app/connectors/__init__.py index 55799eb..cd68a3e 100644 --- a/app/connectors/__init__.py +++ b/app/connectors/__init__.py @@ -39,6 +39,7 @@ def get_movie_connectors() -> list[tuple[str, type]]: ) from app.connectors.freeomovie import FreeoMovieConnector from app.connectors.paradisehill import ParadisehillConnector + from app.connectors.xtapes import XtapesConnector from app.connectors.yourdailypornmovies import YourDailyPornMoviesConnector # Kolejność ingestu: paradisehill FIRST (canonical primary, mirrory się do @@ -76,4 +77,7 @@ def get_movie_connectors() -> list[tuple[str, type]]: # id produktu Adult Empire. Bez długości. Playback: voe + mixdrop (oba mają # gotowe moduły hosterów), vidara.to nieznana, leci jako zwykły embed. ("yourdailypornmovies", YourDailyPornMoviesConnector), + # xtapes.la (WordPress, REST API), kategoria Full Movies: 3-5 filmow dziennie, + # czesto w dniu premiery. Ocena 2026-10-15, szczegoly w app/connectors/xtapes.py. + ("xtapes", XtapesConnector), ] diff --git a/app/connectors/xtapes.py b/app/connectors/xtapes.py new file mode 100644 index 0000000..17c2385 --- /dev/null +++ b/app/connectors/xtapes.py @@ -0,0 +1,225 @@ +"""xtapes.la, konektor filmow (kategoria "Full Movies"). Dodany 2026-10-15. + +WordPress z otwartym REST API, wiec liste bierzemy z `/wp-json/wp/v2/posts` z filtrem +kategorii, a nie z HTML-a. Ocena przed dodaniem (2026-10-15): 1846 filmow, 3 do 5 nowych +dziennie, czesto w dniu premiery; z 30 najnowszych 10 nie mielismy wcale, reszte z +yourdailypornmovies/paradisehill/mangoporn (xtapes dokleja im kolejny link). + +Metadane z REST (`_embed=wp:term`): +- tytul "TYTUL (RRRR) - Full Movie" -> tytul i rok, +- studio to kategoria w nawiasie kwadratowym ("[EvilAngel]", "[Brazzers Movies]"), +- pozostale kategorie to gatunki (bez "Full Movies"), +- obsada to tagi (`post_tag`), +- opis z `content` (czesto pusty). +Ze strony filmu: odtwarzacze (iframe), dlugosc (`itemprop="duration"`), okladka (og:image). + +Odtwarzacze: sextapes.to (P.A.C.K.E.R. + JWPlayer, HLS typu StreamWish) i maxcloud.fit. +maxcloud to posrednik (lista serwerow w base64, za nia kolejny serwis z blokada devtools), +wiec go pomijamy. Link sextapes jest zwiazany z IP rozwiazujacego (z VPS 403 z innego IP), +dlatego rozwiazuje go telefon (packerHoster.ts), a backend go nie ekstrahuje (playback.py). + +Dostep: VPS dostaje od Cloudflare 403 na xtapes.la (REST i strony), przez Bright Data 200. +Proxy tylko do HTML/JSON, nie do wideo. sextapes.to z VPS odpowiada bez proxy. +""" +from __future__ import annotations + +import html as html_mod +import json +import logging +import re +from collections.abc import Iterator +from datetime import datetime +from urllib.parse import urlparse + +from app.config import get_settings +from app.connectors.base import ( + BaseMovieConnector, + RawMovie, + RawPerformer, + RawPlaybackSource, + RawStudio, + RawTag, +) +from app.extractors import browser_get +from app.models.source import SourceKind +from app.normalize.text import slugify + +log = logging.getLogger(__name__) + +_BASE = "https://xtapes.la" +#: Id kategorii "Full Movies" (sprawdzone 2026-10-15; slugi kategorii sa zaciemnione liczbami). +_MOVIES_CATEGORY = 1766 +_PER_PAGE = 30 + +_TITLE_RE = re.compile( + r"^(?P.+?)\s*\((?P<year>(?:19|20)\d{2})\)\s*(?:[-–—]\s*Full\s+Movie)?\s*$", + re.IGNORECASE, +) +_FULL_MOVIE_TAIL_RE = re.compile(r"\s*[-–—]\s*Full\s+Movie\s*$", re.IGNORECASE) +_STUDIO_CAT_RE = re.compile(r"^\[(?P<name>.+)\]$") +_IFRAME_RE = re.compile(r"""<iframe[^>]+src=["']((?:https?:)?//[^"']+)["']""", re.IGNORECASE) +_DURATION_RE = re.compile(r'itemprop="duration"\s+content="P(?:\d+D)?T?(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?"') +_OG_IMAGE_RE = re.compile(r"""property=["']og:image["']\s+content=["']([^"']+)""", re.IGNORECASE) +_TAGS_RE = re.compile(r"<[^>]+>") + +#: Odtwarzacze, ktorych nie zapisujemy: posrednik bez wlasnego wideo i reklamy. +_SKIP_HOSTS = ("maxcloud", "realsrv", "magsrv", "exosrv", "juicyads", "google", "doubleclick") + + +def _host_label(url: str) -> str | None: + host = (urlparse(url).hostname or "").lower().removeprefix("www.") + if not host: + return None + parts = host.split(".") + return parts[-2] if len(parts) >= 2 else parts[0] + + +def parse_title(raw: str) -> tuple[str, int | None]: + """'Tushy Raw V87 (2026) - Full Movie' -> ('Tushy Raw V87', 2026).""" + t = html_mod.unescape(raw).strip() + m = _TITLE_RE.match(t) + if m: + return m.group("title").strip(), int(m.group("year")) + return _FULL_MOVIE_TAIL_RE.sub("", t).strip(), None + + +def parse_duration(page: str) -> int | None: + m = _DURATION_RE.search(page) + if not m or not any(m.groups()): + return None + h, mi, s = (int(x) if x else 0 for x in m.groups()) + total = h * 3600 + mi * 60 + s + return total or None + + +class XtapesConnector(BaseMovieConnector): + kind = SourceKind.scraper + name = "xtapes" + base_url = _BASE + + _MAX_PAGES_DELTA = 3 + _MAX_PAGES_FULL = 20 + + def __init__(self, *, timeout: float = 30.0) -> None: + self._timeout = timeout + self._proxy = get_settings().brightdata_proxy_url + + def close(self) -> None: + pass + + def _get(self, url: str): + """xtapes.la od razu przez proxy (Cloudflare daje VPS-owi 403), bez proxy tylko gdy + go nie skonfigurowano (lokalnie z domowego IP strona odpowiada wprost).""" + r = browser_get(url, timeout=self._timeout, follow_redirects=True, proxy=self._proxy) + if r.status_code >= 400: + raise RuntimeError(f"{r.status_code} for {url}") + return r + + def fetch_movies( + self, *, since: datetime | None = None, limit: int | None = None + ) -> Iterator[RawMovie]: + seen = 0 + max_pages = self._MAX_PAGES_DELTA if since is not None else self._MAX_PAGES_FULL + for page in range(1, max_pages + 1): + url = ( + f"{_BASE}/wp-json/wp/v2/posts?categories={_MOVIES_CATEGORY}" + f"&per_page={_PER_PAGE}&page={page}&_embed=wp:term" + "&_fields=id,date,slug,link,title,content,_links,_embedded" + ) + if since is not None: + # Tylko nowsze od ostatniego udanego przebiegu: strona filmu idzie przez proxy + # (~9 s), a 3 pelne strony REST to 90 filmow, ponad limit 6 min na konektor. + url += f"&after={since.strftime('%Y-%m-%dT%H:%M:%S')}" + try: + posts = json.loads(self._get(url).text) + except Exception as e: + # Za ostatnia strona WP oddaje 400 (rest_post_invalid_page_number). + log.info("%s REST page=%d: %s", self.name, page, e) + return + if not isinstance(posts, list) or not posts: + return + for post in posts: + try: + movie = self._movie_from_post(post) + except Exception as e: + log.warning("%s post %s failed: %s", self.name, post.get("link"), e) + continue + if movie is None: + continue + yield movie + seen += 1 + if limit is not None and seen >= limit: + return + + def _movie_from_post(self, post: dict) -> RawMovie | None: + link = (post.get("link") or "").strip() + title, year = parse_title((post.get("title") or {}).get("rendered", "")) + if not link or not title: + return None + + studio: RawStudio | None = None + tags: list[RawTag] = [] + performers: list[RawPerformer] = [] + for group in (post.get("_embedded") or {}).get("wp:term") or []: + for term in group or []: + name = html_mod.unescape(term.get("name") or "").strip() + tax = term.get("taxonomy") + if not name: + continue + if tax == "category": + sm = _STUDIO_CAT_RE.match(name) + if sm: + sname = sm.group("name").strip() + if studio is None and slugify(sname): + studio = RawStudio( + external_id=f"{self.name}:studio:{slugify(sname)}", + name=sname, + slug=slugify(sname), + ) + elif name.lower() != "full movies" and slugify(name): + tags.append( + RawTag(external_id=f"{self.name}:tag:{slugify(name)}", name=name, slug=slugify(name)) + ) + elif tax == "post_tag" and slugify(name): + performers.append( + RawPerformer(external_id=f"{self.name}:performer:{slugify(name)}", name=name) + ) + + page = self._get(link).text + + # Playback najpierw: bez odtwarzacza film jest bezuzyteczny, nie tworzymy sieroty. + playback: list[RawPlaybackSource] = [] + seen_host: set[str] = set() + for m in _IFRAME_RE.finditer(page): + embed = m.group(1) + if embed.startswith("//"): + embed = "https:" + embed + host = _host_label(embed) + if not host or host in seen_host or any(s in embed for s in _SKIP_HOSTS): + continue + seen_host.add(host) + playback.append( + RawPlaybackSource(origin=f"{self.name}:{host}", page_url=embed, embed_url=embed) + ) + if not playback: + return None + + desc = _TAGS_RE.sub(" ", (post.get("content") or {}).get("rendered") or "") + desc = re.sub(r"\s+", " ", html_mod.unescape(desc)).strip() or None + om = _OG_IMAGE_RE.search(page) + poster = html_mod.unescape(om.group(1)).strip() if om else None + + return RawMovie( + external_id=post.get("slug") or urlparse(link).path.strip("/").split("/")[-1], + title=title, + description=desc, + release_year=year, + duration_sec=parse_duration(page), + url=link, + poster_url=poster, + studio=studio, + performers=performers, + tags=tags, + playback_sources=playback, + raw={"source": self.name, "url": link, "wp_id": post.get("id")}, + ) diff --git a/scripts/ingest_xtapes_backfill.py b/scripts/ingest_xtapes_backfill.py new file mode 100644 index 0000000..503ccbe --- /dev/null +++ b/scripts/ingest_xtapes_backfill.py @@ -0,0 +1,14 @@ +"""Pierwsze zaciagniecie filmow z xtapes.la (2026-10-15): pelny tryb, do 20 stron REST. + +Dalej filmy dochodza codziennie przez _job_movie_ingest (tryb przyrostowy, after=). + docker compose run -d --rm worker python3 /srv/scripts/ingest_xtapes_backfill.py [limit] +""" +import logging +import sys + +from app.connectors.xtapes import XtapesConnector +from app.ingest import ingest_movies_from_connector + +logging.basicConfig(level=logging.INFO) +limit = int(sys.argv[1]) if len(sys.argv) > 1 else None +print(ingest_movies_from_connector(XtapesConnector(), limit=limit, use_delta=False))