diff --git a/app/connectors/__init__.py b/app/connectors/__init__.py index 6598300..a4494a3 100644 --- a/app/connectors/__init__.py +++ b/app/connectors/__init__.py @@ -38,6 +38,7 @@ def get_movie_connectors() -> list[tuple[str, type]]: PandamoviesConnector, StreampornVipConnector, ) + from app.connectors.freeomovie import FreeoMovieConnector from app.connectors.paradisehill import ParadisehillConnector # Kolejność ingestu: paradisehill FIRST (canonical primary, mirrory się do @@ -54,4 +55,7 @@ def get_movie_connectors() -> list[tuple[str, type]]: ("mangoporn", MangopornConnector), ("streampornvip", StreampornVipConnector), ("pandamovies", PandamoviesConnector), + # freeomovie.to (bestia theme, nie dooplay) — mirror: świeże filmy DVD, + # title-trigram attach do canonical + playback z TABS. Ocena 2026-07-02. + ("freeomovie", FreeoMovieConnector), ] diff --git a/app/connectors/freeomovie.py b/app/connectors/freeomovie.py new file mode 100644 index 0000000..7bdf6fa --- /dev/null +++ b/app/connectors/freeomovie.py @@ -0,0 +1,188 @@ +"""freeomovie.to — movie source (WordPress "bestia" theme, NIE dooplay). + +Dodany 2026-07-02 (ocena). Pełnometrażowe filmy, świeże (auto-poster, dziś), tytuły +DVD → title-trigram mirror-attach do canonical (paradisehill/TPDB movies), dokładając +playback_sources. Scope: TYLKO `/category/full-movie/` (homepage miesza scen-klipy, +które orphanowałyby jako filmy). + +Struktura: + - listing `
  • ` → URL filmu + czysty tytuł + - detail: `var TABS=[{label,url},...]` (hostery), `"articleSection"` (gatunki + obsada; + gatunki PRZED "XXX Movies" → tagi), `"datePublished"` (data POSTA, NIE rok produkcji) + +Świadomie NIE bierzemy: + - release_year: datePublished to data uploadu (2026), nie rok filmu → fałszywy rok + psułby year-scoring przy matchu do canonical (film z 2010 dostałby 2026). Lepiej None. + - performerów/studia z articleSection: format miesza performer/alias/studio bez czystego + delimitera → ryzyko junk-performera ("Deeveeous" studio jako performer). Mirror i tak + doczepia się do canonical po tytule, a TPDB enrichment dokłada obsadę/studio autorytatywnie. + +Playback: TABS hostery → MoviePlaybackSource per host (voe/luluvid/vidhide resolvują się +VPS-side; myvidplay = DoodStream clone, phone-side — wymaga myvidplay.com w DOOD_HOSTS). +Pomijamy streamtape (martwy malware) + mxdrop (flaky). +""" +from __future__ import annotations + +import html +import json +import logging +import re +from collections.abc import Iterator +from datetime import datetime +from urllib.parse import urlparse + +from app.connectors.base import ( + BaseMovieConnector, + RawMovie, + RawPlaybackSource, + RawTag, +) +from app.extractors import browser_get +from app.models.source import SourceKind +from app.normalize.text import slugify + +log = logging.getLogger(__name__) + +_BASE = "https://www.freeomovie.to" +_LIST_ITEM_RE = re.compile( + r']*class="thumi"[^>]*>\s*]+href="(?Phttps://www\.freeomovie\.to/[a-z0-9\-]+/)"[^>]*title="(?P[^"]+)"', + re.IGNORECASE, +) +_TABS_RE = re.compile(r"var\s+TABS\s*=\s*(\[.*?\])\s*;", re.DOTALL) +_ARTICLE_SECTION_RE = re.compile(r'"articleSection"\s*:\s*"([^"]+)"') + +# Hostery pomijane (martwe/flaky). +_SKIP_HOSTS = ("streamtape", "mxdrop", "mixdrop", "streamsb") + + +def _host_label(embed_url: str) -> str | None: + host = (urlparse(embed_url).hostname or "").lower() + if not host: + return None + parts = host.replace("www.", "").split(".") + return parts[0] if parts else None + + +class FreeoMovieConnector(BaseMovieConnector): + kind = SourceKind.scraper + name = "freeomovie" + base_url = _BASE + + _MAX_PAGES_DELTA = 3 + _MAX_PAGES_FULL = 40 + + def __init__(self, *, timeout: float = 30.0) -> None: + self._timeout = timeout + + def close(self) -> None: + pass + + def _fetch(self, url: str) -> str: + if not url.startswith("http"): + url = _BASE + url + r = browser_get( + url, + headers={ + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/125.0", + "Accept": "text/html,application/xhtml+xml", + "Referer": _BASE + "/", + }, + timeout=self._timeout, + follow_redirects=True, + ) + if r.status_code >= 400: + raise RuntimeError(f"{r.status_code} for {url}") + return r.text + + def fetch_movies( + self, *, since: datetime | None = None, limit: int | None = None + ) -> Iterator[RawMovie]: + seen = 0 + seen_urls: set[str] = set() + max_pages = self._MAX_PAGES_DELTA if since is not None else self._MAX_PAGES_FULL + for page in range(1, max_pages + 1): + path = "/category/full-movie/" if page == 1 else f"/category/full-movie/page/{page}/" + try: + listing = self._fetch(path) + except Exception as e: + log.warning("freeomovie listing page=%d failed: %s", page, e) + return + items = [(m.group("url"), html.unescape(m.group("title")).strip()) + for m in _LIST_ITEM_RE.finditer(listing)] + if not items: + log.info("freeomovie: empty page=%d, stop", page) + return + for url, title in items: + if url in seen_urls: + continue + seen_urls.add(url) + try: + movie = self._parse_detail(url, title) + except Exception as e: + log.warning("freeomovie detail %s failed: %s", url, e) + continue + if movie is None: + continue + yield movie + seen += 1 + if limit is not None and seen >= limit: + return + + def _parse_detail(self, url: str, title: str) -> RawMovie | None: + detail = self._fetch(url) + if not title: + return None + + # Tagi: gatunki z articleSection PRZED "XXX Movies" (reszta = obsada/studio, skip). + tags: list[RawTag] = [] + seen_tag: set[str] = set() + sm = _ARTICLE_SECTION_RE.search(detail) + if sm: + for part in sm.group(1).split(","): + name = part.strip() + if not name or name.lower() == "xxx movies": + if name.lower() == "xxx movies": + break # dalej idzie obsada/studio — nie tagi + continue + sl = slugify(name) + if not sl or sl in seen_tag: + continue + seen_tag.add(sl) + tags.append(RawTag(external_id=f"{self.name}:tag:{sl}", name=name, slug=sl)) + + # Playback: TABS array hosterów. + playback: list[RawPlaybackSource] = [] + seen_host: set[str] = set() + tm = _TABS_RE.search(detail) + if tm: + try: + arr = json.loads(tm.group(1)) + except (json.JSONDecodeError, ValueError): + arr = [] + for entry in arr: + embed = (entry.get("url") or "").strip() if isinstance(entry, dict) else "" + if not embed.startswith("http"): + continue + host = _host_label(embed) + if not host or host in _SKIP_HOSTS or host in seen_host: + continue + seen_host.add(host) + playback.append( + RawPlaybackSource( + origin=f"{self.name}:{host}", + page_url=embed, + embed_url=embed, + ) + ) + if not playback: + return None # bez playbacku film jest bezużyteczny (nie tworzymy orphana) + + slug = urlparse(url).path.strip("/").split("/")[-1] + return RawMovie( + external_id=slug, + title=title, + url=url, + tags=tags, + playback_sources=playback, + raw={"source": "freeomovie", "url": url}, + )