feat(movies): add freeomovie.to as a movie mirror
freeomovie.to (WordPress bestia theme, not dooplay) publishes fresh full-length DVD titles daily. New standalone BaseMovieConnector scoped to /category/full-movie/ (homepage mixes scene-clips that would orphan as movies). Parses the listing thumi items for clean titles, and the detail var TABS array for playback hosters (voe/luluvid/vidhide resolve VPS-side; myvidplay is a DoodStream clone, phone-side), skipping dead streamtape/ mxdrop. Genre tags come from articleSection (the part before "XXX Movies"); release_year is deliberately NOT set (datePublished is the post/upload date, not the film's production year, which would poison year-scoring on canonical match) and performers/studio are skipped (articleSection has no clean delimiter -> junk-performer risk) -- the movie rides in as a title-trigram mirror and TPDB enrichment supplies cast/studio/tags authoritatively. Verified pilot: 10 movies -> 9 attached to existing canonical movies (mirror playback), 1 new, 0 errors. Follow-up (mobile, not required for playability since voe/luluvid cover it): add 'myvidplay.com' to DOOD_HOSTS in doodstream.ts + OTA for a 4th hoster. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
goon-foss
parent
015bd5bb7a
commit
393dbe548a
2 changed files with 192 additions and 0 deletions
|
|
@ -38,6 +38,7 @@ def get_movie_connectors() -> list[tuple[str, type]]:
|
|||
PandamoviesConnector,
|
||||
StreampornVipConnector,
|
||||
)
|
||||
from app.connectors.freeomovie import FreeoMovieConnector
|
||||
from app.connectors.paradisehill import ParadisehillConnector
|
||||
|
||||
# Kolejność ingestu: paradisehill FIRST (canonical primary, mirrory się do
|
||||
|
|
@ -54,4 +55,7 @@ def get_movie_connectors() -> list[tuple[str, type]]:
|
|||
("mangoporn", MangopornConnector),
|
||||
("streampornvip", StreampornVipConnector),
|
||||
("pandamovies", PandamoviesConnector),
|
||||
# freeomovie.to (bestia theme, nie dooplay) — mirror: świeże filmy DVD,
|
||||
# title-trigram attach do canonical + playback z TABS. Ocena 2026-07-02.
|
||||
("freeomovie", FreeoMovieConnector),
|
||||
]
|
||||
|
|
|
|||
188
app/connectors/freeomovie.py
Normal file
188
app/connectors/freeomovie.py
Normal file
|
|
@ -0,0 +1,188 @@
|
|||
"""freeomovie.to — movie source (WordPress "bestia" theme, NIE dooplay).
|
||||
|
||||
Dodany 2026-07-02 (ocena). Pełnometrażowe filmy, świeże (auto-poster, dziś), tytuły
|
||||
DVD → title-trigram mirror-attach do canonical (paradisehill/TPDB movies), dokładając
|
||||
playback_sources. Scope: TYLKO `/category/full-movie/` (homepage miesza scen-klipy,
|
||||
które orphanowałyby jako filmy).
|
||||
|
||||
Struktura:
|
||||
- listing `<li class="thumi"><a href title>` → URL filmu + czysty tytuł
|
||||
- detail: `var TABS=[{label,url},...]` (hostery), `"articleSection"` (gatunki + obsada;
|
||||
gatunki PRZED "XXX Movies" → tagi), `"datePublished"` (data POSTA, NIE rok produkcji)
|
||||
|
||||
Świadomie NIE bierzemy:
|
||||
- release_year: datePublished to data uploadu (2026), nie rok filmu → fałszywy rok
|
||||
psułby year-scoring przy matchu do canonical (film z 2010 dostałby 2026). Lepiej None.
|
||||
- performerów/studia z articleSection: format miesza performer/alias/studio bez czystego
|
||||
delimitera → ryzyko junk-performera ("Deeveeous" studio jako performer). Mirror i tak
|
||||
doczepia się do canonical po tytule, a TPDB enrichment dokłada obsadę/studio autorytatywnie.
|
||||
|
||||
Playback: TABS hostery → MoviePlaybackSource per host (voe/luluvid/vidhide resolvują się
|
||||
VPS-side; myvidplay = DoodStream clone, phone-side — wymaga myvidplay.com w DOOD_HOSTS).
|
||||
Pomijamy streamtape (martwy malware) + mxdrop (flaky).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from collections.abc import Iterator
|
||||
from datetime import datetime
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from app.connectors.base import (
|
||||
BaseMovieConnector,
|
||||
RawMovie,
|
||||
RawPlaybackSource,
|
||||
RawTag,
|
||||
)
|
||||
from app.extractors import browser_get
|
||||
from app.models.source import SourceKind
|
||||
from app.normalize.text import slugify
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
_BASE = "https://www.freeomovie.to"
|
||||
_LIST_ITEM_RE = re.compile(
|
||||
r'<li[^>]*class="thumi"[^>]*>\s*<a[^>]+href="(?P<url>https://www\.freeomovie\.to/[a-z0-9\-]+/)"[^>]*title="(?P<title>[^"]+)"',
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_TABS_RE = re.compile(r"var\s+TABS\s*=\s*(\[.*?\])\s*;", re.DOTALL)
|
||||
_ARTICLE_SECTION_RE = re.compile(r'"articleSection"\s*:\s*"([^"]+)"')
|
||||
|
||||
# Hostery pomijane (martwe/flaky).
|
||||
_SKIP_HOSTS = ("streamtape", "mxdrop", "mixdrop", "streamsb")
|
||||
|
||||
|
||||
def _host_label(embed_url: str) -> str | None:
|
||||
host = (urlparse(embed_url).hostname or "").lower()
|
||||
if not host:
|
||||
return None
|
||||
parts = host.replace("www.", "").split(".")
|
||||
return parts[0] if parts else None
|
||||
|
||||
|
||||
class FreeoMovieConnector(BaseMovieConnector):
|
||||
kind = SourceKind.scraper
|
||||
name = "freeomovie"
|
||||
base_url = _BASE
|
||||
|
||||
_MAX_PAGES_DELTA = 3
|
||||
_MAX_PAGES_FULL = 40
|
||||
|
||||
def __init__(self, *, timeout: float = 30.0) -> None:
|
||||
self._timeout = timeout
|
||||
|
||||
def close(self) -> None:
|
||||
pass
|
||||
|
||||
def _fetch(self, url: str) -> str:
|
||||
if not url.startswith("http"):
|
||||
url = _BASE + url
|
||||
r = browser_get(
|
||||
url,
|
||||
headers={
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/125.0",
|
||||
"Accept": "text/html,application/xhtml+xml",
|
||||
"Referer": _BASE + "/",
|
||||
},
|
||||
timeout=self._timeout,
|
||||
follow_redirects=True,
|
||||
)
|
||||
if r.status_code >= 400:
|
||||
raise RuntimeError(f"{r.status_code} for {url}")
|
||||
return r.text
|
||||
|
||||
def fetch_movies(
|
||||
self, *, since: datetime | None = None, limit: int | None = None
|
||||
) -> Iterator[RawMovie]:
|
||||
seen = 0
|
||||
seen_urls: set[str] = set()
|
||||
max_pages = self._MAX_PAGES_DELTA if since is not None else self._MAX_PAGES_FULL
|
||||
for page in range(1, max_pages + 1):
|
||||
path = "/category/full-movie/" if page == 1 else f"/category/full-movie/page/{page}/"
|
||||
try:
|
||||
listing = self._fetch(path)
|
||||
except Exception as e:
|
||||
log.warning("freeomovie listing page=%d failed: %s", page, e)
|
||||
return
|
||||
items = [(m.group("url"), html.unescape(m.group("title")).strip())
|
||||
for m in _LIST_ITEM_RE.finditer(listing)]
|
||||
if not items:
|
||||
log.info("freeomovie: empty page=%d, stop", page)
|
||||
return
|
||||
for url, title in items:
|
||||
if url in seen_urls:
|
||||
continue
|
||||
seen_urls.add(url)
|
||||
try:
|
||||
movie = self._parse_detail(url, title)
|
||||
except Exception as e:
|
||||
log.warning("freeomovie detail %s failed: %s", url, e)
|
||||
continue
|
||||
if movie is None:
|
||||
continue
|
||||
yield movie
|
||||
seen += 1
|
||||
if limit is not None and seen >= limit:
|
||||
return
|
||||
|
||||
def _parse_detail(self, url: str, title: str) -> RawMovie | None:
|
||||
detail = self._fetch(url)
|
||||
if not title:
|
||||
return None
|
||||
|
||||
# Tagi: gatunki z articleSection PRZED "XXX Movies" (reszta = obsada/studio, skip).
|
||||
tags: list[RawTag] = []
|
||||
seen_tag: set[str] = set()
|
||||
sm = _ARTICLE_SECTION_RE.search(detail)
|
||||
if sm:
|
||||
for part in sm.group(1).split(","):
|
||||
name = part.strip()
|
||||
if not name or name.lower() == "xxx movies":
|
||||
if name.lower() == "xxx movies":
|
||||
break # dalej idzie obsada/studio — nie tagi
|
||||
continue
|
||||
sl = slugify(name)
|
||||
if not sl or sl in seen_tag:
|
||||
continue
|
||||
seen_tag.add(sl)
|
||||
tags.append(RawTag(external_id=f"{self.name}:tag:{sl}", name=name, slug=sl))
|
||||
|
||||
# Playback: TABS array hosterów.
|
||||
playback: list[RawPlaybackSource] = []
|
||||
seen_host: set[str] = set()
|
||||
tm = _TABS_RE.search(detail)
|
||||
if tm:
|
||||
try:
|
||||
arr = json.loads(tm.group(1))
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
arr = []
|
||||
for entry in arr:
|
||||
embed = (entry.get("url") or "").strip() if isinstance(entry, dict) else ""
|
||||
if not embed.startswith("http"):
|
||||
continue
|
||||
host = _host_label(embed)
|
||||
if not host or host in _SKIP_HOSTS or host in seen_host:
|
||||
continue
|
||||
seen_host.add(host)
|
||||
playback.append(
|
||||
RawPlaybackSource(
|
||||
origin=f"{self.name}:{host}",
|
||||
page_url=embed,
|
||||
embed_url=embed,
|
||||
)
|
||||
)
|
||||
if not playback:
|
||||
return None # bez playbacku film jest bezużyteczny (nie tworzymy orphana)
|
||||
|
||||
slug = urlparse(url).path.strip("/").split("/")[-1]
|
||||
return RawMovie(
|
||||
external_id=slug,
|
||||
title=title,
|
||||
url=url,
|
||||
tags=tags,
|
||||
playback_sources=playback,
|
||||
raw={"source": "freeomovie", "url": url},
|
||||
)
|
||||
Loading…
Add table
Reference in a new issue