Compare commits
No commits in common. "15c1e4e5e1cc82528cf3e76a0cc8dc8d6e562ffb" and "fa609785645bd5635fe6e2870a0de7ef026862d8" have entirely different histories.
15c1e4e5e1
...
fa60978564
18 changed files with 21 additions and 1077 deletions
|
|
@ -136,16 +136,6 @@ class Settings(BaseSettings):
|
||||||
ingest_watchdog_search_max_age_hours: int = Field(
|
ingest_watchdog_search_max_age_hours: int = Field(
|
||||||
default=168, validation_alias="GOON_INGEST_WATCHDOG_SEARCH_MAX_AGE_HOURS"
|
default=168, validation_alias="GOON_INGEST_WATCHDOG_SEARCH_MAX_AGE_HOURS"
|
||||||
)
|
)
|
||||||
# TPDB movie enrichment — wzbogaca filmy obsadą + kategoriami (tagi) + studiem +
|
|
||||||
# reżyserem, plus kanoniczny TPDB UUID do dedupu mirrorów. paradisehill (primary)
|
|
||||||
# prawie nie ma obsady, więc to domyka największą lukę zakładki movies. Batch
|
|
||||||
# najświeższych GRYWALNYCH filmów bez obsady/studia per run; 6h cadence. 0 = off.
|
|
||||||
sched_tpdb_movie_enrich_hours: int = Field(
|
|
||||||
default=6, validation_alias="GOON_SCHED_TPDB_MOVIE_ENRICH_HOURS"
|
|
||||||
)
|
|
||||||
tpdb_movie_enrich_batch: int = Field(
|
|
||||||
default=200, validation_alias="GOON_TPDB_MOVIE_ENRICH_BATCH"
|
|
||||||
)
|
|
||||||
# Taxonomy scene_count refresh — przelicza denormalizowane liczniki scen na
|
# Taxonomy scene_count refresh — przelicza denormalizowane liczniki scen na
|
||||||
# tags/performers/studios (hot-path /tags|/performers|/studios|/favorites czyta
|
# tags/performers/studios (hot-path /tags|/performers|/studios|/favorites czyta
|
||||||
# gotową kolumnę zamiast agregować 6.3M scene_tags per-request). 3h cadence —
|
# gotową kolumnę zamiast agregować 6.3M scene_tags per-request). 3h cadence —
|
||||||
|
|
|
||||||
|
|
@ -36,9 +36,8 @@ def get_movie_connectors() -> list[tuple[str, type]]:
|
||||||
from app.connectors.dooplay import (
|
from app.connectors.dooplay import (
|
||||||
MangopornConnector,
|
MangopornConnector,
|
||||||
PandamoviesConnector,
|
PandamoviesConnector,
|
||||||
StreampornVipConnector,
|
StreampornConnector,
|
||||||
)
|
)
|
||||||
from app.connectors.freeomovie import FreeoMovieConnector
|
|
||||||
from app.connectors.paradisehill import ParadisehillConnector
|
from app.connectors.paradisehill import ParadisehillConnector
|
||||||
|
|
||||||
# Kolejność ingestu: paradisehill FIRST (canonical primary, mirrory się do
|
# Kolejność ingestu: paradisehill FIRST (canonical primary, mirrory się do
|
||||||
|
|
@ -47,15 +46,9 @@ def get_movie_connectors() -> list[tuple[str, type]]:
|
||||||
# streamporn + pandamovies. Powód reorderu (2026-05-30): gdy streamporn wiesza
|
# streamporn + pandamovies. Powód reorderu (2026-05-30): gdy streamporn wiesza
|
||||||
# się intermittentnie, mangoporn musi zdążyć przed nim — patrz per-connector
|
# się intermittentnie, mangoporn musi zdążyć przed nim — patrz per-connector
|
||||||
# timeout w _job_movie_ingest.
|
# timeout w _job_movie_ingest.
|
||||||
# streamporn.nl padł (0 new od 2026-05-23) → zastąpiony żywym streamporn.vip
|
|
||||||
# (StreampornVipConnector, ocena 2026-07-02). Stara klasa .nl zostaje w dooplay.py
|
|
||||||
# dla backref istniejących origins `streamporn:*`, ale nie ingestujemy jej dalej.
|
|
||||||
return [
|
return [
|
||||||
("paradisehill", ParadisehillConnector),
|
("paradisehill", ParadisehillConnector),
|
||||||
("mangoporn", MangopornConnector),
|
("mangoporn", MangopornConnector),
|
||||||
("streampornvip", StreampornVipConnector),
|
("streamporn", StreampornConnector),
|
||||||
("pandamovies", PandamoviesConnector),
|
("pandamovies", PandamoviesConnector),
|
||||||
# freeomovie.to (bestia theme, nie dooplay) — mirror: świeże filmy DVD,
|
|
||||||
# title-trigram attach do canonical + playback z TABS. Ocena 2026-07-02.
|
|
||||||
("freeomovie", FreeoMovieConnector),
|
|
||||||
]
|
]
|
||||||
|
|
|
||||||
|
|
@ -77,9 +77,11 @@ ALL_DIRECT_SCRAPERS: list[type[BaseDirectTubeScraper]] = [
|
||||||
# Mixdrop zrebrandował na m1xdrop.bz, yt-dlp out-of-date, packer/JS extract = fail.
|
# Mixdrop zrebrandował na m1xdrop.bz, yt-dlp out-of-date, packer/JS extract = fail.
|
||||||
# Playmogo = DoodStream CAPTCHA. Porn-app sam olewa xmoviesforyou (brak handlera w
|
# Playmogo = DoodStream CAPTCHA. Porn-app sam olewa xmoviesforyou (brak handlera w
|
||||||
# jadx). 1,321 solo-orphan scen.
|
# jadx). 1,321 solo-orphan scen.
|
||||||
# WatchPornScraper — RE-ENABLE 2026-07-02, przeniesiony do ALL_BROWSE_SCRAPERS.
|
# WatchPornScraper — wyłączony 2026-05-12 (user bug-report). Wszystkie iframes to
|
||||||
# Site przebudowany na KVS (DoodStream-CAPTCHA z 2026-05-12 zniknął); browse
|
# DoodStream variants (playmogo/d0000d/dooood/mivalyo) z CAPTCHA gate. WebView na
|
||||||
# /latest-updates/ + flashvars get_file direct mp4 (extractor watchporn). Patrz watchporn.py.
|
# mobile = black screen (player JS nie inicjalizuje się przez Turnstile). 16%
|
||||||
|
# scen solo (no backup tube), 84% multi-source — user może użyć innego tube. yt-dlp
|
||||||
|
# nie wspiera DoodStream ("Piracy"), własny resolver TBD jeśli warto.
|
||||||
# SiskaScraper — przeniesiony do ALL_BROWSE_SCRAPERS (browse-konwersja 2026-06-20,
|
# SiskaScraper — przeniesiony do ALL_BROWSE_SCRAPERS (browse-konwersja 2026-06-20,
|
||||||
# bo search siski zepsuty site-side — `?s=` ignoruje query). Patrz siska.py.
|
# bo search siski zepsuty site-side — `?s=` ignoruje query). Patrz siska.py.
|
||||||
# Porn4DaysScraper — wyłączony 2026-05-12 (post audit fix). 100% scen na streamtape
|
# Porn4DaysScraper — wyłączony 2026-05-12 (post audit fix). 100% scen na streamtape
|
||||||
|
|
@ -142,11 +144,6 @@ from app.connectors.direct_scrapers.xvideos_browse import XVideosBrowseScraper
|
||||||
ALL_BROWSE_SCRAPERS: list[type[BaseBrowseScraper]] = [
|
ALL_BROWSE_SCRAPERS: list[type[BaseBrowseScraper]] = [
|
||||||
PerverzijaScraper,
|
PerverzijaScraper,
|
||||||
PornDishScraper,
|
PornDishScraper,
|
||||||
# WatchPornScraper — re-enabled 2026-07-02 jako browse (site przebudowany na KVS,
|
|
||||||
# DoodStream-CAPTCHA zniknął). Browse /latest-updates/: og:title, JSON-LD duration/
|
|
||||||
# uploadDate, /models/ performerzy, /tags/ tagi, /categories/ studio. Playback KVS
|
|
||||||
# get_file direct mp4 (extractor watchporn, VPS-side, token nie IP-bound).
|
|
||||||
WatchPornScraper,
|
|
||||||
# Browse równolegle do istniejącego search scrapera (wzorzec xvideos/eporner):
|
# Browse równolegle do istniejącego search scrapera (wzorzec xvideos/eporner):
|
||||||
# search zostaje (pokrycie back-catalogu performerów), browse gwarantuje świeżość
|
# search zostaje (pokrycie back-catalogu performerów), browse gwarantuje świeżość
|
||||||
# wprost z feedu (watchdog 48h zamiast 168h). Konwersja 2026-06-24 (user request).
|
# wprost z feedu (watchdog 48h zamiast 168h). Konwersja 2026-06-24 (user request).
|
||||||
|
|
|
||||||
|
|
@ -1,108 +1,19 @@
|
||||||
"""watchporn.to — browse scraper (KVS engine). Re-enabled 2026-07-02.
|
"""watchporn.to — direct HTML scrape.
|
||||||
|
|
||||||
Był search-scraperem (`?s=`), zamarzł, a potem site przebudowano na KVS z nowym
|
Search: `https://watchporn.to/page/<n>/?s=<q>` (WordPress).
|
||||||
layoutem. DoodStream-CAPTCHA (powód wyłączenia 2026-05-12) zniknął — teraz KVS
|
Scene URL: `https://watchporn.to/videos/<slug>/`.
|
||||||
flashvars `get_file` direct mp4 (extractor `watchporn`, VPS-side, token nie IP-bound).
|
|
||||||
|
|
||||||
Browse `/latest-updates/` → detail page:
|
|
||||||
- title: og:title ("Studio/Creator - Scene Title")
|
|
||||||
- duration + release_date: JSON-LD "duration" (ISO) + "uploadDate"
|
|
||||||
- performerzy: `/models/<slug>/` (pomijamy numeryczne id-slugi), nazwa z tekstu linku
|
|
||||||
- tagi: `/tags/<slug>/`
|
|
||||||
- studio: pierwszy `/categories/<slug>/` (np. EvilAngel, ManyVids)
|
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import html
|
|
||||||
import re
|
import re
|
||||||
|
|
||||||
from app.connectors.base import (
|
from app.connectors.direct_scrapers._search_base import BaseSearchScraper
|
||||||
RawPerformer,
|
|
||||||
RawPlaybackSource,
|
|
||||||
RawScene,
|
|
||||||
RawStudio,
|
|
||||||
RawTag,
|
|
||||||
)
|
|
||||||
from app.connectors.direct_scrapers._browse_base import BaseBrowseScraper, meta_content
|
|
||||||
from app.connectors.direct_scrapers._playtube import _parse_iso_date, _parse_iso_duration
|
|
||||||
from app.normalize.text import slugify
|
|
||||||
|
|
||||||
_BASE = "https://watchporn.to"
|
|
||||||
_SCENE_URL_RE = re.compile(r'href="(https://watchporn\.to/video/\d+/[a-z0-9\-]+/)"', re.IGNORECASE)
|
|
||||||
_MODEL_RE = re.compile(r'href="https://watchporn\.to/models/([a-z0-9\-]+)/"[^>]*>([^<]+)', re.IGNORECASE)
|
|
||||||
_TAG_RE = re.compile(r'href="https://watchporn\.to/tags/([a-z0-9\-]+)/"[^>]*>([^<]+)', re.IGNORECASE)
|
|
||||||
_CAT_RE = re.compile(r'href="https://watchporn\.to/categories/([a-z0-9\-]+)/"[^>]*>([^<]+)', re.IGNORECASE)
|
|
||||||
_DUR_RE = re.compile(r'"duration"\s*:\s*"([^"]+)"')
|
|
||||||
_UPLOAD_RE = re.compile(r'"uploadDate"\s*:\s*"([^"]+)"')
|
|
||||||
|
|
||||||
|
|
||||||
class WatchPornScraper(BaseBrowseScraper):
|
class WatchPornScraper(BaseSearchScraper):
|
||||||
sitetag = "watchporn"
|
sitetag = "watchporn"
|
||||||
|
_search_url_template = "https://watchporn.to/page/{page}/?s={query}"
|
||||||
def _listing_url(self, page: int) -> str:
|
_scene_url_re = re.compile(
|
||||||
return f"{_BASE}/latest-updates/" if page <= 1 else f"{_BASE}/latest-updates/{page}/"
|
r'href="(?P<url>https://watchporn\.to/videos/(?P<slug>[a-z0-9][a-z0-9\-]+))/"',
|
||||||
|
re.IGNORECASE,
|
||||||
def _extract_scene_urls(self, listing_html: str) -> list[str]:
|
|
||||||
seen: set[str] = set()
|
|
||||||
out: list[str] = []
|
|
||||||
for m in _SCENE_URL_RE.finditer(listing_html):
|
|
||||||
url = m.group(1)
|
|
||||||
if url not in seen:
|
|
||||||
seen.add(url)
|
|
||||||
out.append(url)
|
|
||||||
return out
|
|
||||||
|
|
||||||
def _parse_detail(self, scene_url: str, detail_html: str) -> RawScene | None:
|
|
||||||
title = (meta_content(detail_html, property="og:title") or "").strip()
|
|
||||||
if not title:
|
|
||||||
return None
|
|
||||||
|
|
||||||
dm = _DUR_RE.search(detail_html)
|
|
||||||
duration_sec = _parse_iso_duration(dm.group(1)) if dm else None
|
|
||||||
um = _UPLOAD_RE.search(detail_html)
|
|
||||||
release_date = _parse_iso_date(um.group(1)) if um else None
|
|
||||||
|
|
||||||
performers: list[RawPerformer] = []
|
|
||||||
seen_p: set[str] = set()
|
|
||||||
for m in _MODEL_RE.finditer(detail_html):
|
|
||||||
slug = m.group(1)
|
|
||||||
name = html.unescape(m.group(2)).strip()
|
|
||||||
if slug.isdigit() or slug in seen_p or not name:
|
|
||||||
continue
|
|
||||||
seen_p.add(slug)
|
|
||||||
performers.append(RawPerformer(external_id=f"{self.sitetag}:model:{slug}", name=name))
|
|
||||||
|
|
||||||
tags: list[RawTag] = []
|
|
||||||
seen_t: set[str] = set()
|
|
||||||
for m in _TAG_RE.finditer(detail_html):
|
|
||||||
slug = m.group(1)
|
|
||||||
name = html.unescape(m.group(2)).strip()
|
|
||||||
if slug in seen_t or not name:
|
|
||||||
continue
|
|
||||||
seen_t.add(slug)
|
|
||||||
tags.append(RawTag(external_id=f"{self.sitetag}:tag:{slug}", name=name, slug=slug))
|
|
||||||
|
|
||||||
studio: RawStudio | None = None
|
|
||||||
cm = _CAT_RE.search(detail_html)
|
|
||||||
if cm:
|
|
||||||
cname = html.unescape(cm.group(2)).strip()
|
|
||||||
if cname:
|
|
||||||
studio = RawStudio(
|
|
||||||
external_id=f"{self.sitetag}:studio:{slugify(cname)}", name=cname, slug=slugify(cname)
|
|
||||||
)
|
|
||||||
|
|
||||||
return RawScene(
|
|
||||||
external_id=f"{self.sitetag}:{scene_url}",
|
|
||||||
title=title,
|
|
||||||
release_date=release_date,
|
|
||||||
duration_sec=duration_sec,
|
|
||||||
url=scene_url,
|
|
||||||
studio=studio,
|
|
||||||
performers=performers,
|
|
||||||
tags=tags,
|
|
||||||
playback_sources=[
|
|
||||||
RawPlaybackSource(
|
|
||||||
origin=f"tube:{self.sitetag}", page_url=scene_url, duration_sec=duration_sec
|
|
||||||
)
|
|
||||||
],
|
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -447,14 +447,6 @@ class StreampornConnector(DooplayConnector):
|
||||||
base_url = "https://streamporn.nl"
|
base_url = "https://streamporn.nl"
|
||||||
|
|
||||||
|
|
||||||
class StreampornVipConnector(DooplayConnector):
|
|
||||||
# streamporn.nl padł (stale od 2026-05-23); streamporn.vip to żywy sibling z tym
|
|
||||||
# samym PsyPlay theme i świeższym katalogiem (ocena 2026-07-02). Osobny source_name
|
|
||||||
# → nowe origins `streampornvip:<host>`, nie miesza z martwym .nl.
|
|
||||||
name = "streampornvip"
|
|
||||||
base_url = "https://streamporn.vip"
|
|
||||||
|
|
||||||
|
|
||||||
class PandamoviesConnector(DooplayConnector):
|
class PandamoviesConnector(DooplayConnector):
|
||||||
name = "pandamovies"
|
name = "pandamovies"
|
||||||
base_url = "https://pandamovies.pw"
|
base_url = "https://pandamovies.pw"
|
||||||
|
|
|
||||||
|
|
@ -1,201 +0,0 @@
|
||||||
"""freeomovie.to — movie source (WordPress "bestia" theme, NIE dooplay).
|
|
||||||
|
|
||||||
Dodany 2026-07-02 (ocena). Pełnometrażowe filmy, świeże (auto-poster, dziś), tytuły
|
|
||||||
DVD → title-trigram mirror-attach do canonical (paradisehill/TPDB movies), dokładając
|
|
||||||
playback_sources. Scope: TYLKO `/category/full-movie/` (homepage miesza scen-klipy,
|
|
||||||
które orphanowałyby jako filmy).
|
|
||||||
|
|
||||||
Struktura:
|
|
||||||
- listing `<li class="thumi"><a href title>` → URL filmu + czysty tytuł
|
|
||||||
- detail: `var TABS=[{label,url},...]` (hostery), `"articleSection"` (gatunki + obsada;
|
|
||||||
gatunki PRZED "XXX Movies" → tagi), `"datePublished"` (data POSTA, NIE rok produkcji)
|
|
||||||
|
|
||||||
Świadomie NIE bierzemy:
|
|
||||||
- release_year: datePublished to data uploadu (2026), nie rok filmu → fałszywy rok
|
|
||||||
psułby year-scoring przy matchu do canonical (film z 2010 dostałby 2026). Lepiej None.
|
|
||||||
- performerów/studia z articleSection: format miesza performer/alias/studio bez czystego
|
|
||||||
delimitera → ryzyko junk-performera ("Deeveeous" studio jako performer). Mirror i tak
|
|
||||||
doczepia się do canonical po tytule, a TPDB enrichment dokłada obsadę/studio autorytatywnie.
|
|
||||||
|
|
||||||
Playback: TABS hostery → MoviePlaybackSource per host (voe/luluvid/vidhide resolvują się
|
|
||||||
VPS-side; myvidplay = DoodStream clone, phone-side — wymaga myvidplay.com w DOOD_HOSTS).
|
|
||||||
Pomijamy streamtape (martwy malware) + mxdrop (flaky).
|
|
||||||
"""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import html
|
|
||||||
import json
|
|
||||||
import logging
|
|
||||||
import re
|
|
||||||
from collections.abc import Iterator
|
|
||||||
from datetime import datetime
|
|
||||||
from urllib.parse import urlparse
|
|
||||||
|
|
||||||
from app.connectors.base import (
|
|
||||||
BaseMovieConnector,
|
|
||||||
RawMovie,
|
|
||||||
RawPlaybackSource,
|
|
||||||
RawTag,
|
|
||||||
)
|
|
||||||
from app.extractors import browser_get
|
|
||||||
from app.models.source import SourceKind
|
|
||||||
from app.normalize.text import slugify
|
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
_BASE = "https://www.freeomovie.to"
|
|
||||||
_LIST_ITEM_RE = re.compile(
|
|
||||||
r'<li[^>]*class="thumi"[^>]*>\s*<a[^>]+href="(?P<url>https://www\.freeomovie\.to/[a-z0-9\-]+/)"[^>]*title="(?P<title>[^"]+)"',
|
|
||||||
re.IGNORECASE,
|
|
||||||
)
|
|
||||||
_TABS_RE = re.compile(r"var\s+TABS\s*=\s*(\[.*?\])\s*;", re.DOTALL)
|
|
||||||
_ARTICLE_SECTION_RE = re.compile(r'"articleSection"\s*:\s*"([^"]+)"')
|
|
||||||
# Poster: <img class="... rmbd ..." src="..."> (fastpic.org itp.). Bez tego film-orphan
|
|
||||||
# (tytuł bez matchu do canonical) miał poster_url=None → pusty kafelek (report 8f8c10c0).
|
|
||||||
_POSTER_RE = re.compile(
|
|
||||||
r'<img[^>]+class="[^"]*\brmbd\b[^"]*"[^>]+src="(https?://[^"]+)"',
|
|
||||||
re.IGNORECASE,
|
|
||||||
)
|
|
||||||
|
|
||||||
# Hostery pomijane (martwe/flaky). player4me = JS-SPA bez znanego resolvera (28 martwych
|
|
||||||
# źródeł freeomovie:video, report 8f8c10c0).
|
|
||||||
_SKIP_HOSTS = ("streamtape", "mxdrop", "mixdrop", "streamsb", "player4me")
|
|
||||||
|
|
||||||
|
|
||||||
def _host_label(embed_url: str) -> str | None:
|
|
||||||
host = (urlparse(embed_url).hostname or "").lower()
|
|
||||||
if not host:
|
|
||||||
return None
|
|
||||||
parts = host.replace("www.", "").split(".")
|
|
||||||
# Rejestrowalna domena (SLD), nie subdomena: video.player4me.xyz → 'player4me' (nie
|
|
||||||
# 'video'), myvidplay.com → 'myvidplay'. Inaczej origin był śmieciowy (freeomovie:video).
|
|
||||||
return parts[-2] if len(parts) >= 2 else parts[0]
|
|
||||||
|
|
||||||
|
|
||||||
class FreeoMovieConnector(BaseMovieConnector):
|
|
||||||
kind = SourceKind.scraper
|
|
||||||
name = "freeomovie"
|
|
||||||
base_url = _BASE
|
|
||||||
|
|
||||||
_MAX_PAGES_DELTA = 3
|
|
||||||
_MAX_PAGES_FULL = 40
|
|
||||||
|
|
||||||
def __init__(self, *, timeout: float = 30.0) -> None:
|
|
||||||
self._timeout = timeout
|
|
||||||
|
|
||||||
def close(self) -> None:
|
|
||||||
pass
|
|
||||||
|
|
||||||
def _fetch(self, url: str) -> str:
|
|
||||||
if not url.startswith("http"):
|
|
||||||
url = _BASE + url
|
|
||||||
r = browser_get(
|
|
||||||
url,
|
|
||||||
headers={
|
|
||||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/125.0",
|
|
||||||
"Accept": "text/html,application/xhtml+xml",
|
|
||||||
"Referer": _BASE + "/",
|
|
||||||
},
|
|
||||||
timeout=self._timeout,
|
|
||||||
follow_redirects=True,
|
|
||||||
)
|
|
||||||
if r.status_code >= 400:
|
|
||||||
raise RuntimeError(f"{r.status_code} for {url}")
|
|
||||||
return r.text
|
|
||||||
|
|
||||||
def fetch_movies(
|
|
||||||
self, *, since: datetime | None = None, limit: int | None = None
|
|
||||||
) -> Iterator[RawMovie]:
|
|
||||||
seen = 0
|
|
||||||
seen_urls: set[str] = set()
|
|
||||||
max_pages = self._MAX_PAGES_DELTA if since is not None else self._MAX_PAGES_FULL
|
|
||||||
for page in range(1, max_pages + 1):
|
|
||||||
path = "/category/full-movie/" if page == 1 else f"/category/full-movie/page/{page}/"
|
|
||||||
try:
|
|
||||||
listing = self._fetch(path)
|
|
||||||
except Exception as e:
|
|
||||||
log.warning("freeomovie listing page=%d failed: %s", page, e)
|
|
||||||
return
|
|
||||||
items = [(m.group("url"), html.unescape(m.group("title")).strip())
|
|
||||||
for m in _LIST_ITEM_RE.finditer(listing)]
|
|
||||||
if not items:
|
|
||||||
log.info("freeomovie: empty page=%d, stop", page)
|
|
||||||
return
|
|
||||||
for url, title in items:
|
|
||||||
if url in seen_urls:
|
|
||||||
continue
|
|
||||||
seen_urls.add(url)
|
|
||||||
try:
|
|
||||||
movie = self._parse_detail(url, title)
|
|
||||||
except Exception as e:
|
|
||||||
log.warning("freeomovie detail %s failed: %s", url, e)
|
|
||||||
continue
|
|
||||||
if movie is None:
|
|
||||||
continue
|
|
||||||
yield movie
|
|
||||||
seen += 1
|
|
||||||
if limit is not None and seen >= limit:
|
|
||||||
return
|
|
||||||
|
|
||||||
def _parse_detail(self, url: str, title: str) -> RawMovie | None:
|
|
||||||
detail = self._fetch(url)
|
|
||||||
if not title:
|
|
||||||
return None
|
|
||||||
|
|
||||||
# Tagi: gatunki z articleSection PRZED "XXX Movies" (reszta = obsada/studio, skip).
|
|
||||||
tags: list[RawTag] = []
|
|
||||||
seen_tag: set[str] = set()
|
|
||||||
sm = _ARTICLE_SECTION_RE.search(detail)
|
|
||||||
if sm:
|
|
||||||
for part in sm.group(1).split(","):
|
|
||||||
name = part.strip()
|
|
||||||
if not name or name.lower() == "xxx movies":
|
|
||||||
if name.lower() == "xxx movies":
|
|
||||||
break # dalej idzie obsada/studio — nie tagi
|
|
||||||
continue
|
|
||||||
sl = slugify(name)
|
|
||||||
if not sl or sl in seen_tag:
|
|
||||||
continue
|
|
||||||
seen_tag.add(sl)
|
|
||||||
tags.append(RawTag(external_id=f"{self.name}:tag:{sl}", name=name, slug=sl))
|
|
||||||
|
|
||||||
# Playback: TABS array hosterów.
|
|
||||||
playback: list[RawPlaybackSource] = []
|
|
||||||
seen_host: set[str] = set()
|
|
||||||
tm = _TABS_RE.search(detail)
|
|
||||||
if tm:
|
|
||||||
try:
|
|
||||||
arr = json.loads(tm.group(1))
|
|
||||||
except (json.JSONDecodeError, ValueError):
|
|
||||||
arr = []
|
|
||||||
for entry in arr:
|
|
||||||
embed = (entry.get("url") or "").strip() if isinstance(entry, dict) else ""
|
|
||||||
if not embed.startswith("http"):
|
|
||||||
continue
|
|
||||||
host = _host_label(embed)
|
|
||||||
if not host or host in _SKIP_HOSTS or host in seen_host:
|
|
||||||
continue
|
|
||||||
seen_host.add(host)
|
|
||||||
playback.append(
|
|
||||||
RawPlaybackSource(
|
|
||||||
origin=f"{self.name}:{host}",
|
|
||||||
page_url=embed,
|
|
||||||
embed_url=embed,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
if not playback:
|
|
||||||
return None # bez playbacku film jest bezużyteczny (nie tworzymy orphana)
|
|
||||||
|
|
||||||
pm = _POSTER_RE.search(detail)
|
|
||||||
poster_url = html.unescape(pm.group(1)) if pm else None
|
|
||||||
|
|
||||||
slug = urlparse(url).path.strip("/").split("/")[-1]
|
|
||||||
return RawMovie(
|
|
||||||
external_id=slug,
|
|
||||||
title=title,
|
|
||||||
url=url,
|
|
||||||
poster_url=poster_url,
|
|
||||||
tags=tags,
|
|
||||||
playback_sources=playback,
|
|
||||||
raw={"source": "freeomovie", "url": url},
|
|
||||||
)
|
|
||||||
|
|
@ -49,7 +49,6 @@ def _is_retryable_http_error(exc: BaseException) -> bool:
|
||||||
from app.config import get_settings
|
from app.config import get_settings
|
||||||
from app.connectors.base import (
|
from app.connectors.base import (
|
||||||
BaseConnector,
|
BaseConnector,
|
||||||
RawMovie,
|
|
||||||
RawPerformer,
|
RawPerformer,
|
||||||
RawScene,
|
RawScene,
|
||||||
RawStudio,
|
RawStudio,
|
||||||
|
|
@ -196,37 +195,6 @@ class TPDBConnector(BaseConnector):
|
||||||
first = data[0]
|
first = data[0]
|
||||||
return str(first.get("id")) if first.get("id") else None
|
return str(first.get("id")) if first.get("id") else None
|
||||||
|
|
||||||
def search_movies(self, query: str, *, per_page: int = 10) -> list[dict[str, Any]]:
|
|
||||||
"""GET /movies?q=<query> → surowe payloady filmów (list endpoint jest już
|
|
||||||
hydrated: tags + performers + site embedded, bez detail-fetcha).
|
|
||||||
|
|
||||||
Używane do enrichmentu movies: wyszukujemy kandydatów po tytule, scorujemy
|
|
||||||
i mapujemy najlepszy przez `_parse_movie`. Pusty list na błąd/brak."""
|
|
||||||
if not query.strip():
|
|
||||||
return []
|
|
||||||
with self._client() as client:
|
|
||||||
try:
|
|
||||||
payload = self._get(client, "/movies", {"q": query, "per_page": per_page})
|
|
||||||
except httpx.HTTPStatusError as e:
|
|
||||||
log.warning("tpdb /movies q=%s failed: %s", query, e)
|
|
||||||
return []
|
|
||||||
return payload.get("data") or []
|
|
||||||
|
|
||||||
def fetch_movie(self, movie_uuid: str) -> dict[str, Any] | None:
|
|
||||||
"""GET /movies/<uuid> → pojedynczy film (detail). None gdy 404/błąd.
|
|
||||||
|
|
||||||
NB: list endpoint jest już hydrated, więc detail rzadko potrzebny — trzymamy
|
|
||||||
dla spójności / gdyby TPDB kiedyś przeniósł część pól tylko do detalu."""
|
|
||||||
with self._client() as client:
|
|
||||||
try:
|
|
||||||
payload = self._get(client, f"/movies/{movie_uuid}", {})
|
|
||||||
except httpx.HTTPStatusError as e:
|
|
||||||
if e.response.status_code == 404:
|
|
||||||
return None
|
|
||||||
log.warning("tpdb /movies/%s failed: %s", movie_uuid, e)
|
|
||||||
return None
|
|
||||||
return payload.get("data")
|
|
||||||
|
|
||||||
def _paginate_scenes(
|
def _paginate_scenes(
|
||||||
self,
|
self,
|
||||||
params: dict[str, Any],
|
params: dict[str, Any],
|
||||||
|
|
@ -359,60 +327,3 @@ def _parse_scene(raw: dict[str, Any]) -> RawScene | None:
|
||||||
fingerprints=[], # TPDB nie publikuje pHashy w głównym endpoint
|
fingerprints=[], # TPDB nie publikuje pHashy w głównym endpoint
|
||||||
raw=raw,
|
raw=raw,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def _parse_movie(raw: dict[str, Any]) -> RawMovie | None:
|
|
||||||
"""TPDB movie payload → RawMovie. Reużywa _parse_studio/_parse_performer/_parse_tag
|
|
||||||
(movies mają identyczny kształt site/performers/tags co sceny)."""
|
|
||||||
external_id = raw.get("id")
|
|
||||||
title = raw.get("title")
|
|
||||||
if not external_id or not title:
|
|
||||||
return None
|
|
||||||
|
|
||||||
performers: list[RawPerformer] = []
|
|
||||||
for p in raw.get("performers") or []:
|
|
||||||
parsed = _parse_performer(p)
|
|
||||||
if parsed is not None:
|
|
||||||
performers.append(parsed)
|
|
||||||
tags: list[RawTag] = []
|
|
||||||
for t in raw.get("tags") or []:
|
|
||||||
parsed_t = _parse_tag(t)
|
|
||||||
if parsed_t is not None:
|
|
||||||
tags.append(parsed_t)
|
|
||||||
|
|
||||||
directors = raw.get("directors") or []
|
|
||||||
director = ", ".join(d.get("name") for d in directors if isinstance(d, dict) and d.get("name")) or None
|
|
||||||
|
|
||||||
d = _parse_date(raw.get("date"))
|
|
||||||
# duration bywa 1s-placeholderem na movies — poniżej 60s traktujemy jak brak.
|
|
||||||
dur = raw.get("duration")
|
|
||||||
duration_sec = int(dur) if dur and int(dur) > 60 else None
|
|
||||||
|
|
||||||
def _img(field: str) -> str | None:
|
|
||||||
v = raw.get(field)
|
|
||||||
if isinstance(v, str) and v:
|
|
||||||
return v
|
|
||||||
if isinstance(v, dict):
|
|
||||||
return v.get("large") or v.get("full") or v.get("medium") or None
|
|
||||||
return None
|
|
||||||
|
|
||||||
poster = _img("poster") or _img("posters")
|
|
||||||
backdrop = _img("background") or _img("backdrop")
|
|
||||||
|
|
||||||
return RawMovie(
|
|
||||||
external_id=str(external_id),
|
|
||||||
title=title,
|
|
||||||
description=raw.get("description"),
|
|
||||||
release_year=d.year if d else None,
|
|
||||||
release_date=d,
|
|
||||||
duration_sec=duration_sec,
|
|
||||||
director=director,
|
|
||||||
rating=float(raw["rating"]) if raw.get("rating") not in (None, "") else None,
|
|
||||||
poster_url=poster,
|
|
||||||
backdrop_url=backdrop,
|
|
||||||
url=raw.get("url"),
|
|
||||||
studio=_parse_studio(raw.get("site")),
|
|
||||||
performers=performers,
|
|
||||||
tags=tags,
|
|
||||||
raw=raw,
|
|
||||||
)
|
|
||||||
|
|
|
||||||
|
|
@ -1 +0,0 @@
|
||||||
"""Post-ingest enrichment of canonical entities from authoritative metadata sources."""
|
|
||||||
|
|
@ -1,273 +0,0 @@
|
||||||
"""TPDB movie enrichment + dedup.
|
|
||||||
|
|
||||||
TPDB `/movies` jest naszym kanonicznym źródłem metadanych filmów: obsada, kategorie
|
|
||||||
(tagi), studio, reżyser, rok — plus stabilny UUID do dedupu. paradisehill (primary
|
|
||||||
movie source) prawie nie ma obsady, więc TPDB wypełnia największą lukę.
|
|
||||||
|
|
||||||
Zasada: TPDB TYLKO wzbogaca ISTNIEJĄCE filmy (z paradisehill/dooplay) i dedupuje,
|
|
||||||
NIGDY nie tworzy nowych — TPDB nie ma playbacku, więc nowy film byłby niegrywalny.
|
|
||||||
|
|
||||||
Flow per film:
|
|
||||||
1. skip, jeśli film ma już TPDB movie ref (movie_external_refs jest tylko-movie,
|
|
||||||
więc dowolny ref z tpdb source = już wzbogacony),
|
|
||||||
2. search TPDB /movies?q=<tytuł>, wybierz najlepszego kandydata (token-set na tytule
|
|
||||||
+ guard roku ±2, próg `min_title`),
|
|
||||||
3. DEDUP: jeśli ten TPDB UUID jest już przypięty do INNEGO naszego filmu → to ten
|
|
||||||
sam film (mirror) → merge_movies (keep = starszy created_at),
|
|
||||||
4. wzbogać ocalały film: obsada (resolve_performer → MoviePerformer), tagi
|
|
||||||
(resolve_tag → MovieTag source=tpdb), studio (fill studio_id), reżyser/rok/
|
|
||||||
poster/rating (fill-only), przypnij MovieExternalRef(tpdb, uuid).
|
|
||||||
|
|
||||||
Match jest zachowawczy (próg 0.90 + guard roku), bo TPDB search zwraca dużo (np.
|
|
||||||
"Pirates" → gay Knightbreeders przed feature) — bez scoringu wzięlibyśmy zły film.
|
|
||||||
"""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import logging
|
|
||||||
import uuid
|
|
||||||
from datetime import UTC, datetime
|
|
||||||
|
|
||||||
from rapidfuzz import fuzz
|
|
||||||
from sqlalchemy import func, select
|
|
||||||
from sqlalchemy.orm import Session
|
|
||||||
|
|
||||||
from app.connectors.tpdb import TPDBConnector, _parse_movie
|
|
||||||
from app.models.movie import Movie, MovieExternalRef, MoviePerformer
|
|
||||||
from app.models.movie_playback_source import MoviePlaybackSource
|
|
||||||
from app.normalize.movies import normalize_movie
|
|
||||||
from app.normalize.text import normalize
|
|
||||||
from app.resolve.movie_merge import merge_movies
|
|
||||||
from app.resolve.movie_resolver import _sync_performers, _sync_tags
|
|
||||||
from app.resolve.performer_resolver import resolve_performer
|
|
||||||
from app.resolve.studio_resolver import resolve_studio
|
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
_DASH = str.maketrans({"–": "-", "—": "-", "‑": "-"})
|
|
||||||
|
|
||||||
|
|
||||||
def _cand_year(raw: dict) -> int | None:
|
|
||||||
d = raw.get("date")
|
|
||||||
if d and len(str(d)) >= 4:
|
|
||||||
try:
|
|
||||||
return int(str(d)[:4])
|
|
||||||
except ValueError:
|
|
||||||
return None
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def _best_match(connector: TPDBConnector, movie: Movie, *, min_title: float) -> dict | None:
|
|
||||||
"""Najlepszy TPDB movie payload dla naszego filmu, albo None.
|
|
||||||
|
|
||||||
`token_sort_ratio` (a NIE token_set) na znormalizowanych tytułach: odporny na
|
|
||||||
inwersję ("Title, The" ↔ "The Title") i kolejność, ALE penalizuje różnicę długości,
|
|
||||||
więc krótki generyczny tytuł nie łapie dłuższego nadzbioru (bug: "Fantasies" →
|
|
||||||
"Tara's Fetish Fantasies", bo token_set nagradza podzbiór). Precyzja > recall:
|
|
||||||
lepiej pominąć niż wzbogacić zły film. Guard roku ±2 (inny rok = inna edycja)."""
|
|
||||||
query = (movie.title or "").translate(_DASH).strip()[:60]
|
|
||||||
if not query:
|
|
||||||
return None
|
|
||||||
my_norm = normalize(movie.title)
|
|
||||||
best: dict | None = None
|
|
||||||
best_score = 0.0
|
|
||||||
for raw in connector.search_movies(query, per_page=10):
|
|
||||||
cand_title = raw.get("title")
|
|
||||||
if not cand_title:
|
|
||||||
continue
|
|
||||||
score = fuzz.token_sort_ratio(my_norm, normalize(cand_title)) / 100.0
|
|
||||||
if score <= best_score:
|
|
||||||
continue
|
|
||||||
cy = _cand_year(raw)
|
|
||||||
if movie.release_year and cy and abs(movie.release_year - cy) > 2:
|
|
||||||
continue # guard: inny rok → prawdopodobnie inny film (Taxi 2 ≠ Taxi Violeur 2)
|
|
||||||
best_score = score
|
|
||||||
best = raw
|
|
||||||
return best if best is not None and best_score >= min_title else None
|
|
||||||
|
|
||||||
|
|
||||||
def _fill_scalar_fields(movie: Movie, norm) -> None:
|
|
||||||
"""Fill-only — nie nadpisujemy pól ustawionych przez primary (paradisehill/mirror)."""
|
|
||||||
if norm.director and not movie.director:
|
|
||||||
movie.director = norm.director
|
|
||||||
if norm.release_year and not movie.release_year:
|
|
||||||
movie.release_year = norm.release_year
|
|
||||||
if norm.release_date and not movie.release_date:
|
|
||||||
movie.release_date = norm.release_date
|
|
||||||
if norm.duration_sec and not movie.duration_sec:
|
|
||||||
movie.duration_sec = norm.duration_sec
|
|
||||||
if norm.description and not movie.description:
|
|
||||||
movie.description = norm.description
|
|
||||||
if norm.poster_url and not movie.poster_url:
|
|
||||||
movie.poster_url = norm.poster_url
|
|
||||||
if norm.backdrop_url and not movie.backdrop_url:
|
|
||||||
movie.backdrop_url = norm.backdrop_url
|
|
||||||
if norm.rating is not None and movie.rating is None:
|
|
||||||
movie.rating = norm.rating
|
|
||||||
|
|
||||||
|
|
||||||
def enrich_movie(
|
|
||||||
session: Session,
|
|
||||||
movie: Movie,
|
|
||||||
*,
|
|
||||||
connector: TPDBConnector,
|
|
||||||
source_id: uuid.UUID,
|
|
||||||
min_title: float = 0.90,
|
|
||||||
) -> str:
|
|
||||||
"""Zwraca: 'skip' | 'no_match' | 'enriched' | 'merged'."""
|
|
||||||
already = session.execute(
|
|
||||||
select(MovieExternalRef.external_id).where(
|
|
||||||
MovieExternalRef.source_id == source_id,
|
|
||||||
MovieExternalRef.movie_id == movie.id,
|
|
||||||
)
|
|
||||||
).first()
|
|
||||||
if already is not None:
|
|
||||||
return "skip"
|
|
||||||
|
|
||||||
raw = _best_match(connector, movie, min_title=min_title)
|
|
||||||
if raw is None:
|
|
||||||
return "no_match"
|
|
||||||
ext_id = str(raw["id"])
|
|
||||||
|
|
||||||
outcome = "enriched"
|
|
||||||
# DEDUP: ten TPDB UUID już przypięty do innego naszego filmu → mirror tego samego.
|
|
||||||
other = session.execute(
|
|
||||||
select(MovieExternalRef).where(
|
|
||||||
MovieExternalRef.source_id == source_id,
|
|
||||||
MovieExternalRef.external_id == ext_id,
|
|
||||||
)
|
|
||||||
).scalar_one_or_none()
|
|
||||||
if other is not None and other.movie_id != movie.id:
|
|
||||||
m2 = session.get(Movie, other.movie_id)
|
|
||||||
if m2 is not None:
|
|
||||||
now = datetime.now(UTC)
|
|
||||||
keep, drop = (
|
|
||||||
(movie, m2)
|
|
||||||
if (movie.created_at or now) <= (m2.created_at or now)
|
|
||||||
else (m2, movie)
|
|
||||||
)
|
|
||||||
movie = merge_movies(
|
|
||||||
session, keep_id=keep.id, drop_id=drop.id, resolved_by="tpdb:dedup"
|
|
||||||
)
|
|
||||||
outcome = "merged"
|
|
||||||
|
|
||||||
rm = _parse_movie(raw)
|
|
||||||
if rm is None:
|
|
||||||
return "no_match"
|
|
||||||
norm = normalize_movie(rm)
|
|
||||||
|
|
||||||
if norm.studio is not None:
|
|
||||||
studio = resolve_studio(session, norm=norm.studio, source_id=source_id)
|
|
||||||
if studio is not None and not movie.studio_id:
|
|
||||||
movie.studio_id = studio.id
|
|
||||||
|
|
||||||
# Dedup wchodzących performerów po external_id (TPDB potrafi wylistować tego samego
|
|
||||||
# kanonicznego performera 2×: pod aliasem i kanonicznie) — bez tego resolve_performer
|
|
||||||
# próbuje wstawić performer_external_refs 2× → UniqueViolation (jak w scene_resolver).
|
|
||||||
resolved: list[tuple[uuid.UUID, str | None]] = []
|
|
||||||
seen_perf_keys: set[str] = set()
|
|
||||||
for p_norm in norm.performers:
|
|
||||||
key = p_norm.external_id or normalize(p_norm.name)
|
|
||||||
if key in seen_perf_keys:
|
|
||||||
continue
|
|
||||||
seen_perf_keys.add(key)
|
|
||||||
performer = resolve_performer(session, norm=p_norm, source_id=source_id)
|
|
||||||
resolved.append((performer.id, p_norm.as_alias_in_scene))
|
|
||||||
_sync_performers(session, movie_id=movie.id, resolved=resolved)
|
|
||||||
_sync_tags(session, movie_id=movie.id, norm=norm, source_id=source_id)
|
|
||||||
_fill_scalar_fields(movie, norm)
|
|
||||||
|
|
||||||
ref = session.execute(
|
|
||||||
select(MovieExternalRef).where(
|
|
||||||
MovieExternalRef.source_id == source_id,
|
|
||||||
MovieExternalRef.external_id == ext_id,
|
|
||||||
)
|
|
||||||
).scalar_one_or_none()
|
|
||||||
if ref is None:
|
|
||||||
session.add(
|
|
||||||
MovieExternalRef(
|
|
||||||
source_id=source_id,
|
|
||||||
external_id=ext_id,
|
|
||||||
movie_id=movie.id,
|
|
||||||
confidence=1.0,
|
|
||||||
url=rm.url,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
elif ref.movie_id != movie.id:
|
|
||||||
ref.movie_id = movie.id
|
|
||||||
|
|
||||||
return outcome
|
|
||||||
|
|
||||||
|
|
||||||
def _candidate_movies(session: Session, *, source_id: uuid.UUID, limit: int) -> list[Movie]:
|
|
||||||
"""Filmy jeszcze nie wzbogacone TPDB, priorytet: mają żywy playback (są grywalne)
|
|
||||||
i brak obsady LUB brak studia (największa luka). Reszta później."""
|
|
||||||
has_tpdb = (
|
|
||||||
select(MovieExternalRef.movie_id)
|
|
||||||
.where(MovieExternalRef.source_id == source_id)
|
|
||||||
.scalar_subquery()
|
|
||||||
)
|
|
||||||
has_playback = (
|
|
||||||
select(MoviePlaybackSource.movie_id)
|
|
||||||
.where(MoviePlaybackSource.dead_at.is_(None))
|
|
||||||
.scalar_subquery()
|
|
||||||
)
|
|
||||||
perf_count = (
|
|
||||||
select(func.count())
|
|
||||||
.select_from(MoviePerformer)
|
|
||||||
.where(MoviePerformer.movie_id == Movie.id)
|
|
||||||
.correlate(Movie)
|
|
||||||
.scalar_subquery()
|
|
||||||
)
|
|
||||||
stmt = (
|
|
||||||
select(Movie)
|
|
||||||
.where(
|
|
||||||
Movie.id.not_in(has_tpdb),
|
|
||||||
Movie.id.in_(has_playback),
|
|
||||||
)
|
|
||||||
.where((perf_count == 0) | (Movie.studio_id.is_(None)))
|
|
||||||
.order_by(Movie.created_at.desc())
|
|
||||||
.limit(limit)
|
|
||||||
)
|
|
||||||
return list(session.execute(stmt).scalars().all())
|
|
||||||
|
|
||||||
|
|
||||||
def run_tpdb_movie_enrich(
|
|
||||||
session_factory,
|
|
||||||
*,
|
|
||||||
limit: int = 200,
|
|
||||||
min_title: float = 0.90,
|
|
||||||
) -> dict[str, int]:
|
|
||||||
"""Batch: wzbogać do `limit` filmów. Commit per-film (jeden błąd nie cofa reszty).
|
|
||||||
`session_factory` = kontekstowy scope (app.db.session_scope)."""
|
|
||||||
from app.ingest import get_or_create_source
|
|
||||||
from app.models.source import SourceKind
|
|
||||||
|
|
||||||
connector = TPDBConnector()
|
|
||||||
counters = {"seen": 0, "enriched": 0, "merged": 0, "no_match": 0, "skip": 0, "errors": 0}
|
|
||||||
|
|
||||||
with session_factory() as session:
|
|
||||||
src = get_or_create_source(session, kind=SourceKind.tpdb, name="tpdb")
|
|
||||||
source_id = src.id
|
|
||||||
session.commit()
|
|
||||||
movies = _candidate_movies(session, source_id=source_id, limit=limit)
|
|
||||||
movie_ids = [m.id for m in movies]
|
|
||||||
|
|
||||||
log.info("tpdb-movie-enrich: %d candidate movies", len(movie_ids))
|
|
||||||
for mid in movie_ids:
|
|
||||||
counters["seen"] += 1
|
|
||||||
try:
|
|
||||||
with session_factory() as session:
|
|
||||||
movie = session.get(Movie, mid)
|
|
||||||
if movie is None:
|
|
||||||
continue
|
|
||||||
outcome = enrich_movie(
|
|
||||||
session, movie, connector=connector, source_id=source_id, min_title=min_title
|
|
||||||
)
|
|
||||||
session.commit()
|
|
||||||
counters[outcome] = counters.get(outcome, 0) + 1
|
|
||||||
except Exception as e: # pragma: no cover - defensywnie, jeden film nie wywala batcha
|
|
||||||
counters["errors"] += 1
|
|
||||||
log.warning("tpdb-movie-enrich failed for %s: %s", mid, e)
|
|
||||||
|
|
||||||
log.info("tpdb-movie-enrich done: %s", counters)
|
|
||||||
return counters
|
|
||||||
|
|
@ -38,7 +38,6 @@ from app.extractors.tubes import (
|
||||||
porn00,
|
porn00,
|
||||||
porntrex,
|
porntrex,
|
||||||
sxyprn,
|
sxyprn,
|
||||||
watchporn,
|
|
||||||
xhamster,
|
xhamster,
|
||||||
yespornvip,
|
yespornvip,
|
||||||
)
|
)
|
||||||
|
|
@ -97,9 +96,7 @@ _REGISTRY: dict[str, Callable[[str], list[StreamSource] | None]] = {
|
||||||
# Aggregator tubes — generic embed-iframe → hoster unpacker
|
# Aggregator tubes — generic embed-iframe → hoster unpacker
|
||||||
"latestpornvideocom": latestpornvideo.extract,
|
"latestpornvideocom": latestpornvideo.extract,
|
||||||
"xmoviesforyoucom": _embed_iframe.extract,
|
"xmoviesforyoucom": _embed_iframe.extract,
|
||||||
# watchporn — 2026-07-02 przebudowany na KVS (DoodStream-CAPTCHA zniknął).
|
"watchporn": _embed_iframe.extract,
|
||||||
# flashvars get_file direct mp4, same-session 302 resolve, token nie IP-bound.
|
|
||||||
"watchporn": watchporn.extract,
|
|
||||||
"siskavideo": _embed_iframe.extract,
|
"siskavideo": _embed_iframe.extract,
|
||||||
"porn4dayspw": _embed_iframe.extract,
|
"porn4dayspw": _embed_iframe.extract,
|
||||||
"porndishcom": _embed_iframe.extract,
|
"porndishcom": _embed_iframe.extract,
|
||||||
|
|
|
||||||
|
|
@ -283,15 +283,6 @@ def _extract_direct_stream_urls(page_html: str, page_url: str) -> list[StreamSou
|
||||||
# i embed pages z .mp4 suffix (playmogo/dood /d/ — to HTML, nie video).
|
# i embed pages z .mp4 suffix (playmogo/dood /d/ — to HTML, nie video).
|
||||||
if _NOT_DIRECT_STREAM_RE.search(url):
|
if _NOT_DIRECT_STREAM_RE.search(url):
|
||||||
continue
|
continue
|
||||||
# KVS false-positives (report c156e4b7/8db71220, watchporn przez _embed_iframe):
|
|
||||||
# preview_/videos_screenshots to krótkie trailery (404), a surowy get_file/ bez
|
|
||||||
# podpisanego query-tokena daje 403 — kt_player dokłada token z license_code (patrz
|
|
||||||
# natywny watchporn.extract). Żadne z nich to pełne wideo → pomijamy.
|
|
||||||
low = url.lower()
|
|
||||||
if "videos_screenshots/" in low or "/preview_" in low:
|
|
||||||
continue
|
|
||||||
if "/get_file/" in low and "?" not in url:
|
|
||||||
continue
|
|
||||||
# Quality wykrycie z nazwy pliku
|
# Quality wykrycie z nazwy pliku
|
||||||
q_int = 0
|
q_int = 0
|
||||||
q_label = "mp4"
|
q_label = "mp4"
|
||||||
|
|
|
||||||
|
|
@ -1,125 +0,0 @@
|
||||||
"""watchporn.to — KVS engine direct stream extractor (re-enabled 2026-07-02).
|
|
||||||
|
|
||||||
Site przebudowany na KVS: detail page ma flashvars `video_url`/`video_alt_url*`,
|
|
||||||
każdy to `get_file/<srv>/<token>/<path>_<q>p.mp4/?v-acctoken=...`. Ten sam wzorzec co
|
|
||||||
porntrex. get_file 302 → finalny CDN url; resolvujemy same-session (token bywa
|
|
||||||
cookie/session-bound) i oddajemy finalny url per jakość. Token nie IP-bound
|
|
||||||
(cross-IP 206 z VPS zweryfikowane 2026-07-02) → mobile gra direct, zero proxy.
|
|
||||||
Wcześniejszy DoodStream-CAPTCHA zniknął.
|
|
||||||
"""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import logging
|
|
||||||
import re
|
|
||||||
import time
|
|
||||||
|
|
||||||
from app.extractors._fetch import _DEFAULT_IMPERSONATE, _DEFAULT_UA, _HAS_CURL_CFFI, fetch_tube_html
|
|
||||||
from app.extractors._models import HosterDead, StreamSource
|
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
_BASE = "https://watchporn.to"
|
|
||||||
|
|
||||||
_DEAD_RE = re.compile(
|
|
||||||
r"this video (?:was|has been) deleted|video (?:was|has been) removed"
|
|
||||||
r"|no longer available|video is unavailable",
|
|
||||||
re.IGNORECASE,
|
|
||||||
)
|
|
||||||
_URL_RE = re.compile(
|
|
||||||
r"(video(?:_alt)?_url\d*)\s*:\s*'(https?://[^']+/get_file/[^']+)'",
|
|
||||||
re.IGNORECASE,
|
|
||||||
)
|
|
||||||
_TEXT_RE = re.compile(
|
|
||||||
r"(video(?:_alt)?_url\d*)_text\s*:\s*'([^']*)'",
|
|
||||||
re.IGNORECASE,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _quality_rank(label: str | None) -> int:
|
|
||||||
if not label:
|
|
||||||
return -1
|
|
||||||
m = re.search(r"(\d{3,4})\s*p", label, re.IGNORECASE)
|
|
||||||
return int(m.group(1)) if m else -1
|
|
||||||
|
|
||||||
|
|
||||||
def _resolve_get_file(session, get_file_url: str, timeout: float) -> str | None:
|
|
||||||
sep = "&" if "?" in get_file_url else "?"
|
|
||||||
url = f"{get_file_url}{sep}rnd={int(time.time() * 1000)}"
|
|
||||||
try:
|
|
||||||
r = session.get(
|
|
||||||
url,
|
|
||||||
timeout=timeout,
|
|
||||||
allow_redirects=True,
|
|
||||||
stream=True,
|
|
||||||
headers={"Referer": _BASE + "/", "Range": "bytes=0-1"},
|
|
||||||
)
|
|
||||||
final = str(r.url)
|
|
||||||
status = r.status_code
|
|
||||||
r.close()
|
|
||||||
except Exception as e:
|
|
||||||
log.info("watchporn: get_file resolve failed (%s): %s", get_file_url[:60], e)
|
|
||||||
return None
|
|
||||||
if status >= 400 or "/get_file/" in final:
|
|
||||||
log.info("watchporn: get_file resolve bad status=%s final=%s", status, final[:70])
|
|
||||||
return None
|
|
||||||
return final
|
|
||||||
|
|
||||||
|
|
||||||
def extract(page_url: str, *, timeout: float = 60.0) -> list[StreamSource] | None:
|
|
||||||
session = None
|
|
||||||
if _HAS_CURL_CFFI:
|
|
||||||
from curl_cffi import requests as _cf_requests
|
|
||||||
session = _cf_requests.Session(impersonate=_DEFAULT_IMPERSONATE)
|
|
||||||
try:
|
|
||||||
resp = session.get(
|
|
||||||
page_url,
|
|
||||||
headers={"User-Agent": _DEFAULT_UA, "Accept": "text/html,application/xhtml+xml"},
|
|
||||||
timeout=timeout,
|
|
||||||
allow_redirects=True,
|
|
||||||
)
|
|
||||||
html_text = resp.text if resp.status_code < 400 else ""
|
|
||||||
except Exception as e:
|
|
||||||
log.info("watchporn: page fetch failed %s: %s", page_url, e)
|
|
||||||
html_text = ""
|
|
||||||
if not html_text:
|
|
||||||
html_text = fetch_tube_html(page_url, timeout=timeout)
|
|
||||||
session = None
|
|
||||||
else:
|
|
||||||
html_text = fetch_tube_html(page_url, timeout=timeout)
|
|
||||||
|
|
||||||
if html_text and _DEAD_RE.search(html_text):
|
|
||||||
raise HosterDead(f"watchporn {page_url}: video deleted/removed")
|
|
||||||
|
|
||||||
quality_by_var: dict[str, str] = {}
|
|
||||||
for m in _TEXT_RE.finditer(html_text):
|
|
||||||
quality_by_var[m.group(1).lower()] = m.group(2).strip()
|
|
||||||
|
|
||||||
seen: set[str] = set()
|
|
||||||
result: list[StreamSource] = []
|
|
||||||
for m in _URL_RE.finditer(html_text):
|
|
||||||
var_name = m.group(1).lower()
|
|
||||||
url = m.group(2)
|
|
||||||
if url in seen:
|
|
||||||
continue
|
|
||||||
seen.add(url)
|
|
||||||
quality = quality_by_var.get(var_name)
|
|
||||||
final_link = url
|
|
||||||
if session is not None:
|
|
||||||
resolved = _resolve_get_file(session, url, timeout)
|
|
||||||
if resolved:
|
|
||||||
final_link = resolved
|
|
||||||
result.append(
|
|
||||||
StreamSource(
|
|
||||||
link=final_link,
|
|
||||||
type="mp4",
|
|
||||||
quality=quality or None,
|
|
||||||
referer=_BASE + "/",
|
|
||||||
raw={"mobile_direct_ok": True},
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
if not result:
|
|
||||||
log.info("watchporn: no KVS video_url in flashvars on %s", page_url)
|
|
||||||
return None
|
|
||||||
result.sort(key=lambda s: _quality_rank(s.quality), reverse=True)
|
|
||||||
return result
|
|
||||||
|
|
@ -1,190 +0,0 @@
|
||||||
"""Scalanie dwóch kanonicznych movies w jeden (dedup).
|
|
||||||
|
|
||||||
`keep_id` przejmuje od `drop_id`: external_refs, performers, tags, chapters,
|
|
||||||
playback_sources — z deduplikacją na kluczach złączeń. Potem `drop` Movie jest
|
|
||||||
usuwany. Mirror `scene_merge.merge_scenes` dla movies.
|
|
||||||
|
|
||||||
Używane przez TPDB enrichment: gdy dwa nasze filmy (mirrory paradisehill/dooplay)
|
|
||||||
mapują się na TEN SAM TPDB movie UUID, to ten sam film → merge. `MergeKind.movie`
|
|
||||||
już istnieje w merge_candidates.
|
|
||||||
"""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import logging
|
|
||||||
import uuid
|
|
||||||
from datetime import UTC, datetime
|
|
||||||
|
|
||||||
from sqlalchemy import or_, select, update
|
|
||||||
from sqlalchemy.orm import Session
|
|
||||||
|
|
||||||
from app.models.merge_candidate import MergeCandidate, MergeKind, MergeStatus
|
|
||||||
from app.models.movie import (
|
|
||||||
Movie,
|
|
||||||
MovieChapter,
|
|
||||||
MovieExternalRef,
|
|
||||||
MoviePerformer,
|
|
||||||
MovieTag,
|
|
||||||
)
|
|
||||||
from app.models.movie_playback_source import MoviePlaybackSource
|
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
|
|
||||||
class MovieMergeError(Exception):
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
def merge_movies(
|
|
||||||
session: Session,
|
|
||||||
*,
|
|
||||||
keep_id: uuid.UUID,
|
|
||||||
drop_id: uuid.UUID,
|
|
||||||
resolved_by: str | None = None,
|
|
||||||
) -> Movie:
|
|
||||||
if keep_id == drop_id:
|
|
||||||
raise MovieMergeError("cannot merge movie into itself")
|
|
||||||
keep = session.get(Movie, keep_id)
|
|
||||||
drop = session.get(Movie, drop_id)
|
|
||||||
if keep is None or drop is None:
|
|
||||||
raise MovieMergeError("movie not found")
|
|
||||||
|
|
||||||
_move_external_refs(session, keep_id=keep_id, drop_id=drop_id)
|
|
||||||
_move_performers(session, keep_id=keep_id, drop_id=drop_id)
|
|
||||||
_move_tags(session, keep_id=keep_id, drop_id=drop_id)
|
|
||||||
_move_chapters(session, keep_id=keep_id, drop_id=drop_id)
|
|
||||||
_move_playback_sources(session, keep_id=keep_id, drop_id=drop_id)
|
|
||||||
_coalesce_canonical_fields(keep, drop)
|
|
||||||
|
|
||||||
session.delete(drop)
|
|
||||||
session.flush()
|
|
||||||
_close_pending_candidates(session, movie_id=drop_id, resolved_by=resolved_by)
|
|
||||||
log.info("merged movie %s ← %s", keep_id, drop_id)
|
|
||||||
return keep
|
|
||||||
|
|
||||||
|
|
||||||
# ---- helpery --------------------------------------------------------------
|
|
||||||
|
|
||||||
def _move_external_refs(session: Session, *, keep_id: uuid.UUID, drop_id: uuid.UUID) -> None:
|
|
||||||
for ref in session.execute(
|
|
||||||
select(MovieExternalRef).where(MovieExternalRef.movie_id == drop_id)
|
|
||||||
).scalars().all():
|
|
||||||
clash = session.execute(
|
|
||||||
select(MovieExternalRef).where(
|
|
||||||
MovieExternalRef.source_id == ref.source_id,
|
|
||||||
MovieExternalRef.external_id == ref.external_id,
|
|
||||||
MovieExternalRef.movie_id == keep_id,
|
|
||||||
)
|
|
||||||
).scalar_one_or_none()
|
|
||||||
if clash is not None:
|
|
||||||
session.delete(ref)
|
|
||||||
else:
|
|
||||||
ref.movie_id = keep_id
|
|
||||||
|
|
||||||
|
|
||||||
def _move_performers(session: Session, *, keep_id: uuid.UUID, drop_id: uuid.UUID) -> None:
|
|
||||||
for link in session.execute(
|
|
||||||
select(MoviePerformer).where(MoviePerformer.movie_id == drop_id)
|
|
||||||
).scalars().all():
|
|
||||||
clash = session.execute(
|
|
||||||
select(MoviePerformer).where(
|
|
||||||
MoviePerformer.movie_id == keep_id,
|
|
||||||
MoviePerformer.performer_id == link.performer_id,
|
|
||||||
)
|
|
||||||
).scalar_one_or_none()
|
|
||||||
if clash is not None:
|
|
||||||
if link.as_alias and not clash.as_alias:
|
|
||||||
clash.as_alias = link.as_alias
|
|
||||||
session.delete(link)
|
|
||||||
else:
|
|
||||||
link.movie_id = keep_id
|
|
||||||
|
|
||||||
|
|
||||||
def _move_tags(session: Session, *, keep_id: uuid.UUID, drop_id: uuid.UUID) -> None:
|
|
||||||
for link in session.execute(
|
|
||||||
select(MovieTag).where(MovieTag.movie_id == drop_id)
|
|
||||||
).scalars().all():
|
|
||||||
clash = session.execute(
|
|
||||||
select(MovieTag).where(
|
|
||||||
MovieTag.movie_id == keep_id, MovieTag.tag_id == link.tag_id
|
|
||||||
)
|
|
||||||
).scalar_one_or_none()
|
|
||||||
if clash is not None:
|
|
||||||
session.delete(link)
|
|
||||||
else:
|
|
||||||
link.movie_id = keep_id
|
|
||||||
|
|
||||||
|
|
||||||
def _move_chapters(session: Session, *, keep_id: uuid.UUID, drop_id: uuid.UUID) -> None:
|
|
||||||
"""Chaptery są lokalne per-movie (UQ movie_id, chapter_index). Kolizja indeksu z
|
|
||||||
keep → keep swoje (kasujemy drop's); brak kolizji → przepięcie."""
|
|
||||||
for ch in session.execute(
|
|
||||||
select(MovieChapter).where(MovieChapter.movie_id == drop_id)
|
|
||||||
).scalars().all():
|
|
||||||
clash = session.execute(
|
|
||||||
select(MovieChapter).where(
|
|
||||||
MovieChapter.movie_id == keep_id,
|
|
||||||
MovieChapter.chapter_index == ch.chapter_index,
|
|
||||||
)
|
|
||||||
).scalar_one_or_none()
|
|
||||||
if clash is not None:
|
|
||||||
session.delete(ch)
|
|
||||||
else:
|
|
||||||
ch.movie_id = keep_id
|
|
||||||
|
|
||||||
|
|
||||||
def _move_playback_sources(session: Session, *, keep_id: uuid.UUID, drop_id: uuid.UUID) -> None:
|
|
||||||
"""Unique (origin, page_url) jest GLOBALNY → drop i keep nie mogą współdzielić
|
|
||||||
tego samego źródła, więc samo przepięcie movie_id nie grozi kolizją."""
|
|
||||||
session.execute(
|
|
||||||
update(MoviePlaybackSource)
|
|
||||||
.where(MoviePlaybackSource.movie_id == drop_id)
|
|
||||||
.values(movie_id=keep_id)
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _coalesce_canonical_fields(keep: Movie, drop: Movie) -> None:
|
|
||||||
"""Wypełnij braki w `keep` polami z `drop` (bez nadpisywania ustawionych)."""
|
|
||||||
if not keep.description and drop.description:
|
|
||||||
keep.description = drop.description
|
|
||||||
if not keep.duration_sec and drop.duration_sec:
|
|
||||||
keep.duration_sec = drop.duration_sec
|
|
||||||
if not keep.director and drop.director:
|
|
||||||
keep.director = drop.director
|
|
||||||
if not keep.country and drop.country:
|
|
||||||
keep.country = drop.country
|
|
||||||
if not keep.release_date and drop.release_date:
|
|
||||||
keep.release_date = drop.release_date
|
|
||||||
if not keep.release_year and drop.release_year:
|
|
||||||
keep.release_year = drop.release_year
|
|
||||||
if not keep.studio_id and drop.studio_id:
|
|
||||||
keep.studio_id = drop.studio_id
|
|
||||||
if not keep.poster_url and drop.poster_url:
|
|
||||||
keep.poster_url = drop.poster_url
|
|
||||||
if not keep.backdrop_url and drop.backdrop_url:
|
|
||||||
keep.backdrop_url = drop.backdrop_url
|
|
||||||
if keep.rating is None and drop.rating is not None:
|
|
||||||
keep.rating = drop.rating
|
|
||||||
if drop.title and len(drop.title) > len(keep.title or ""):
|
|
||||||
keep.title = drop.title
|
|
||||||
keep.title_normalized = drop.title_normalized
|
|
||||||
# created_at = najwcześniejsze "first seen" (NEW badge correctness, jak w scenach).
|
|
||||||
if drop.created_at and keep.created_at and drop.created_at < keep.created_at:
|
|
||||||
keep.created_at = drop.created_at
|
|
||||||
|
|
||||||
|
|
||||||
def _close_pending_candidates(
|
|
||||||
session: Session, *, movie_id: uuid.UUID, resolved_by: str | None
|
|
||||||
) -> None:
|
|
||||||
session.execute(
|
|
||||||
update(MergeCandidate)
|
|
||||||
.where(
|
|
||||||
MergeCandidate.kind == MergeKind.movie,
|
|
||||||
MergeCandidate.status == MergeStatus.pending,
|
|
||||||
or_(MergeCandidate.left_id == movie_id, MergeCandidate.right_id == movie_id),
|
|
||||||
)
|
|
||||||
.values(
|
|
||||||
status=MergeStatus.rejected,
|
|
||||||
resolved_at=datetime.now(UTC),
|
|
||||||
resolved_by=resolved_by or "auto:movie_dropped",
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
@ -349,25 +349,6 @@ def _job_performer_continuous(refresh_after_days: int) -> None:
|
||||||
log.exception("[scheduler] performer-continuous failed")
|
log.exception("[scheduler] performer-continuous failed")
|
||||||
|
|
||||||
|
|
||||||
def _job_tpdb_movie_enrich(batch: int) -> None:
|
|
||||||
"""Wzbogaca filmy metadanymi z TPDB /movies: obsada, kategorie (tagi), studio,
|
|
||||||
reżyser + kanoniczny UUID do dedupu mirrorów. Batch najświeższych grywalnych
|
|
||||||
filmów bez obsady/studia per run. paradisehill (primary) prawie nie ma obsady,
|
|
||||||
więc to domyka największą lukę. Patrz app/enrich/tpdb_movies.py."""
|
|
||||||
log.info("[scheduler] tpdb-movie-enrich starting (batch=%d)", batch)
|
|
||||||
try:
|
|
||||||
from app.db import session_scope
|
|
||||||
from app.enrich.tpdb_movies import run_tpdb_movie_enrich
|
|
||||||
|
|
||||||
_run_with_timeout(
|
|
||||||
lambda: run_tpdb_movie_enrich(session_scope, limit=batch),
|
|
||||||
label="tpdb-movie-enrich",
|
|
||||||
)
|
|
||||||
log.info("[scheduler] tpdb-movie-enrich done")
|
|
||||||
except Exception:
|
|
||||||
log.exception("[scheduler] tpdb-movie-enrich failed")
|
|
||||||
|
|
||||||
|
|
||||||
def build_scheduler(cfg: dict[str, Any]) -> BlockingScheduler:
|
def build_scheduler(cfg: dict[str, Any]) -> BlockingScheduler:
|
||||||
"""Buduje scheduler na podstawie cfg dictu.
|
"""Buduje scheduler na podstawie cfg dictu.
|
||||||
|
|
||||||
|
|
@ -445,21 +426,6 @@ def build_scheduler(cfg: dict[str, Any]) -> BlockingScheduler:
|
||||||
)
|
)
|
||||||
log.info("scheduler: bulk-dedup performers every %dh", cfg["bulk_dedup_hours"])
|
log.info("scheduler: bulk-dedup performers every %dh", cfg["bulk_dedup_hours"])
|
||||||
|
|
||||||
if cfg.get("tpdb_movie_enrich_hours"):
|
|
||||||
_tme_batch = cfg.get("tpdb_movie_enrich_batch") or 200
|
|
||||||
sched.add_job(
|
|
||||||
lambda: _job_tpdb_movie_enrich(_tme_batch),
|
|
||||||
IntervalTrigger(hours=cfg["tpdb_movie_enrich_hours"], start_date=INTERVAL_ANCHOR),
|
|
||||||
id="tpdb_movie_enrich",
|
|
||||||
replace_existing=True,
|
|
||||||
max_instances=1,
|
|
||||||
coalesce=True,
|
|
||||||
)
|
|
||||||
log.info(
|
|
||||||
"scheduler: tpdb-movie-enrich every %dh (batch=%d)",
|
|
||||||
cfg["tpdb_movie_enrich_hours"], _tme_batch,
|
|
||||||
)
|
|
||||||
|
|
||||||
if cfg.get("thumb_dedup_hours"):
|
if cfg.get("thumb_dedup_hours"):
|
||||||
sched.add_job(
|
sched.add_job(
|
||||||
_job_thumb_asset_dedup,
|
_job_thumb_asset_dedup,
|
||||||
|
|
|
||||||
|
|
@ -207,10 +207,6 @@ def run_forever() -> int:
|
||||||
# Bulk-dedup performers — safety net dla duplikatów które resolver
|
# Bulk-dedup performers — safety net dla duplikatów które resolver
|
||||||
# pominął (np. freshporno scen przed fixem release_date). Run 12h.
|
# pominął (np. freshporno scen przed fixem release_date). Run 12h.
|
||||||
"bulk_dedup_hours": getattr(settings, "sched_bulk_dedup_hours", 12) or None,
|
"bulk_dedup_hours": getattr(settings, "sched_bulk_dedup_hours", 12) or None,
|
||||||
# TPDB movie enrichment — obsada/kategorie/studio/reżyser + UUID do dedupu.
|
|
||||||
# paradisehill prawie nie ma obsady; batch grywalnych filmów bez obsady/studia.
|
|
||||||
"tpdb_movie_enrich_hours": getattr(settings, "sched_tpdb_movie_enrich_hours", 6) or None,
|
|
||||||
"tpdb_movie_enrich_batch": getattr(settings, "tpdb_movie_enrich_batch", 200),
|
|
||||||
# Thumb-asset dedup — hdporn.gg/fullmovies.xxx same-video-różne-tytuły (reports
|
# Thumb-asset dedup — hdporn.gg/fullmovies.xxx same-video-różne-tytuły (reports
|
||||||
# 205b17d9/5a2944cb). bulk_dedup tego nie łapie; dupy odrastają przy re-ingeście.
|
# 205b17d9/5a2944cb). bulk_dedup tego nie łapie; dupy odrastają przy re-ingeście.
|
||||||
"thumb_dedup_hours": getattr(settings, "sched_thumb_dedup_hours", 12) or None,
|
"thumb_dedup_hours": getattr(settings, "sched_thumb_dedup_hours", 12) or None,
|
||||||
|
|
|
||||||
|
|
@ -16,13 +16,6 @@ export type ChangelogEntry = {
|
||||||
};
|
};
|
||||||
|
|
||||||
export const CHANGELOG: ChangelogEntry[] = [
|
export const CHANGELOG: ChangelogEntry[] = [
|
||||||
{
|
|
||||||
id: '2026-07-03',
|
|
||||||
date: 'July 2026',
|
|
||||||
items: [
|
|
||||||
'Movie posters show up again (freeomovie), and more of its sources actually play (myvidplay).',
|
|
||||||
],
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
id: '2026-07-02b',
|
id: '2026-07-02b',
|
||||||
date: 'July 2026',
|
date: 'July 2026',
|
||||||
|
|
|
||||||
|
|
@ -38,9 +38,6 @@ const DOOD_HOSTS = new Set([
|
||||||
'ds2play.com', 'ds2video.com',
|
'ds2play.com', 'ds2video.com',
|
||||||
'd000d.com', 'd0o0d.com', 'do0od.com', 'do7go.com',
|
'd000d.com', 'd0o0d.com', 'do0od.com', 'do7go.com',
|
||||||
'dooood.com', 'd0000d.com',
|
'dooood.com', 'd0000d.com',
|
||||||
// myvidplay.com: 301 -> playmogo.com (doodcdn clone). Najczęstszy hoster freeomovie
|
|
||||||
// (63 filmy); bez tego isDoodStream() false -> WebView -> nie gra (report 8f8c10c0).
|
|
||||||
'myvidplay.com',
|
|
||||||
]);
|
]);
|
||||||
|
|
||||||
const UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36';
|
const UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36';
|
||||||
|
|
|
||||||
|
|
@ -33,12 +33,12 @@ export function DonateScreen() {
|
||||||
<Text style={styles.footnoteTitle}>Why crypto?</Text>
|
<Text style={styles.footnoteTitle}>Why crypto?</Text>
|
||||||
<Text style={styles.footnoteBody}>
|
<Text style={styles.footnoteBody}>
|
||||||
Mainstream payment processors (Stripe, PayPal, Patreon) refuse adult
|
Mainstream payment processors (Stripe, PayPal, Patreon) refuse adult
|
||||||
projects, even open-source tooling like this one. Crypto is the only
|
projects — even open-source tooling like this one. Crypto is the only
|
||||||
channel that does not require us to identify our donors and does not
|
channel that does not require us to identify our donors and does not
|
||||||
let third parties freeze the project mid-month.
|
let third parties freeze the project mid-month.
|
||||||
</Text>
|
</Text>
|
||||||
<Text style={[styles.footnoteBody, { marginTop: 10 }]}>
|
<Text style={[styles.footnoteBody, { marginTop: 10 }]}>
|
||||||
Monero is privacy-preserving by design; if you care about not being
|
Monero is privacy-preserving by design — if you care about not being
|
||||||
linked to this app on-chain, send XMR. Bitcoin and USDT are public
|
linked to this app on-chain, send XMR. Bitcoin and USDT are public
|
||||||
ledgers; use a fresh wallet if anonymity matters to you.
|
ledgers; use a fresh wallet if anonymity matters to you.
|
||||||
</Text>
|
</Text>
|
||||||
|
|
@ -83,7 +83,7 @@ function CoinCard({ coin }: { coin: DonateCoin }) {
|
||||||
{placeholder ? (
|
{placeholder ? (
|
||||||
<View style={styles.placeholderBox}>
|
<View style={styles.placeholderBox}>
|
||||||
<Text style={styles.placeholderText}>
|
<Text style={styles.placeholderText}>
|
||||||
Address not configured yet, see{' '}
|
Address not configured yet — see{' '}
|
||||||
<Text style={{ color: theme.muted, fontWeight: '700' }}>mobile/src/lib/donate.ts</Text>
|
<Text style={{ color: theme.muted, fontWeight: '700' }}>mobile/src/lib/donate.ts</Text>
|
||||||
</Text>
|
</Text>
|
||||||
</View>
|
</View>
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue