From 385aeb3ddb891dba6a6a1e0999267d0cc9cd3ea7 Mon Sep 17 00:00:00 2001 From: goon-foss Date: Tue, 8 Sep 2026 10:04:01 +0200 Subject: [PATCH] perf(api): lista filmow serializowana wsadowo (7 zapytan/film -> 8 na strone), /sources z agregatem w pamieci na 5 min - GET /movies 371 -> 82 ms, z has_playback 477 -> 209 ms; GET /sources 604 -> 8 ms - VACUUM ANALYZE tabel filmow po tygodniu przemialu (11k martwych wierszy w movies) --- app/api/movies.py | 247 +++++++++++++++++++++++++++------------------ app/api/sources.py | 30 ++++-- 2 files changed, 167 insertions(+), 110 deletions(-) diff --git a/app/api/movies.py b/app/api/movies.py index 24c07fb..76d2933 100644 --- a/app/api/movies.py +++ b/app/api/movies.py @@ -2,6 +2,7 @@ from __future__ import annotations import uuid +from collections import defaultdict from typing import Annotated from fastapi import APIRouter, Depends, HTTPException, Query @@ -132,8 +133,8 @@ def list_movies( base = base.limit(per_page).offset((page - 1) * per_page) - movies = session.execute(base).scalars().all() - items = [_movie_to_out(session, m, device_id=device_id) for m in movies] + movies = list(session.execute(base).scalars().all()) + items = _movies_to_out(session, movies, device_id=device_id) return MovieListOut(items=items, total=total, page=page, per_page=per_page) @@ -225,115 +226,161 @@ def get_movie( def _movie_to_out(session: Session, movie: Movie, *, device_id: str = LEGACY_DEVICE) -> MovieOut: - studio_out: StudioOut | None = None - if movie.studio_id: - studio = session.get(Studio, movie.studio_id) - if studio is not None: - studio_out = StudioOut.model_validate(studio) + return _movies_to_out(session, [movie], device_id=device_id)[0] - performer_rows = session.execute( - select(Performer, MoviePerformer.as_alias) - .join(MoviePerformer, MoviePerformer.performer_id == Performer.id) - .where(MoviePerformer.movie_id == movie.id) + +def _movies_to_out( + session: Session, movies: list[Movie], *, device_id: str = LEGACY_DEVICE +) -> list[MovieOut]: + """Serializacja wsadowa: po jednym zapytaniu na relacje dla calej strony listy. + + Wczesniej kazdy film robil 7 zapytan (studio, obsada, tagi, rozdzialy, refy, + zrodla, ulubione + postep), czyli ~210 na strone 30 filmow i 350 ms na + GET /movies (Sentry: N+1 Query list_movies, pomiar 2026-09-08). Logika per film + jest ta sama co w starym `_movie_to_out`, tylko dane przychodza z map. + """ + if not movies: + return [] + ids = [m.id for m in movies] + + studio_ids = {m.studio_id for m in movies if m.studio_id} + studios: dict = {} + if studio_ids: + studios = { + st.id: st for st in session.execute(select(Studio).where(Studio.id.in_(studio_ids))).scalars() + } + + perf_by: dict = defaultdict(list) + for mid, perf, alias in session.execute( + select(MoviePerformer.movie_id, Performer, MoviePerformer.as_alias) + .join(Performer, MoviePerformer.performer_id == Performer.id) + .where(MoviePerformer.movie_id.in_(ids)) .order_by(MoviePerformer.position.asc().nulls_last()) - ).all() - performers = [ - PerformerOut( - id=p.id, - canonical_name=p.canonical_name, - slug=p.slug, - gender=p.gender.value if p.gender else None, - as_alias=alias, - ) - for p, alias in performer_rows - ] + ).all(): + perf_by[mid].append((perf, alias)) - tag_rows = session.execute( - select(Tag).join(MovieTag, MovieTag.tag_id == Tag.id) - .where(MovieTag.movie_id == movie.id) + tags_by: dict = defaultdict(list) + for mid, tag in session.execute( + select(MovieTag.movie_id, Tag) + .join(Tag, MovieTag.tag_id == Tag.id) + .where(MovieTag.movie_id.in_(ids)) .order_by(Tag.name.asc()) - ).scalars().all() - tags = [TagOut.model_validate(t) for t in tag_rows] + ).all(): + tags_by[mid].append(tag) - chapter_rows = session.execute( - select(MovieChapter).where(MovieChapter.movie_id == movie.id) - .order_by(MovieChapter.chapter_index.asc()) - ).scalars().all() - chapters = [MovieChapterOut.model_validate(c) for c in chapter_rows] + chapters_by: dict = defaultdict(list) + for ch in session.execute( + select(MovieChapter).where(MovieChapter.movie_id.in_(ids)).order_by(MovieChapter.chapter_index.asc()) + ).scalars(): + chapters_by[ch.movie_id].append(ch) - ref_rows = session.execute( + refs_by: dict = defaultdict(list) + for ref, name in session.execute( select(MovieExternalRef, Source.name) .join(Source, Source.id == MovieExternalRef.source_id) - .where(MovieExternalRef.movie_id == movie.id) - ).all() - external_refs = [ - ExternalRefOut( - source=name, - external_id=ref.external_id, - url=ref.url, - last_seen=ref.last_seen, - ) - for ref, name in ref_rows - ] + .where(MovieExternalRef.movie_id.in_(ids)) + ).all(): + refs_by[ref.movie_id].append((ref, name)) - pb_rows = session.execute( + pb_by: dict = defaultdict(list) + for pb in session.execute( select(MoviePlaybackSource) - .where(MoviePlaybackSource.movie_id == movie.id) + .where(MoviePlaybackSource.movie_id.in_(ids)) .where(MoviePlaybackSource.dead_at.is_(None)) .order_by(MoviePlaybackSource.created_at.desc()) - ).scalars().all() - pb_rows = [p for p in pb_rows if p.origin not in _MOVIE_DROP_ORIGINS] - # Bug-report 2026-05-16: raw landing origins (`mangoporn`/`pandamovies`/ - # `streamporn` BEZ `:host`) otwierały WebView z reklamami pełnoekranowymi - # i myliły usera. Ukrywamy raw landing GDY ten sam movie ma co najmniej - # jeden sub-host entry (origin zawiera `:`). Jeśli movie nie ma sub-hosters - # (bo theme HTML się zmienił lub regex nie złapał), zostawiamy landing jako - # last-resort. - has_subhost = any(":" in p.origin for p in pb_rows) - if has_subhost: - pb_rows = [p for p in pb_rows if p.origin not in _MOVIE_LANDING_HIDE] - pb_rows = sorted(pb_rows, key=lambda p: _movie_origin_priority(p.origin)) - # Dedup po realnym celu: 3 mirrory dooplay (mangoporn/pandamovies/streamporn) - # zapisują TEN SAM embed hostera jako osobne wiersze (np. luluvid/e/X z każdego - # mirrora) → film miał ~100 linków, z czego ~połowa to ten sam film (bug-report - # 41ca1fa4). Zostawiamy pierwszy = najwyższy priorytet (sort wyżej). - seen_targets: set[str] = set() - deduped: list[MoviePlaybackSource] = [] - for p in pb_rows: - target = (p.embed_url or p.stream_url or p.page_url or "").strip() - if target and target in seen_targets: - continue - if target: - seen_targets.add(target) - deduped.append(p) - pb_rows = deduped - playback_sources = [PlaybackSourceOut.model_validate(p) for p in pb_rows] + ).scalars(): + pb_by[pb.movie_id].append(pb) - is_fav = session.get(FavoriteMovie, (device_id, movie.id)) is not None - progress = session.get(MoviePlayProgress, (device_id, movie.id)) - - return MovieOut( - id=movie.id, - title=movie.title, - slug=movie.slug, - release_year=movie.release_year, - release_date=movie.release_date, - duration_sec=movie.duration_sec, - description=movie.description, - director=movie.director, - country=movie.country, - rating=movie.rating, - poster_url=movie.poster_url, - backdrop_url=movie.backdrop_url, - studio=studio_out, - performers=performers, - tags=tags, - chapters=chapters, - external_refs=external_refs, - playback_sources=playback_sources, - created_at=movie.created_at, - is_favorite=is_fav, - last_played_at=progress.last_played_at if progress else None, - finished=progress.finished if progress else False, - position_sec=progress.position_sec if progress else 0, + favs = set( + session.execute( + select(FavoriteMovie.movie_id) + .where(FavoriteMovie.device_id == device_id) + .where(FavoriteMovie.movie_id.in_(ids)) + ).scalars() ) + progress_by = { + pr.movie_id: pr + for pr in session.execute( + select(MoviePlayProgress) + .where(MoviePlayProgress.device_id == device_id) + .where(MoviePlayProgress.movie_id.in_(ids)) + ).scalars() + } + + out: list[MovieOut] = [] + for movie in movies: + studio = studios.get(movie.studio_id) if movie.studio_id else None + studio_out = StudioOut.model_validate(studio) if studio is not None else None + performers = [ + PerformerOut( + id=perf.id, + canonical_name=perf.canonical_name, + slug=perf.slug, + gender=perf.gender.value if perf.gender else None, + as_alias=alias, + ) + for perf, alias in perf_by.get(movie.id, []) + ] + tags = [TagOut.model_validate(t) for t in tags_by.get(movie.id, [])] + chapters = [MovieChapterOut.model_validate(c) for c in chapters_by.get(movie.id, [])] + external_refs = [ + ExternalRefOut(source=name, external_id=ref.external_id, url=ref.url, last_seen=ref.last_seen) + for ref, name in refs_by.get(movie.id, []) + ] + pb_rows = list(pb_by.get(movie.id, [])) + pb_rows = [p for p in pb_rows if p.origin not in _MOVIE_DROP_ORIGINS] + # Bug-report 2026-05-16: raw landing origins (`mangoporn`/`pandamovies`/ + # `streamporn` BEZ `:host`) otwierały WebView z reklamami pełnoekranowymi + # i myliły usera. Ukrywamy raw landing GDY ten sam movie ma co najmniej + # jeden sub-host entry (origin zawiera `:`). Jeśli movie nie ma sub-hosters + # (bo theme HTML się zmienił lub regex nie złapał), zostawiamy landing jako + # last-resort. + has_subhost = any(":" in p.origin for p in pb_rows) + if has_subhost: + pb_rows = [p for p in pb_rows if p.origin not in _MOVIE_LANDING_HIDE] + pb_rows = sorted(pb_rows, key=lambda p: _movie_origin_priority(p.origin)) + # Dedup po realnym celu: 3 mirrory dooplay (mangoporn/pandamovies/streamporn) + # zapisują TEN SAM embed hostera jako osobne wiersze (np. luluvid/e/X z każdego + # mirrora) → film miał ~100 linków, z czego ~połowa to ten sam film (bug-report + # 41ca1fa4). Zostawiamy pierwszy = najwyższy priorytet (sort wyżej). + seen_targets: set[str] = set() + deduped: list[MoviePlaybackSource] = [] + for p in pb_rows: + target = (p.embed_url or p.stream_url or p.page_url or "").strip() + if target and target in seen_targets: + continue + if target: + seen_targets.add(target) + deduped.append(p) + pb_rows = deduped + playback_sources = [PlaybackSourceOut.model_validate(p) for p in pb_rows] + + progress = progress_by.get(movie.id) + out.append( + MovieOut( + id=movie.id, + title=movie.title, + slug=movie.slug, + release_year=movie.release_year, + release_date=movie.release_date, + duration_sec=movie.duration_sec, + description=movie.description, + director=movie.director, + country=movie.country, + rating=movie.rating, + poster_url=movie.poster_url, + backdrop_url=movie.backdrop_url, + studio=studio_out, + performers=performers, + tags=tags, + chapters=chapters, + external_refs=external_refs, + playback_sources=playback_sources, + created_at=movie.created_at, + is_favorite=movie.id in favs, + last_played_at=progress.last_played_at if progress else None, + finished=progress.finished if progress else False, + position_sec=progress.position_sec if progress else 0, + ) + ) + return out diff --git a/app/api/sources.py b/app/api/sources.py index 27b81a4..c273680 100644 --- a/app/api/sources.py +++ b/app/api/sources.py @@ -16,6 +16,7 @@ from __future__ import annotations import logging import os import re +import time from datetime import datetime from pathlib import Path from typing import Annotated @@ -85,6 +86,8 @@ class SourceOut(BaseModel): _ICON_DIR = Path(__file__).resolve().parent.parent / "static" / "site-icons" +_SOURCES_TTL = 300.0 +_SOURCES_CACHE: dict = {"at": 0.0, "rows": None} def _icon_url(sitetag: str) -> str | None: @@ -139,16 +142,23 @@ def list_sources( Filter: `origin LIKE 'tube:%'` (drop canonical:* TPDB trailery to inna semantyka). """ - rows = session.execute( - select( - PlaybackSource.origin, - func.count(PlaybackSource.id).label("scene_count"), - func.max(PlaybackSource.last_seen_at).label("last_scraped_at"), - ) - .where(PlaybackSource.dead_at.is_(None)) - .where(PlaybackSource.origin.like("tube:%")) - .group_by(PlaybackSource.origin) - ).all() + # Agregat idzie po 4,1 mln zywych zrodel (600 ms na kazde wejscie w Sites, Sentry: + # Slow DB Query list_sources). Liczby zmieniaja sie o promile na godzine, wiec + # wynik zyje w pamieci procesu przez _SOURCES_TTL sekund. + now = time.monotonic() + if _SOURCES_CACHE["rows"] is None or now - _SOURCES_CACHE["at"] > _SOURCES_TTL: + _SOURCES_CACHE["rows"] = session.execute( + select( + PlaybackSource.origin, + func.count(PlaybackSource.id).label("scene_count"), + func.max(PlaybackSource.last_seen_at).label("last_scraped_at"), + ) + .where(PlaybackSource.dead_at.is_(None)) + .where(PlaybackSource.origin.like("tube:%")) + .group_by(PlaybackSource.origin) + ).all() + _SOURCES_CACHE["at"] = now + rows = _SOURCES_CACHE["rows"] # Oceny z source_stats (policzone offline przez run_source_stats). Origin → row. stats = {s.origin: s for s in session.execute(select(SourceStats)).scalars().all()}