perf(api): lista filmow serializowana wsadowo (7 zapytan/film -> 8 na strone), /sources z agregatem w pamieci na 5 min
- GET /movies 371 -> 82 ms, z has_playback 477 -> 209 ms; GET /sources 604 -> 8 ms - VACUUM ANALYZE tabel filmow po tygodniu przemialu (11k martwych wierszy w movies)
This commit is contained in:
goon-foss
parent
44842478ae
commit
385aeb3ddb
2 changed files with 167 additions and 110 deletions
|
|
@ -2,6 +2,7 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from collections import defaultdict
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
|
|
@ -132,8 +133,8 @@ def list_movies(
|
|||
|
||||
base = base.limit(per_page).offset((page - 1) * per_page)
|
||||
|
||||
movies = session.execute(base).scalars().all()
|
||||
items = [_movie_to_out(session, m, device_id=device_id) for m in movies]
|
||||
movies = list(session.execute(base).scalars().all())
|
||||
items = _movies_to_out(session, movies, device_id=device_id)
|
||||
|
||||
return MovieListOut(items=items, total=total, page=page, per_page=per_page)
|
||||
|
||||
|
|
@ -225,63 +226,108 @@ def get_movie(
|
|||
|
||||
|
||||
def _movie_to_out(session: Session, movie: Movie, *, device_id: str = LEGACY_DEVICE) -> MovieOut:
|
||||
studio_out: StudioOut | None = None
|
||||
if movie.studio_id:
|
||||
studio = session.get(Studio, movie.studio_id)
|
||||
if studio is not None:
|
||||
studio_out = StudioOut.model_validate(studio)
|
||||
return _movies_to_out(session, [movie], device_id=device_id)[0]
|
||||
|
||||
performer_rows = session.execute(
|
||||
select(Performer, MoviePerformer.as_alias)
|
||||
.join(MoviePerformer, MoviePerformer.performer_id == Performer.id)
|
||||
.where(MoviePerformer.movie_id == movie.id)
|
||||
|
||||
def _movies_to_out(
|
||||
session: Session, movies: list[Movie], *, device_id: str = LEGACY_DEVICE
|
||||
) -> list[MovieOut]:
|
||||
"""Serializacja wsadowa: po jednym zapytaniu na relacje dla calej strony listy.
|
||||
|
||||
Wczesniej kazdy film robil 7 zapytan (studio, obsada, tagi, rozdzialy, refy,
|
||||
zrodla, ulubione + postep), czyli ~210 na strone 30 filmow i 350 ms na
|
||||
GET /movies (Sentry: N+1 Query list_movies, pomiar 2026-09-08). Logika per film
|
||||
jest ta sama co w starym `_movie_to_out`, tylko dane przychodza z map.
|
||||
"""
|
||||
if not movies:
|
||||
return []
|
||||
ids = [m.id for m in movies]
|
||||
|
||||
studio_ids = {m.studio_id for m in movies if m.studio_id}
|
||||
studios: dict = {}
|
||||
if studio_ids:
|
||||
studios = {
|
||||
st.id: st for st in session.execute(select(Studio).where(Studio.id.in_(studio_ids))).scalars()
|
||||
}
|
||||
|
||||
perf_by: dict = defaultdict(list)
|
||||
for mid, perf, alias in session.execute(
|
||||
select(MoviePerformer.movie_id, Performer, MoviePerformer.as_alias)
|
||||
.join(Performer, MoviePerformer.performer_id == Performer.id)
|
||||
.where(MoviePerformer.movie_id.in_(ids))
|
||||
.order_by(MoviePerformer.position.asc().nulls_last())
|
||||
).all()
|
||||
performers = [
|
||||
PerformerOut(
|
||||
id=p.id,
|
||||
canonical_name=p.canonical_name,
|
||||
slug=p.slug,
|
||||
gender=p.gender.value if p.gender else None,
|
||||
as_alias=alias,
|
||||
)
|
||||
for p, alias in performer_rows
|
||||
]
|
||||
).all():
|
||||
perf_by[mid].append((perf, alias))
|
||||
|
||||
tag_rows = session.execute(
|
||||
select(Tag).join(MovieTag, MovieTag.tag_id == Tag.id)
|
||||
.where(MovieTag.movie_id == movie.id)
|
||||
tags_by: dict = defaultdict(list)
|
||||
for mid, tag in session.execute(
|
||||
select(MovieTag.movie_id, Tag)
|
||||
.join(Tag, MovieTag.tag_id == Tag.id)
|
||||
.where(MovieTag.movie_id.in_(ids))
|
||||
.order_by(Tag.name.asc())
|
||||
).scalars().all()
|
||||
tags = [TagOut.model_validate(t) for t in tag_rows]
|
||||
).all():
|
||||
tags_by[mid].append(tag)
|
||||
|
||||
chapter_rows = session.execute(
|
||||
select(MovieChapter).where(MovieChapter.movie_id == movie.id)
|
||||
.order_by(MovieChapter.chapter_index.asc())
|
||||
).scalars().all()
|
||||
chapters = [MovieChapterOut.model_validate(c) for c in chapter_rows]
|
||||
chapters_by: dict = defaultdict(list)
|
||||
for ch in session.execute(
|
||||
select(MovieChapter).where(MovieChapter.movie_id.in_(ids)).order_by(MovieChapter.chapter_index.asc())
|
||||
).scalars():
|
||||
chapters_by[ch.movie_id].append(ch)
|
||||
|
||||
ref_rows = session.execute(
|
||||
refs_by: dict = defaultdict(list)
|
||||
for ref, name in session.execute(
|
||||
select(MovieExternalRef, Source.name)
|
||||
.join(Source, Source.id == MovieExternalRef.source_id)
|
||||
.where(MovieExternalRef.movie_id == movie.id)
|
||||
).all()
|
||||
external_refs = [
|
||||
ExternalRefOut(
|
||||
source=name,
|
||||
external_id=ref.external_id,
|
||||
url=ref.url,
|
||||
last_seen=ref.last_seen,
|
||||
)
|
||||
for ref, name in ref_rows
|
||||
]
|
||||
.where(MovieExternalRef.movie_id.in_(ids))
|
||||
).all():
|
||||
refs_by[ref.movie_id].append((ref, name))
|
||||
|
||||
pb_rows = session.execute(
|
||||
pb_by: dict = defaultdict(list)
|
||||
for pb in session.execute(
|
||||
select(MoviePlaybackSource)
|
||||
.where(MoviePlaybackSource.movie_id == movie.id)
|
||||
.where(MoviePlaybackSource.movie_id.in_(ids))
|
||||
.where(MoviePlaybackSource.dead_at.is_(None))
|
||||
.order_by(MoviePlaybackSource.created_at.desc())
|
||||
).scalars().all()
|
||||
).scalars():
|
||||
pb_by[pb.movie_id].append(pb)
|
||||
|
||||
favs = set(
|
||||
session.execute(
|
||||
select(FavoriteMovie.movie_id)
|
||||
.where(FavoriteMovie.device_id == device_id)
|
||||
.where(FavoriteMovie.movie_id.in_(ids))
|
||||
).scalars()
|
||||
)
|
||||
progress_by = {
|
||||
pr.movie_id: pr
|
||||
for pr in session.execute(
|
||||
select(MoviePlayProgress)
|
||||
.where(MoviePlayProgress.device_id == device_id)
|
||||
.where(MoviePlayProgress.movie_id.in_(ids))
|
||||
).scalars()
|
||||
}
|
||||
|
||||
out: list[MovieOut] = []
|
||||
for movie in movies:
|
||||
studio = studios.get(movie.studio_id) if movie.studio_id else None
|
||||
studio_out = StudioOut.model_validate(studio) if studio is not None else None
|
||||
performers = [
|
||||
PerformerOut(
|
||||
id=perf.id,
|
||||
canonical_name=perf.canonical_name,
|
||||
slug=perf.slug,
|
||||
gender=perf.gender.value if perf.gender else None,
|
||||
as_alias=alias,
|
||||
)
|
||||
for perf, alias in perf_by.get(movie.id, [])
|
||||
]
|
||||
tags = [TagOut.model_validate(t) for t in tags_by.get(movie.id, [])]
|
||||
chapters = [MovieChapterOut.model_validate(c) for c in chapters_by.get(movie.id, [])]
|
||||
external_refs = [
|
||||
ExternalRefOut(source=name, external_id=ref.external_id, url=ref.url, last_seen=ref.last_seen)
|
||||
for ref, name in refs_by.get(movie.id, [])
|
||||
]
|
||||
pb_rows = list(pb_by.get(movie.id, []))
|
||||
pb_rows = [p for p in pb_rows if p.origin not in _MOVIE_DROP_ORIGINS]
|
||||
# Bug-report 2026-05-16: raw landing origins (`mangoporn`/`pandamovies`/
|
||||
# `streamporn` BEZ `:host`) otwierały WebView z reklamami pełnoekranowymi
|
||||
|
|
@ -309,10 +355,9 @@ def _movie_to_out(session: Session, movie: Movie, *, device_id: str = LEGACY_DEV
|
|||
pb_rows = deduped
|
||||
playback_sources = [PlaybackSourceOut.model_validate(p) for p in pb_rows]
|
||||
|
||||
is_fav = session.get(FavoriteMovie, (device_id, movie.id)) is not None
|
||||
progress = session.get(MoviePlayProgress, (device_id, movie.id))
|
||||
|
||||
return MovieOut(
|
||||
progress = progress_by.get(movie.id)
|
||||
out.append(
|
||||
MovieOut(
|
||||
id=movie.id,
|
||||
title=movie.title,
|
||||
slug=movie.slug,
|
||||
|
|
@ -332,8 +377,10 @@ def _movie_to_out(session: Session, movie: Movie, *, device_id: str = LEGACY_DEV
|
|||
external_refs=external_refs,
|
||||
playback_sources=playback_sources,
|
||||
created_at=movie.created_at,
|
||||
is_favorite=is_fav,
|
||||
is_favorite=movie.id in favs,
|
||||
last_played_at=progress.last_played_at if progress else None,
|
||||
finished=progress.finished if progress else False,
|
||||
position_sec=progress.position_sec if progress else 0,
|
||||
)
|
||||
)
|
||||
return out
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ from __future__ import annotations
|
|||
import logging
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Annotated
|
||||
|
|
@ -85,6 +86,8 @@ class SourceOut(BaseModel):
|
|||
|
||||
|
||||
_ICON_DIR = Path(__file__).resolve().parent.parent / "static" / "site-icons"
|
||||
_SOURCES_TTL = 300.0
|
||||
_SOURCES_CACHE: dict = {"at": 0.0, "rows": None}
|
||||
|
||||
|
||||
def _icon_url(sitetag: str) -> str | None:
|
||||
|
|
@ -139,7 +142,12 @@ def list_sources(
|
|||
|
||||
Filter: `origin LIKE 'tube:%'` (drop canonical:* TPDB trailery to inna semantyka).
|
||||
"""
|
||||
rows = session.execute(
|
||||
# Agregat idzie po 4,1 mln zywych zrodel (600 ms na kazde wejscie w Sites, Sentry:
|
||||
# Slow DB Query list_sources). Liczby zmieniaja sie o promile na godzine, wiec
|
||||
# wynik zyje w pamieci procesu przez _SOURCES_TTL sekund.
|
||||
now = time.monotonic()
|
||||
if _SOURCES_CACHE["rows"] is None or now - _SOURCES_CACHE["at"] > _SOURCES_TTL:
|
||||
_SOURCES_CACHE["rows"] = session.execute(
|
||||
select(
|
||||
PlaybackSource.origin,
|
||||
func.count(PlaybackSource.id).label("scene_count"),
|
||||
|
|
@ -149,6 +157,8 @@ def list_sources(
|
|||
.where(PlaybackSource.origin.like("tube:%"))
|
||||
.group_by(PlaybackSource.origin)
|
||||
).all()
|
||||
_SOURCES_CACHE["at"] = now
|
||||
rows = _SOURCES_CACHE["rows"]
|
||||
|
||||
# Oceny z source_stats (policzone offline przez run_source_stats). Origin → row.
|
||||
stats = {s.origin: s for s in session.execute(select(SourceStats)).scalars().all()}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue