perf(scenes): drop exact count on filtered lists; index scene_tags(tag_id)

The filtered scene-list endpoints (default feed sends min_duration_sec=60, plus
has_playback / tag / q filters) took ~4.5s — and an idle server. Profiling showed
the entire cost was the bounded COUNT subquery over the EXISTS filters: Postgres
would not reliably early-terminate at the cap under psycopg bound params, scanning
the whole matching set (~858k for has_playback). Counting over the PK and using a
literal LIMIT helped some cases but the plan stayed unstable.

Fix: stop computing an exact count for filtered lists entirely. The mobile client
paginates by has_more (per_page+1 fetch), never by total — total is only the "N+"
UI counter. Derive total as a lower bound from the page + has_more after the fetch.
This removes the count query from every filtered request.

Result (end-to-end, authenticated): default feed 4.5s -> ~0.1s, has_playback
4.4s -> ~0.1s, q/studio/normal-tag filters all <0.3s. Also added index
scene_tags(tag_id, scene_id) (PK led with scene_id, so tag->scenes did a seq scan).

Remaining: a single enormous tag (e.g. "anal", ~163k scenes) ordered by recency
still gathers-all-then-sorts in the fetch (~5s); normal tags are <0.5s. Tracked
in #22 for a denormalized recency-ordered approach.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
jtrzupek 2026-06-02 12:00:36 +02:00
parent 20a8dc8e27
commit 983bf62416
2 changed files with 58 additions and 32 deletions

View file

@ -0,0 +1,33 @@
"""scene_tags (tag_id, scene_id) index — tag-filtered scene lists
Revision ID: 0020_scene_tags_tag_id_index
Revises: 0019_taxonomy_scene_counts
Create Date: 2026-06-02
Perf fix (2026-06-02): `/scenes?tags=<slug>` był ~6s. scene_tags PK to
(scene_id, tag_id) wiodąca kolumna scene_id, więc lookup "sceny z tagiem X"
(tag_id scene_id) nie miał indeksu i robił Parallel Seq Scan po 2.8M scene_tags,
materializował wszystkie pasujące sceny i sortował. Indeks (tag_id, scene_id)
pozwala planerowi znaleźć sceny danego tagu po indeksie (i z literalnym LIMIT
patrz scenes.py iść index-walk + early-stop zamiast gather-all+sort).
"""
from collections.abc import Sequence
from alembic import op
revision: str = "0020_scene_tags_tag_id_index"
down_revision: str | None = "0019_taxonomy_scene_counts"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
op.create_index(
"ix_scene_tags_tag_id_scene_id",
"scene_tags",
["tag_id", "scene_id"],
)
def downgrade() -> None:
op.drop_index("ix_scene_tags_tag_id_scene_id", table_name="scene_tags")

View file

@ -8,7 +8,7 @@ from typing import Annotated
from fastapi import APIRouter, Depends, HTTPException, Query, status
from pydantic import BaseModel
from sqlalchemy import distinct, exists, func, select
from sqlalchemy import distinct, exists, func, literal_column, select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session
@ -46,14 +46,6 @@ _VALID_SORTS = {"created_at", "release_date", "title", "studio"}
_DEFAULT_COUNT_CACHE: dict = {"ts": 0.0, "val": 0}
_DEFAULT_COUNT_TTL = 600.0
# Bounded count dla list FILTROWANYCH (origin/tag/q/studio/performer/...). Exhaustive
# count z per-row stub-filter EXISTS bierze ~3-5s przy 1.7M scen (zmierzone). Liczymy
# count tylko do CAP+1 — `LIMIT` ucina po znalezieniu CAP+1 pasujących, więc koszt to
# O(min(matches, CAP)) zamiast O(all). >CAP → UI pokazuje "{CAP}+". Paginacja idzie po
# has_more (fetch per_page+1), więc bounded total NIE psuje infinite-scroll.
_COUNT_CAP = 1000
def _default_scene_count(session: Session) -> int:
import time as _time
now = _time.monotonic()
@ -310,13 +302,6 @@ def list_scenes(
Scene.release_date.is_not(None) | canonical_exists | has_performer
)
# Count strategy:
# - PURE default (brak jakiegokolwiek filtra): cached count całego katalogu
# (full-scan + EXISTS ~950ms, TTL 10 min — patrz _default_scene_count).
# - FILTROWANE (origin/tag/q/studio/performer/quality/duration/...): bounded
# count do _COUNT_CAP. Exhaustive count z per-row stub EXISTS to ~3-5s; bounded
# ucina po CAP+1 trafieniach. Mobile paginuje po has_more (niżej), nie po total,
# więc cap nie psuje infinite-scroll.
_is_pure_default = (
not include_stubs and not q and not studio_slug_list and not tag_slug_list
and not perf_id_strings and origin is None and has_playback is None
@ -324,22 +309,17 @@ def list_scenes(
and max_duration_sec is None and released_within_days is None
and min_quality_p is None
)
# Count strategy:
# - PURE default: cached pełny licznik katalogu (TTL 10 min).
# - FILTROWANE: NIE liczymy dokładnie. Bounded-count nad EXISTS-filtrami był
# dominującym kosztem (~4s na has_playback / min_duration / duży tag) i plan
# był NIESTABILNY (literal LIMIT + count-nad-PK pomogły w części przypadków,
# ale planer i tak czasem skanuje cały zbiór zamiast urwać). Mobile paginuje
# po `has_more` (per_page+1 fetch), NIE po `total` — `total` to tylko licznik
# "N+" w UI. Wyprowadzamy go z has_more PO fetchu (patrz niżej): dolna granica
# + flaga "jest więcej". Eliminuje cały koszt count z każdej filtrowanej listy.
total_capped = False
if _is_pure_default:
total = _default_scene_count(session)
else:
# PERF (2026-06-02): `base` to `select(Scene)` (wszystkie kolumny). Count nad
# `SELECT scenes.* ... LIMIT 1001` z bound-params psycopg dobierał zły plan i
# zajmował ~4s (has_playback) / ~6s (tags) — mimo że to samo z `SELECT id` /
# literałami robi ~30ms. Liczymy nad samym PK: identyczna semantyka, ~100× szybciej.
count_base = base.with_only_columns(Scene.id)
cnt = session.execute(
select(func.count()).select_from(count_base.limit(_COUNT_CAP + 1).subquery())
).scalar_one()
if cnt > _COUNT_CAP:
total, total_capped = _COUNT_CAP, True
else:
total = cnt
total: int | None = _default_scene_count(session) if _is_pure_default else None
# Sort: zawsze tie-break po created_at desc dla determinizmu paginacji.
if sort == "release_date":
@ -366,14 +346,27 @@ def list_scenes(
# Fetch per_page+1 — obecność (per_page+1)-szego wiersza = jest kolejna strona.
# To źródło prawdy dla paginacji (mobile getNextPageParam), niezależne od bounded
# `total`. Nadmiarowy wiersz odcinamy przed serializacją.
# LIMIT/OFFSET literalne (NIE bound-param) — patrz wyżej: sparametryzowany LIMIT
# psuje early-termination i przy filtrach EXISTS planer robi gather-all+sort (sekundy)
# zamiast limit-aware index-walk po `ix_scenes_created_at_desc`. page/per_page to
# walidowane inty (Query ge=1, le=200), więc literal_column jest bezpieczne.
_off = (page - 1) * per_page
rows = (
session.execute(ordered.offset((page - 1) * per_page).limit(per_page + 1))
session.execute(
ordered.offset(literal_column(str(_off))).limit(literal_column(str(per_page + 1)))
)
.scalars()
.all()
)
has_more = len(rows) > per_page
rows = rows[:per_page]
# Filtrowane listy: total = dolna granica z dotychczas-widzianych wierszy, a
# total_capped=has_more daje UI "N+" (jest kolejna strona). Bez osobnego count query.
if total is None:
total = (page - 1) * per_page + len(rows)
total_capped = has_more
items = _build_scenes_out_batch(session, list(rows))
return SceneListOut(