From 983bf62416747a3a8cffbdad66d0006d3c28ec09 Mon Sep 17 00:00:00 2001 From: jtrzupek Date: Tue, 2 Jun 2026 12:00:36 +0200 Subject: [PATCH] perf(scenes): drop exact count on filtered lists; index scene_tags(tag_id) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The filtered scene-list endpoints (default feed sends min_duration_sec=60, plus has_playback / tag / q filters) took ~4.5s — and an idle server. Profiling showed the entire cost was the bounded COUNT subquery over the EXISTS filters: Postgres would not reliably early-terminate at the cap under psycopg bound params, scanning the whole matching set (~858k for has_playback). Counting over the PK and using a literal LIMIT helped some cases but the plan stayed unstable. Fix: stop computing an exact count for filtered lists entirely. The mobile client paginates by has_more (per_page+1 fetch), never by total — total is only the "N+" UI counter. Derive total as a lower bound from the page + has_more after the fetch. This removes the count query from every filtered request. Result (end-to-end, authenticated): default feed 4.5s -> ~0.1s, has_playback 4.4s -> ~0.1s, q/studio/normal-tag filters all <0.3s. Also added index scene_tags(tag_id, scene_id) (PK led with scene_id, so tag->scenes did a seq scan). Remaining: a single enormous tag (e.g. "anal", ~163k scenes) ordered by recency still gathers-all-then-sorts in the fetch (~5s); normal tags are <0.5s. Tracked in #22 for a denormalized recency-ordered approach. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../20260602_0020_scene_tags_tag_id_index.py | 33 +++++++++++ app/api/scenes.py | 57 ++++++++----------- 2 files changed, 58 insertions(+), 32 deletions(-) create mode 100644 alembic/versions/20260602_0020_scene_tags_tag_id_index.py diff --git a/alembic/versions/20260602_0020_scene_tags_tag_id_index.py b/alembic/versions/20260602_0020_scene_tags_tag_id_index.py new file mode 100644 index 0000000..d95b794 --- /dev/null +++ b/alembic/versions/20260602_0020_scene_tags_tag_id_index.py @@ -0,0 +1,33 @@ +"""scene_tags (tag_id, scene_id) index — tag-filtered scene lists + +Revision ID: 0020_scene_tags_tag_id_index +Revises: 0019_taxonomy_scene_counts +Create Date: 2026-06-02 + +Perf fix (2026-06-02): `/scenes?tags=` był ~6s. scene_tags PK to +(scene_id, tag_id) — wiodąca kolumna scene_id, więc lookup "sceny z tagiem X" +(tag_id → scene_id) nie miał indeksu i robił Parallel Seq Scan po 2.8M scene_tags, +materializował wszystkie pasujące sceny i sortował. Indeks (tag_id, scene_id) +pozwala planerowi znaleźć sceny danego tagu po indeksie (i z literalnym LIMIT — +patrz scenes.py — iść index-walk + early-stop zamiast gather-all+sort). +""" +from collections.abc import Sequence + +from alembic import op + +revision: str = "0020_scene_tags_tag_id_index" +down_revision: str | None = "0019_taxonomy_scene_counts" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + op.create_index( + "ix_scene_tags_tag_id_scene_id", + "scene_tags", + ["tag_id", "scene_id"], + ) + + +def downgrade() -> None: + op.drop_index("ix_scene_tags_tag_id_scene_id", table_name="scene_tags") diff --git a/app/api/scenes.py b/app/api/scenes.py index 3a6f809..b7f7f7a 100644 --- a/app/api/scenes.py +++ b/app/api/scenes.py @@ -8,7 +8,7 @@ from typing import Annotated from fastapi import APIRouter, Depends, HTTPException, Query, status from pydantic import BaseModel -from sqlalchemy import distinct, exists, func, select +from sqlalchemy import distinct, exists, func, literal_column, select from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import Session @@ -46,14 +46,6 @@ _VALID_SORTS = {"created_at", "release_date", "title", "studio"} _DEFAULT_COUNT_CACHE: dict = {"ts": 0.0, "val": 0} _DEFAULT_COUNT_TTL = 600.0 -# Bounded count dla list FILTROWANYCH (origin/tag/q/studio/performer/...). Exhaustive -# count z per-row stub-filter EXISTS bierze ~3-5s przy 1.7M scen (zmierzone). Liczymy -# count tylko do CAP+1 — `LIMIT` ucina po znalezieniu CAP+1 pasujących, więc koszt to -# O(min(matches, CAP)) zamiast O(all). >CAP → UI pokazuje "{CAP}+". Paginacja idzie po -# has_more (fetch per_page+1), więc bounded total NIE psuje infinite-scroll. -_COUNT_CAP = 1000 - - def _default_scene_count(session: Session) -> int: import time as _time now = _time.monotonic() @@ -310,13 +302,6 @@ def list_scenes( Scene.release_date.is_not(None) | canonical_exists | has_performer ) - # Count strategy: - # - PURE default (brak jakiegokolwiek filtra): cached count całego katalogu - # (full-scan + EXISTS ~950ms, TTL 10 min — patrz _default_scene_count). - # - FILTROWANE (origin/tag/q/studio/performer/quality/duration/...): bounded - # count do _COUNT_CAP. Exhaustive count z per-row stub EXISTS to ~3-5s; bounded - # ucina po CAP+1 trafieniach. Mobile paginuje po has_more (niżej), nie po total, - # więc cap nie psuje infinite-scroll. _is_pure_default = ( not include_stubs and not q and not studio_slug_list and not tag_slug_list and not perf_id_strings and origin is None and has_playback is None @@ -324,22 +309,17 @@ def list_scenes( and max_duration_sec is None and released_within_days is None and min_quality_p is None ) + # Count strategy: + # - PURE default: cached pełny licznik katalogu (TTL 10 min). + # - FILTROWANE: NIE liczymy dokładnie. Bounded-count nad EXISTS-filtrami był + # dominującym kosztem (~4s na has_playback / min_duration / duży tag) i plan + # był NIESTABILNY (literal LIMIT + count-nad-PK pomogły w części przypadków, + # ale planer i tak czasem skanuje cały zbiór zamiast urwać). Mobile paginuje + # po `has_more` (per_page+1 fetch), NIE po `total` — `total` to tylko licznik + # "N+" w UI. Wyprowadzamy go z has_more PO fetchu (patrz niżej): dolna granica + # + flaga "jest więcej". Eliminuje cały koszt count z każdej filtrowanej listy. total_capped = False - if _is_pure_default: - total = _default_scene_count(session) - else: - # PERF (2026-06-02): `base` to `select(Scene)` (wszystkie kolumny). Count nad - # `SELECT scenes.* ... LIMIT 1001` z bound-params psycopg dobierał zły plan i - # zajmował ~4s (has_playback) / ~6s (tags) — mimo że to samo z `SELECT id` / - # literałami robi ~30ms. Liczymy nad samym PK: identyczna semantyka, ~100× szybciej. - count_base = base.with_only_columns(Scene.id) - cnt = session.execute( - select(func.count()).select_from(count_base.limit(_COUNT_CAP + 1).subquery()) - ).scalar_one() - if cnt > _COUNT_CAP: - total, total_capped = _COUNT_CAP, True - else: - total = cnt + total: int | None = _default_scene_count(session) if _is_pure_default else None # Sort: zawsze tie-break po created_at desc dla determinizmu paginacji. if sort == "release_date": @@ -366,14 +346,27 @@ def list_scenes( # Fetch per_page+1 — obecność (per_page+1)-szego wiersza = jest kolejna strona. # To źródło prawdy dla paginacji (mobile getNextPageParam), niezależne od bounded # `total`. Nadmiarowy wiersz odcinamy przed serializacją. + # LIMIT/OFFSET literalne (NIE bound-param) — patrz wyżej: sparametryzowany LIMIT + # psuje early-termination i przy filtrach EXISTS planer robi gather-all+sort (sekundy) + # zamiast limit-aware index-walk po `ix_scenes_created_at_desc`. page/per_page to + # walidowane inty (Query ge=1, le=200), więc literal_column jest bezpieczne. + _off = (page - 1) * per_page rows = ( - session.execute(ordered.offset((page - 1) * per_page).limit(per_page + 1)) + session.execute( + ordered.offset(literal_column(str(_off))).limit(literal_column(str(per_page + 1))) + ) .scalars() .all() ) has_more = len(rows) > per_page rows = rows[:per_page] + # Filtrowane listy: total = dolna granica z dotychczas-widzianych wierszy, a + # total_capped=has_more daje UI "N+" (jest kolejna strona). Bez osobnego count query. + if total is None: + total = (page - 1) * per_page + len(rows) + total_capped = has_more + items = _build_scenes_out_batch(session, list(rows)) return SceneListOut(