From 5ae5dbb201d4e89795627f9ec71a8930ecabc504 Mon Sep 17 00:00:00 2001 From: jtrzupek Date: Sun, 31 May 2026 19:24:26 +0200 Subject: [PATCH] perf(scenes): bounded count + has_more for filtered scene lists MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Filtered /scenes (tag/origin/q/studio/performer) ran exhaustive COUNT with stub-filter EXISTS over 1.7M rows: TAG 5.1s, ORIGIN 4.9s, SEARCH 3.1s. Mobile relied on `loaded < total` for infinite-scroll, making exact count mandatory and ruling out approximate shortcuts. Backend: - SceneListOut gains has_more (bool) and total_capped (bool), both optional for backward compat with old mobile - Filtered count uses LIMIT _COUNT_CAP+1 (1000) subquery — cost is O(min(matches, cap)) instead of O(all). Measured: TAG 5.1s→664ms, SEARCH 3.1s→138ms, ORIGIN 4.9s→1.07s (also fixes SiteScenes showing global count ~1M instead of per-site count) - has_more from fetching per_page+1 rows (essentially free); extra row stripped before serialisation - Pure-default list (no filters at all) keeps TTL-cached full count Mobile: - getNextPageParam uses has_more ?? fallback to loaded --- app/api/scenes.py | 58 +++++++++++++++----- app/api/schemas.py | 6 ++ mobile/src/screens/PerformerScenesScreen.tsx | 3 +- mobile/src/screens/ScenesScreen.tsx | 12 +++- mobile/src/screens/SiteScenesScreen.tsx | 12 ++-- mobile/src/screens/StudioScenesScreen.tsx | 2 +- mobile/src/screens/TagScenesScreen.tsx | 2 +- mobile/src/types.ts | 6 ++ 8 files changed, 77 insertions(+), 24 deletions(-) diff --git a/app/api/scenes.py b/app/api/scenes.py index e233f2f..8d30810 100644 --- a/app/api/scenes.py +++ b/app/api/scenes.py @@ -46,6 +46,13 @@ _VALID_SORTS = {"created_at", "release_date", "title", "studio"} _DEFAULT_COUNT_CACHE: dict = {"ts": 0.0, "val": 0} _DEFAULT_COUNT_TTL = 600.0 +# Bounded count dla list FILTROWANYCH (origin/tag/q/studio/performer/...). Exhaustive +# count z per-row stub-filter EXISTS bierze ~3-5s przy 1.7M scen (zmierzone). Liczymy +# count tylko do CAP+1 — `LIMIT` ucina po znalezieniu CAP+1 pasujących, więc koszt to +# O(min(matches, CAP)) zamiast O(all). >CAP → UI pokazuje "{CAP}+". Paginacja idzie po +# has_more (fetch per_page+1), więc bounded total NIE psuje infinite-scroll. +_COUNT_CAP = 1000 + def _default_scene_count(session: Session) -> int: import time as _time @@ -303,20 +310,31 @@ def list_scenes( Scene.release_date.is_not(None) | canonical_exists | has_performer ) - # Count: dla dużych baz (~400k scen) pełny count z 3 nested EXISTS bierze ~5s. - # Liczymy total na uproszczonym query (bez stub-filter w count) — daje ~5% off - # ale jest akceptowalne dla user-facing pagination header. Items query NADAL - # ma stub-filter, więc lista pokazuje poprawne sceny. Liczba w header jest - # przybliżoną górną granicą — co dla 400k scen i tak nie ma sensu reading dokładnie. - if not include_stubs and not q and not studio_slug_list and not tags and not perf_id_strings: - # Fast path: typowy default request (lista bez filtra) — count tylko po - # has_playback (single EXISTS). Mimo to przy 1.69M scen full-scan z EXISTS - # bierze ~950ms (zmierzone 2026-05-31), a liczba zmienia się wolno (ingest - # ~kilkadziesiąt scen/h) i jest z definicji przybliżona. TTL-cache 10 min: - # pierwszy request po wygaśnięciu płaci ~950ms, reszta czyta z pamięci. + # Count strategy: + # - PURE default (brak jakiegokolwiek filtra): cached count całego katalogu + # (full-scan + EXISTS ~950ms, TTL 10 min — patrz _default_scene_count). + # - FILTROWANE (origin/tag/q/studio/performer/quality/duration/...): bounded + # count do _COUNT_CAP. Exhaustive count z per-row stub EXISTS to ~3-5s; bounded + # ucina po CAP+1 trafieniach. Mobile paginuje po has_more (niżej), nie po total, + # więc cap nie psuje infinite-scroll. + _is_pure_default = ( + not include_stubs and not q and not studio_slug_list and not tag_slug_list + and not perf_id_strings and origin is None and has_playback is None + and not has_animated_thumbnail and min_duration_sec is None + and max_duration_sec is None and released_within_days is None + and min_quality_p is None + ) + total_capped = False + if _is_pure_default: total = _default_scene_count(session) else: - total = session.execute(select(func.count()).select_from(base.subquery())).scalar_one() + cnt = session.execute( + select(func.count()).select_from(base.limit(_COUNT_CAP + 1).subquery()) + ).scalar_one() + if cnt > _COUNT_CAP: + total, total_capped = _COUNT_CAP, True + else: + total = cnt # Sort: zawsze tie-break po created_at desc dla determinizmu paginacji. if sort == "release_date": @@ -340,15 +358,27 @@ def list_scenes( Scene.created_at.desc(), Scene.release_date.desc().nullslast() ) + # Fetch per_page+1 — obecność (per_page+1)-szego wiersza = jest kolejna strona. + # To źródło prawdy dla paginacji (mobile getNextPageParam), niezależne od bounded + # `total`. Nadmiarowy wiersz odcinamy przed serializacją. rows = ( - session.execute(ordered.offset((page - 1) * per_page).limit(per_page)) + session.execute(ordered.offset((page - 1) * per_page).limit(per_page + 1)) .scalars() .all() ) + has_more = len(rows) > per_page + rows = rows[:per_page] items = _build_scenes_out_batch(session, list(rows)) - return SceneListOut(items=items, total=total, page=page, per_page=per_page) + return SceneListOut( + items=items, + total=total, + page=page, + per_page=per_page, + has_more=has_more, + total_capped=total_capped, + ) @router.get("/{scene_id}", response_model=SceneOut) diff --git a/app/api/schemas.py b/app/api/schemas.py index d9396c9..2fdaca2 100644 --- a/app/api/schemas.py +++ b/app/api/schemas.py @@ -83,6 +83,12 @@ class SceneListOut(BaseModel): total: int page: int per_page: int + # has_more: czy istnieje kolejna strona. Liczone z fetcha per_page+1 (≈darmowe), + # NIE z `total` — bo dla filtrowanych list `total` jest bounded ("1000+") żeby + # uniknąć ~5s exhaustive count. Mobile paginuje po has_more, nie po total. + has_more: bool = False + # total_capped: True gdy `total` to bounded cap (są >total wyników). UI: "{total}+". + total_capped: bool = False class MovieChapterOut(BaseModel): diff --git a/mobile/src/screens/PerformerScenesScreen.tsx b/mobile/src/screens/PerformerScenesScreen.tsx index ff5594c..9d8087a 100644 --- a/mobile/src/screens/PerformerScenesScreen.tsx +++ b/mobile/src/screens/PerformerScenesScreen.tsx @@ -223,6 +223,7 @@ export function PerformerScenesScreen() { const movies = moviesQuery.data?.items ?? []; const scenesTotal = scenesQuery.data?.total ?? 0; + const scenesSuffix = scenesQuery.data?.total_capped ? '+' : ''; const moviesTotal = moviesQuery.data?.total ?? 0; // Bug-report 2026-05-17 (562cf95c): "Przy przełączaniu na movies app crashed". @@ -283,7 +284,7 @@ export function PerformerScenesScreen() { {scenesQuery.data - ? `${scenesTotal} ${scenesTotal === 1 ? 'scene' : 'scenes'}` + ? `${scenesTotal}${scenesSuffix} ${scenesTotal === 1 ? 'scene' : 'scenes'}` : ' '} diff --git a/mobile/src/screens/ScenesScreen.tsx b/mobile/src/screens/ScenesScreen.tsx index cbf44fa..0c332a3 100644 --- a/mobile/src/screens/ScenesScreen.tsx +++ b/mobile/src/screens/ScenesScreen.tsx @@ -62,12 +62,18 @@ export function ScenesScreen() { }), initialPageParam: 1, getNextPageParam: (lastPage) => { - const loaded = lastPage.page * lastPage.per_page; - return loaded < lastPage.total ? lastPage.page + 1 : undefined; + // Paginuj po has_more (źródło prawdy z fetcha per_page+1). `total` jest dla + // list filtrowanych bounded ("1000+"), więc NIE nadaje się do liczenia stron. + // Fallback na loaded p.items) ?? []; const total = data?.pages[0]?.total ?? 0; + // total bywa bounded ("1000+") dla list filtrowanych (q/tag) — patrz backend _COUNT_CAP. + const totalLabel = `${total}${data?.pages[0]?.total_capped ? '+' : ''}`; const activeCount = filter.tagSlugs.length + @@ -133,7 +139,7 @@ export function ScenesScreen() { isFetchingNextPage ? ( ) : !hasNextPage && items.length > 0 ? ( - {`${items.length} / ${total}`} + {`${items.length} / ${totalLabel}`} ) : null } ListEmptyComponent={!isLoading ? no scenes : null} diff --git a/mobile/src/screens/SiteScenesScreen.tsx b/mobile/src/screens/SiteScenesScreen.tsx index cf1c3ac..8969329 100644 --- a/mobile/src/screens/SiteScenesScreen.tsx +++ b/mobile/src/screens/SiteScenesScreen.tsx @@ -67,12 +67,16 @@ export function SiteScenesScreen() { }), initialPageParam: 1, getNextPageParam: (lastPage) => { - const loaded = lastPage.page * lastPage.per_page; - return loaded < lastPage.total ? lastPage.page + 1 : undefined; + // Paginuj po has_more (z fetcha per_page+1). `total` bywa bounded ("1000+"). + const more = + lastPage.has_more ?? lastPage.page * lastPage.per_page < lastPage.total; + return more ? lastPage.page + 1 : undefined; }, }); const items = data?.pages.flatMap((p) => p.items) ?? []; const total = data?.pages[0]?.total ?? 0; + // total bywa bounded ("1000+") dla dużych site'ów — patrz backend _COUNT_CAP. + const totalLabel = `${total}${data?.pages[0]?.total_capped ? '+' : ''}`; return ( @@ -120,7 +124,7 @@ export function SiteScenesScreen() { ListHeaderComponent={ data ? ( - {total} {total === 1 ? 'scene' : 'scenes'} · sorted by release date + {totalLabel} {total === 1 ? 'scene' : 'scenes'} · sorted by release date ) : null } @@ -128,7 +132,7 @@ export function SiteScenesScreen() { isFetchingNextPage ? ( ) : !hasNextPage && items.length > 0 ? ( - {`${items.length} / ${total}`} + {`${items.length} / ${totalLabel}`} ) : null } ListEmptyComponent={!isLoading ? no scenes : null} diff --git a/mobile/src/screens/StudioScenesScreen.tsx b/mobile/src/screens/StudioScenesScreen.tsx index bec9dce..cb3ce06 100644 --- a/mobile/src/screens/StudioScenesScreen.tsx +++ b/mobile/src/screens/StudioScenesScreen.tsx @@ -147,7 +147,7 @@ export function StudioScenesScreen() { ListHeaderComponent={ - {data ? `${data.total} ${data.total === 1 ? 'scene' : 'scenes'}` : ' '} + {data ? `${data.total}${data.total_capped ? '+' : ''} ${data.total === 1 ? 'scene' : 'scenes'}` : ' '} } diff --git a/mobile/src/screens/TagScenesScreen.tsx b/mobile/src/screens/TagScenesScreen.tsx index c5945e3..32a096a 100644 --- a/mobile/src/screens/TagScenesScreen.tsx +++ b/mobile/src/screens/TagScenesScreen.tsx @@ -55,7 +55,7 @@ export function TagScenesScreen() { ListHeaderComponent={ data ? ( - {data.total} {data.total === 1 ? 'scene' : 'scenes'} · with playback + {data.total}{data.total_capped ? '+' : ''} {data.total === 1 ? 'scene' : 'scenes'} · with playback ) : null } diff --git a/mobile/src/types.ts b/mobile/src/types.ts index c79320a..adeed2a 100644 --- a/mobile/src/types.ts +++ b/mobile/src/types.ts @@ -166,6 +166,12 @@ export interface SceneListOut { total: number; page: number; per_page: number; + // has_more: czy jest kolejna strona (z fetcha per_page+1). Paginuj po TYM, nie po + // `total` — bo dla list filtrowanych `total` jest bounded ("1000+"). Optional dla + // wstecznej zgodności ze starym backendem (undefined → fallback na total w UI). + has_more?: boolean; + // total_capped: true gdy `total` to cap (jest >total wyników) → pokaż "{total}+". + total_capped?: boolean; } export interface MovieChapterOut {