perf(scenes): bounded count + has_more for filtered scene lists
Filtered /scenes (tag/origin/q/studio/performer) ran exhaustive COUNT with
stub-filter EXISTS over 1.7M rows: TAG 5.1s, ORIGIN 4.9s, SEARCH 3.1s.
Mobile relied on `loaded < total` for infinite-scroll, making exact count
mandatory and ruling out approximate shortcuts.
Backend:
- SceneListOut gains has_more (bool) and total_capped (bool), both optional
for backward compat with old mobile
- Filtered count uses LIMIT _COUNT_CAP+1 (1000) subquery — cost is
O(min(matches, cap)) instead of O(all). Measured: TAG 5.1s→664ms,
SEARCH 3.1s→138ms, ORIGIN 4.9s→1.07s (also fixes SiteScenes showing
global count ~1M instead of per-site count)
- has_more from fetching per_page+1 rows (essentially free); extra row
stripped before serialisation
- Pure-default list (no filters at all) keeps TTL-cached full count
Mobile:
- getNextPageParam uses has_more ?? fallback to loaded<total
- Display shows "{total}+" when total_capped=true (5 screens)
Verified on emulator: tag "Big Tits" → "1000 scenes" loaded, no 500s,
backward compat confirmed (old APK works against new backend).
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
jtrzupek
parent
2163fee245
commit
5ae5dbb201
8 changed files with 77 additions and 24 deletions
|
|
@ -46,6 +46,13 @@ _VALID_SORTS = {"created_at", "release_date", "title", "studio"}
|
|||
_DEFAULT_COUNT_CACHE: dict = {"ts": 0.0, "val": 0}
|
||||
_DEFAULT_COUNT_TTL = 600.0
|
||||
|
||||
# Bounded count dla list FILTROWANYCH (origin/tag/q/studio/performer/...). Exhaustive
|
||||
# count z per-row stub-filter EXISTS bierze ~3-5s przy 1.7M scen (zmierzone). Liczymy
|
||||
# count tylko do CAP+1 — `LIMIT` ucina po znalezieniu CAP+1 pasujących, więc koszt to
|
||||
# O(min(matches, CAP)) zamiast O(all). >CAP → UI pokazuje "{CAP}+". Paginacja idzie po
|
||||
# has_more (fetch per_page+1), więc bounded total NIE psuje infinite-scroll.
|
||||
_COUNT_CAP = 1000
|
||||
|
||||
|
||||
def _default_scene_count(session: Session) -> int:
|
||||
import time as _time
|
||||
|
|
@ -303,20 +310,31 @@ def list_scenes(
|
|||
Scene.release_date.is_not(None) | canonical_exists | has_performer
|
||||
)
|
||||
|
||||
# Count: dla dużych baz (~400k scen) pełny count z 3 nested EXISTS bierze ~5s.
|
||||
# Liczymy total na uproszczonym query (bez stub-filter w count) — daje ~5% off
|
||||
# ale jest akceptowalne dla user-facing pagination header. Items query NADAL
|
||||
# ma stub-filter, więc lista pokazuje poprawne sceny. Liczba w header jest
|
||||
# przybliżoną górną granicą — co dla 400k scen i tak nie ma sensu reading dokładnie.
|
||||
if not include_stubs and not q and not studio_slug_list and not tags and not perf_id_strings:
|
||||
# Fast path: typowy default request (lista bez filtra) — count tylko po
|
||||
# has_playback (single EXISTS). Mimo to przy 1.69M scen full-scan z EXISTS
|
||||
# bierze ~950ms (zmierzone 2026-05-31), a liczba zmienia się wolno (ingest
|
||||
# ~kilkadziesiąt scen/h) i jest z definicji przybliżona. TTL-cache 10 min:
|
||||
# pierwszy request po wygaśnięciu płaci ~950ms, reszta czyta z pamięci.
|
||||
# Count strategy:
|
||||
# - PURE default (brak jakiegokolwiek filtra): cached count całego katalogu
|
||||
# (full-scan + EXISTS ~950ms, TTL 10 min — patrz _default_scene_count).
|
||||
# - FILTROWANE (origin/tag/q/studio/performer/quality/duration/...): bounded
|
||||
# count do _COUNT_CAP. Exhaustive count z per-row stub EXISTS to ~3-5s; bounded
|
||||
# ucina po CAP+1 trafieniach. Mobile paginuje po has_more (niżej), nie po total,
|
||||
# więc cap nie psuje infinite-scroll.
|
||||
_is_pure_default = (
|
||||
not include_stubs and not q and not studio_slug_list and not tag_slug_list
|
||||
and not perf_id_strings and origin is None and has_playback is None
|
||||
and not has_animated_thumbnail and min_duration_sec is None
|
||||
and max_duration_sec is None and released_within_days is None
|
||||
and min_quality_p is None
|
||||
)
|
||||
total_capped = False
|
||||
if _is_pure_default:
|
||||
total = _default_scene_count(session)
|
||||
else:
|
||||
total = session.execute(select(func.count()).select_from(base.subquery())).scalar_one()
|
||||
cnt = session.execute(
|
||||
select(func.count()).select_from(base.limit(_COUNT_CAP + 1).subquery())
|
||||
).scalar_one()
|
||||
if cnt > _COUNT_CAP:
|
||||
total, total_capped = _COUNT_CAP, True
|
||||
else:
|
||||
total = cnt
|
||||
|
||||
# Sort: zawsze tie-break po created_at desc dla determinizmu paginacji.
|
||||
if sort == "release_date":
|
||||
|
|
@ -340,15 +358,27 @@ def list_scenes(
|
|||
Scene.created_at.desc(), Scene.release_date.desc().nullslast()
|
||||
)
|
||||
|
||||
# Fetch per_page+1 — obecność (per_page+1)-szego wiersza = jest kolejna strona.
|
||||
# To źródło prawdy dla paginacji (mobile getNextPageParam), niezależne od bounded
|
||||
# `total`. Nadmiarowy wiersz odcinamy przed serializacją.
|
||||
rows = (
|
||||
session.execute(ordered.offset((page - 1) * per_page).limit(per_page))
|
||||
session.execute(ordered.offset((page - 1) * per_page).limit(per_page + 1))
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
has_more = len(rows) > per_page
|
||||
rows = rows[:per_page]
|
||||
|
||||
items = _build_scenes_out_batch(session, list(rows))
|
||||
|
||||
return SceneListOut(items=items, total=total, page=page, per_page=per_page)
|
||||
return SceneListOut(
|
||||
items=items,
|
||||
total=total,
|
||||
page=page,
|
||||
per_page=per_page,
|
||||
has_more=has_more,
|
||||
total_capped=total_capped,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{scene_id}", response_model=SceneOut)
|
||||
|
|
|
|||
|
|
@ -83,6 +83,12 @@ class SceneListOut(BaseModel):
|
|||
total: int
|
||||
page: int
|
||||
per_page: int
|
||||
# has_more: czy istnieje kolejna strona. Liczone z fetcha per_page+1 (≈darmowe),
|
||||
# NIE z `total` — bo dla filtrowanych list `total` jest bounded ("1000+") żeby
|
||||
# uniknąć ~5s exhaustive count. Mobile paginuje po has_more, nie po total.
|
||||
has_more: bool = False
|
||||
# total_capped: True gdy `total` to bounded cap (są >total wyników). UI: "{total}+".
|
||||
total_capped: bool = False
|
||||
|
||||
|
||||
class MovieChapterOut(BaseModel):
|
||||
|
|
|
|||
|
|
@ -223,6 +223,7 @@ export function PerformerScenesScreen() {
|
|||
|
||||
const movies = moviesQuery.data?.items ?? [];
|
||||
const scenesTotal = scenesQuery.data?.total ?? 0;
|
||||
const scenesSuffix = scenesQuery.data?.total_capped ? '+' : '';
|
||||
const moviesTotal = moviesQuery.data?.total ?? 0;
|
||||
|
||||
// Bug-report 2026-05-17 (562cf95c): "Przy przełączaniu na movies app crashed".
|
||||
|
|
@ -283,7 +284,7 @@ export function PerformerScenesScreen() {
|
|||
<View style={styles.header}>
|
||||
<Text style={styles.subtitle}>
|
||||
{scenesQuery.data
|
||||
? `${scenesTotal} ${scenesTotal === 1 ? 'scene' : 'scenes'}`
|
||||
? `${scenesTotal}${scenesSuffix} ${scenesTotal === 1 ? 'scene' : 'scenes'}`
|
||||
: ' '}
|
||||
</Text>
|
||||
</View>
|
||||
|
|
|
|||
|
|
@ -62,12 +62,18 @@ export function ScenesScreen() {
|
|||
}),
|
||||
initialPageParam: 1,
|
||||
getNextPageParam: (lastPage) => {
|
||||
const loaded = lastPage.page * lastPage.per_page;
|
||||
return loaded < lastPage.total ? lastPage.page + 1 : undefined;
|
||||
// Paginuj po has_more (źródło prawdy z fetcha per_page+1). `total` jest dla
|
||||
// list filtrowanych bounded ("1000+"), więc NIE nadaje się do liczenia stron.
|
||||
// Fallback na loaded<total gdy stary backend nie zwraca has_more.
|
||||
const more =
|
||||
lastPage.has_more ?? lastPage.page * lastPage.per_page < lastPage.total;
|
||||
return more ? lastPage.page + 1 : undefined;
|
||||
},
|
||||
});
|
||||
const items = data?.pages.flatMap((p) => p.items) ?? [];
|
||||
const total = data?.pages[0]?.total ?? 0;
|
||||
// total bywa bounded ("1000+") dla list filtrowanych (q/tag) — patrz backend _COUNT_CAP.
|
||||
const totalLabel = `${total}${data?.pages[0]?.total_capped ? '+' : ''}`;
|
||||
|
||||
const activeCount =
|
||||
filter.tagSlugs.length +
|
||||
|
|
@ -133,7 +139,7 @@ export function ScenesScreen() {
|
|||
isFetchingNextPage ? (
|
||||
<ActivityIndicator color={theme.muted} style={{ marginVertical: 18 }} />
|
||||
) : !hasNextPage && items.length > 0 ? (
|
||||
<Text style={styles.muted}>{`${items.length} / ${total}`}</Text>
|
||||
<Text style={styles.muted}>{`${items.length} / ${totalLabel}`}</Text>
|
||||
) : null
|
||||
}
|
||||
ListEmptyComponent={!isLoading ? <Text style={styles.muted}>no scenes</Text> : null}
|
||||
|
|
|
|||
|
|
@ -67,12 +67,16 @@ export function SiteScenesScreen() {
|
|||
}),
|
||||
initialPageParam: 1,
|
||||
getNextPageParam: (lastPage) => {
|
||||
const loaded = lastPage.page * lastPage.per_page;
|
||||
return loaded < lastPage.total ? lastPage.page + 1 : undefined;
|
||||
// Paginuj po has_more (z fetcha per_page+1). `total` bywa bounded ("1000+").
|
||||
const more =
|
||||
lastPage.has_more ?? lastPage.page * lastPage.per_page < lastPage.total;
|
||||
return more ? lastPage.page + 1 : undefined;
|
||||
},
|
||||
});
|
||||
const items = data?.pages.flatMap((p) => p.items) ?? [];
|
||||
const total = data?.pages[0]?.total ?? 0;
|
||||
// total bywa bounded ("1000+") dla dużych site'ów — patrz backend _COUNT_CAP.
|
||||
const totalLabel = `${total}${data?.pages[0]?.total_capped ? '+' : ''}`;
|
||||
|
||||
return (
|
||||
<View style={styles.container}>
|
||||
|
|
@ -120,7 +124,7 @@ export function SiteScenesScreen() {
|
|||
ListHeaderComponent={
|
||||
data ? (
|
||||
<Text style={styles.subtitle}>
|
||||
{total} {total === 1 ? 'scene' : 'scenes'} · sorted by release date
|
||||
{totalLabel} {total === 1 ? 'scene' : 'scenes'} · sorted by release date
|
||||
</Text>
|
||||
) : null
|
||||
}
|
||||
|
|
@ -128,7 +132,7 @@ export function SiteScenesScreen() {
|
|||
isFetchingNextPage ? (
|
||||
<ActivityIndicator color={theme.muted} style={{ marginVertical: 18 }} />
|
||||
) : !hasNextPage && items.length > 0 ? (
|
||||
<Text style={styles.muted}>{`${items.length} / ${total}`}</Text>
|
||||
<Text style={styles.muted}>{`${items.length} / ${totalLabel}`}</Text>
|
||||
) : null
|
||||
}
|
||||
ListEmptyComponent={!isLoading ? <Text style={styles.muted}>no scenes</Text> : null}
|
||||
|
|
|
|||
|
|
@ -147,7 +147,7 @@ export function StudioScenesScreen() {
|
|||
ListHeaderComponent={
|
||||
<View style={styles.header}>
|
||||
<Text style={styles.subtitle}>
|
||||
{data ? `${data.total} ${data.total === 1 ? 'scene' : 'scenes'}` : ' '}
|
||||
{data ? `${data.total}${data.total_capped ? '+' : ''} ${data.total === 1 ? 'scene' : 'scenes'}` : ' '}
|
||||
</Text>
|
||||
</View>
|
||||
}
|
||||
|
|
|
|||
|
|
@ -55,7 +55,7 @@ export function TagScenesScreen() {
|
|||
ListHeaderComponent={
|
||||
data ? (
|
||||
<Text style={styles.subtitle}>
|
||||
{data.total} {data.total === 1 ? 'scene' : 'scenes'} · with playback
|
||||
{data.total}{data.total_capped ? '+' : ''} {data.total === 1 ? 'scene' : 'scenes'} · with playback
|
||||
</Text>
|
||||
) : null
|
||||
}
|
||||
|
|
|
|||
|
|
@ -166,6 +166,12 @@ export interface SceneListOut {
|
|||
total: number;
|
||||
page: number;
|
||||
per_page: number;
|
||||
// has_more: czy jest kolejna strona (z fetcha per_page+1). Paginuj po TYM, nie po
|
||||
// `total` — bo dla list filtrowanych `total` jest bounded ("1000+"). Optional dla
|
||||
// wstecznej zgodności ze starym backendem (undefined → fallback na total w UI).
|
||||
has_more?: boolean;
|
||||
// total_capped: true gdy `total` to cap (jest >total wyników) → pokaż "{total}+".
|
||||
total_capped?: boolean;
|
||||
}
|
||||
|
||||
export interface MovieChapterOut {
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue