perf(scenes): bounded count + has_more for filtered scene lists
Filtered /scenes (tag/origin/q/studio/performer) ran exhaustive COUNT with
stub-filter EXISTS over 1.7M rows: TAG 5.1s, ORIGIN 4.9s, SEARCH 3.1s.
Mobile relied on `loaded < total` for infinite-scroll, making exact count
mandatory and ruling out approximate shortcuts.
Backend:
- SceneListOut gains has_more (bool) and total_capped (bool), both optional
for backward compat with old mobile
- Filtered count uses LIMIT _COUNT_CAP+1 (1000) subquery — cost is
O(min(matches, cap)) instead of O(all). Measured: TAG 5.1s→664ms,
SEARCH 3.1s→138ms, ORIGIN 4.9s→1.07s (also fixes SiteScenes showing
global count ~1M instead of per-site count)
- has_more from fetching per_page+1 rows (essentially free); extra row
stripped before serialisation
- Pure-default list (no filters at all) keeps TTL-cached full count
Mobile:
- getNextPageParam uses has_more ?? fallback to loaded<total
- Display shows "{total}+" when total_capped=true (5 screens)
Verified on emulator: tag "Big Tits" → "1000 scenes" loaded, no 500s,
backward compat confirmed (old APK works against new backend).
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
2163fee245
commit
5ae5dbb201
8 changed files with 77 additions and 24 deletions
|
|
@ -46,6 +46,13 @@ _VALID_SORTS = {"created_at", "release_date", "title", "studio"}
|
||||||
_DEFAULT_COUNT_CACHE: dict = {"ts": 0.0, "val": 0}
|
_DEFAULT_COUNT_CACHE: dict = {"ts": 0.0, "val": 0}
|
||||||
_DEFAULT_COUNT_TTL = 600.0
|
_DEFAULT_COUNT_TTL = 600.0
|
||||||
|
|
||||||
|
# Bounded count dla list FILTROWANYCH (origin/tag/q/studio/performer/...). Exhaustive
|
||||||
|
# count z per-row stub-filter EXISTS bierze ~3-5s przy 1.7M scen (zmierzone). Liczymy
|
||||||
|
# count tylko do CAP+1 — `LIMIT` ucina po znalezieniu CAP+1 pasujących, więc koszt to
|
||||||
|
# O(min(matches, CAP)) zamiast O(all). >CAP → UI pokazuje "{CAP}+". Paginacja idzie po
|
||||||
|
# has_more (fetch per_page+1), więc bounded total NIE psuje infinite-scroll.
|
||||||
|
_COUNT_CAP = 1000
|
||||||
|
|
||||||
|
|
||||||
def _default_scene_count(session: Session) -> int:
|
def _default_scene_count(session: Session) -> int:
|
||||||
import time as _time
|
import time as _time
|
||||||
|
|
@ -303,20 +310,31 @@ def list_scenes(
|
||||||
Scene.release_date.is_not(None) | canonical_exists | has_performer
|
Scene.release_date.is_not(None) | canonical_exists | has_performer
|
||||||
)
|
)
|
||||||
|
|
||||||
# Count: dla dużych baz (~400k scen) pełny count z 3 nested EXISTS bierze ~5s.
|
# Count strategy:
|
||||||
# Liczymy total na uproszczonym query (bez stub-filter w count) — daje ~5% off
|
# - PURE default (brak jakiegokolwiek filtra): cached count całego katalogu
|
||||||
# ale jest akceptowalne dla user-facing pagination header. Items query NADAL
|
# (full-scan + EXISTS ~950ms, TTL 10 min — patrz _default_scene_count).
|
||||||
# ma stub-filter, więc lista pokazuje poprawne sceny. Liczba w header jest
|
# - FILTROWANE (origin/tag/q/studio/performer/quality/duration/...): bounded
|
||||||
# przybliżoną górną granicą — co dla 400k scen i tak nie ma sensu reading dokładnie.
|
# count do _COUNT_CAP. Exhaustive count z per-row stub EXISTS to ~3-5s; bounded
|
||||||
if not include_stubs and not q and not studio_slug_list and not tags and not perf_id_strings:
|
# ucina po CAP+1 trafieniach. Mobile paginuje po has_more (niżej), nie po total,
|
||||||
# Fast path: typowy default request (lista bez filtra) — count tylko po
|
# więc cap nie psuje infinite-scroll.
|
||||||
# has_playback (single EXISTS). Mimo to przy 1.69M scen full-scan z EXISTS
|
_is_pure_default = (
|
||||||
# bierze ~950ms (zmierzone 2026-05-31), a liczba zmienia się wolno (ingest
|
not include_stubs and not q and not studio_slug_list and not tag_slug_list
|
||||||
# ~kilkadziesiąt scen/h) i jest z definicji przybliżona. TTL-cache 10 min:
|
and not perf_id_strings and origin is None and has_playback is None
|
||||||
# pierwszy request po wygaśnięciu płaci ~950ms, reszta czyta z pamięci.
|
and not has_animated_thumbnail and min_duration_sec is None
|
||||||
|
and max_duration_sec is None and released_within_days is None
|
||||||
|
and min_quality_p is None
|
||||||
|
)
|
||||||
|
total_capped = False
|
||||||
|
if _is_pure_default:
|
||||||
total = _default_scene_count(session)
|
total = _default_scene_count(session)
|
||||||
else:
|
else:
|
||||||
total = session.execute(select(func.count()).select_from(base.subquery())).scalar_one()
|
cnt = session.execute(
|
||||||
|
select(func.count()).select_from(base.limit(_COUNT_CAP + 1).subquery())
|
||||||
|
).scalar_one()
|
||||||
|
if cnt > _COUNT_CAP:
|
||||||
|
total, total_capped = _COUNT_CAP, True
|
||||||
|
else:
|
||||||
|
total = cnt
|
||||||
|
|
||||||
# Sort: zawsze tie-break po created_at desc dla determinizmu paginacji.
|
# Sort: zawsze tie-break po created_at desc dla determinizmu paginacji.
|
||||||
if sort == "release_date":
|
if sort == "release_date":
|
||||||
|
|
@ -340,15 +358,27 @@ def list_scenes(
|
||||||
Scene.created_at.desc(), Scene.release_date.desc().nullslast()
|
Scene.created_at.desc(), Scene.release_date.desc().nullslast()
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Fetch per_page+1 — obecność (per_page+1)-szego wiersza = jest kolejna strona.
|
||||||
|
# To źródło prawdy dla paginacji (mobile getNextPageParam), niezależne od bounded
|
||||||
|
# `total`. Nadmiarowy wiersz odcinamy przed serializacją.
|
||||||
rows = (
|
rows = (
|
||||||
session.execute(ordered.offset((page - 1) * per_page).limit(per_page))
|
session.execute(ordered.offset((page - 1) * per_page).limit(per_page + 1))
|
||||||
.scalars()
|
.scalars()
|
||||||
.all()
|
.all()
|
||||||
)
|
)
|
||||||
|
has_more = len(rows) > per_page
|
||||||
|
rows = rows[:per_page]
|
||||||
|
|
||||||
items = _build_scenes_out_batch(session, list(rows))
|
items = _build_scenes_out_batch(session, list(rows))
|
||||||
|
|
||||||
return SceneListOut(items=items, total=total, page=page, per_page=per_page)
|
return SceneListOut(
|
||||||
|
items=items,
|
||||||
|
total=total,
|
||||||
|
page=page,
|
||||||
|
per_page=per_page,
|
||||||
|
has_more=has_more,
|
||||||
|
total_capped=total_capped,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@router.get("/{scene_id}", response_model=SceneOut)
|
@router.get("/{scene_id}", response_model=SceneOut)
|
||||||
|
|
|
||||||
|
|
@ -83,6 +83,12 @@ class SceneListOut(BaseModel):
|
||||||
total: int
|
total: int
|
||||||
page: int
|
page: int
|
||||||
per_page: int
|
per_page: int
|
||||||
|
# has_more: czy istnieje kolejna strona. Liczone z fetcha per_page+1 (≈darmowe),
|
||||||
|
# NIE z `total` — bo dla filtrowanych list `total` jest bounded ("1000+") żeby
|
||||||
|
# uniknąć ~5s exhaustive count. Mobile paginuje po has_more, nie po total.
|
||||||
|
has_more: bool = False
|
||||||
|
# total_capped: True gdy `total` to bounded cap (są >total wyników). UI: "{total}+".
|
||||||
|
total_capped: bool = False
|
||||||
|
|
||||||
|
|
||||||
class MovieChapterOut(BaseModel):
|
class MovieChapterOut(BaseModel):
|
||||||
|
|
|
||||||
|
|
@ -223,6 +223,7 @@ export function PerformerScenesScreen() {
|
||||||
|
|
||||||
const movies = moviesQuery.data?.items ?? [];
|
const movies = moviesQuery.data?.items ?? [];
|
||||||
const scenesTotal = scenesQuery.data?.total ?? 0;
|
const scenesTotal = scenesQuery.data?.total ?? 0;
|
||||||
|
const scenesSuffix = scenesQuery.data?.total_capped ? '+' : '';
|
||||||
const moviesTotal = moviesQuery.data?.total ?? 0;
|
const moviesTotal = moviesQuery.data?.total ?? 0;
|
||||||
|
|
||||||
// Bug-report 2026-05-17 (562cf95c): "Przy przełączaniu na movies app crashed".
|
// Bug-report 2026-05-17 (562cf95c): "Przy przełączaniu na movies app crashed".
|
||||||
|
|
@ -283,7 +284,7 @@ export function PerformerScenesScreen() {
|
||||||
<View style={styles.header}>
|
<View style={styles.header}>
|
||||||
<Text style={styles.subtitle}>
|
<Text style={styles.subtitle}>
|
||||||
{scenesQuery.data
|
{scenesQuery.data
|
||||||
? `${scenesTotal} ${scenesTotal === 1 ? 'scene' : 'scenes'}`
|
? `${scenesTotal}${scenesSuffix} ${scenesTotal === 1 ? 'scene' : 'scenes'}`
|
||||||
: ' '}
|
: ' '}
|
||||||
</Text>
|
</Text>
|
||||||
</View>
|
</View>
|
||||||
|
|
|
||||||
|
|
@ -62,12 +62,18 @@ export function ScenesScreen() {
|
||||||
}),
|
}),
|
||||||
initialPageParam: 1,
|
initialPageParam: 1,
|
||||||
getNextPageParam: (lastPage) => {
|
getNextPageParam: (lastPage) => {
|
||||||
const loaded = lastPage.page * lastPage.per_page;
|
// Paginuj po has_more (źródło prawdy z fetcha per_page+1). `total` jest dla
|
||||||
return loaded < lastPage.total ? lastPage.page + 1 : undefined;
|
// list filtrowanych bounded ("1000+"), więc NIE nadaje się do liczenia stron.
|
||||||
|
// Fallback na loaded<total gdy stary backend nie zwraca has_more.
|
||||||
|
const more =
|
||||||
|
lastPage.has_more ?? lastPage.page * lastPage.per_page < lastPage.total;
|
||||||
|
return more ? lastPage.page + 1 : undefined;
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
const items = data?.pages.flatMap((p) => p.items) ?? [];
|
const items = data?.pages.flatMap((p) => p.items) ?? [];
|
||||||
const total = data?.pages[0]?.total ?? 0;
|
const total = data?.pages[0]?.total ?? 0;
|
||||||
|
// total bywa bounded ("1000+") dla list filtrowanych (q/tag) — patrz backend _COUNT_CAP.
|
||||||
|
const totalLabel = `${total}${data?.pages[0]?.total_capped ? '+' : ''}`;
|
||||||
|
|
||||||
const activeCount =
|
const activeCount =
|
||||||
filter.tagSlugs.length +
|
filter.tagSlugs.length +
|
||||||
|
|
@ -133,7 +139,7 @@ export function ScenesScreen() {
|
||||||
isFetchingNextPage ? (
|
isFetchingNextPage ? (
|
||||||
<ActivityIndicator color={theme.muted} style={{ marginVertical: 18 }} />
|
<ActivityIndicator color={theme.muted} style={{ marginVertical: 18 }} />
|
||||||
) : !hasNextPage && items.length > 0 ? (
|
) : !hasNextPage && items.length > 0 ? (
|
||||||
<Text style={styles.muted}>{`${items.length} / ${total}`}</Text>
|
<Text style={styles.muted}>{`${items.length} / ${totalLabel}`}</Text>
|
||||||
) : null
|
) : null
|
||||||
}
|
}
|
||||||
ListEmptyComponent={!isLoading ? <Text style={styles.muted}>no scenes</Text> : null}
|
ListEmptyComponent={!isLoading ? <Text style={styles.muted}>no scenes</Text> : null}
|
||||||
|
|
|
||||||
|
|
@ -67,12 +67,16 @@ export function SiteScenesScreen() {
|
||||||
}),
|
}),
|
||||||
initialPageParam: 1,
|
initialPageParam: 1,
|
||||||
getNextPageParam: (lastPage) => {
|
getNextPageParam: (lastPage) => {
|
||||||
const loaded = lastPage.page * lastPage.per_page;
|
// Paginuj po has_more (z fetcha per_page+1). `total` bywa bounded ("1000+").
|
||||||
return loaded < lastPage.total ? lastPage.page + 1 : undefined;
|
const more =
|
||||||
|
lastPage.has_more ?? lastPage.page * lastPage.per_page < lastPage.total;
|
||||||
|
return more ? lastPage.page + 1 : undefined;
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
const items = data?.pages.flatMap((p) => p.items) ?? [];
|
const items = data?.pages.flatMap((p) => p.items) ?? [];
|
||||||
const total = data?.pages[0]?.total ?? 0;
|
const total = data?.pages[0]?.total ?? 0;
|
||||||
|
// total bywa bounded ("1000+") dla dużych site'ów — patrz backend _COUNT_CAP.
|
||||||
|
const totalLabel = `${total}${data?.pages[0]?.total_capped ? '+' : ''}`;
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<View style={styles.container}>
|
<View style={styles.container}>
|
||||||
|
|
@ -120,7 +124,7 @@ export function SiteScenesScreen() {
|
||||||
ListHeaderComponent={
|
ListHeaderComponent={
|
||||||
data ? (
|
data ? (
|
||||||
<Text style={styles.subtitle}>
|
<Text style={styles.subtitle}>
|
||||||
{total} {total === 1 ? 'scene' : 'scenes'} · sorted by release date
|
{totalLabel} {total === 1 ? 'scene' : 'scenes'} · sorted by release date
|
||||||
</Text>
|
</Text>
|
||||||
) : null
|
) : null
|
||||||
}
|
}
|
||||||
|
|
@ -128,7 +132,7 @@ export function SiteScenesScreen() {
|
||||||
isFetchingNextPage ? (
|
isFetchingNextPage ? (
|
||||||
<ActivityIndicator color={theme.muted} style={{ marginVertical: 18 }} />
|
<ActivityIndicator color={theme.muted} style={{ marginVertical: 18 }} />
|
||||||
) : !hasNextPage && items.length > 0 ? (
|
) : !hasNextPage && items.length > 0 ? (
|
||||||
<Text style={styles.muted}>{`${items.length} / ${total}`}</Text>
|
<Text style={styles.muted}>{`${items.length} / ${totalLabel}`}</Text>
|
||||||
) : null
|
) : null
|
||||||
}
|
}
|
||||||
ListEmptyComponent={!isLoading ? <Text style={styles.muted}>no scenes</Text> : null}
|
ListEmptyComponent={!isLoading ? <Text style={styles.muted}>no scenes</Text> : null}
|
||||||
|
|
|
||||||
|
|
@ -147,7 +147,7 @@ export function StudioScenesScreen() {
|
||||||
ListHeaderComponent={
|
ListHeaderComponent={
|
||||||
<View style={styles.header}>
|
<View style={styles.header}>
|
||||||
<Text style={styles.subtitle}>
|
<Text style={styles.subtitle}>
|
||||||
{data ? `${data.total} ${data.total === 1 ? 'scene' : 'scenes'}` : ' '}
|
{data ? `${data.total}${data.total_capped ? '+' : ''} ${data.total === 1 ? 'scene' : 'scenes'}` : ' '}
|
||||||
</Text>
|
</Text>
|
||||||
</View>
|
</View>
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -55,7 +55,7 @@ export function TagScenesScreen() {
|
||||||
ListHeaderComponent={
|
ListHeaderComponent={
|
||||||
data ? (
|
data ? (
|
||||||
<Text style={styles.subtitle}>
|
<Text style={styles.subtitle}>
|
||||||
{data.total} {data.total === 1 ? 'scene' : 'scenes'} · with playback
|
{data.total}{data.total_capped ? '+' : ''} {data.total === 1 ? 'scene' : 'scenes'} · with playback
|
||||||
</Text>
|
</Text>
|
||||||
) : null
|
) : null
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -166,6 +166,12 @@ export interface SceneListOut {
|
||||||
total: number;
|
total: number;
|
||||||
page: number;
|
page: number;
|
||||||
per_page: number;
|
per_page: number;
|
||||||
|
// has_more: czy jest kolejna strona (z fetcha per_page+1). Paginuj po TYM, nie po
|
||||||
|
// `total` — bo dla list filtrowanych `total` jest bounded ("1000+"). Optional dla
|
||||||
|
// wstecznej zgodności ze starym backendem (undefined → fallback na total w UI).
|
||||||
|
has_more?: boolean;
|
||||||
|
// total_capped: true gdy `total` to cap (jest >total wyników) → pokaż "{total}+".
|
||||||
|
total_capped?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface MovieChapterOut {
|
export interface MovieChapterOut {
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue