diff --git a/app/api/scenes.py b/app/api/scenes.py index 3f0813a..55b556c 100644 --- a/app/api/scenes.py +++ b/app/api/scenes.py @@ -1,7 +1,9 @@ """GET /scenes, lista i szczegóły scen z bazy kanonicznej.""" from __future__ import annotations +import collections import logging +import math import re import hashlib import uuid @@ -55,6 +57,47 @@ router = APIRouter(prefix="/scenes", tags=["scenes"], dependencies=[Depends(requ #: Ile wynikow filtra bierze udzial w losowaniu (patrz sort=random z filtrem). _RANDOM_POOL = 5000 +#: Glowny feed "newest ingest": najwiekszy udzial jednego zrodla na stronie i ile pierwszych +#: stron tak ukladamy (dalej zwykla kolejnosc po created_at). Patrz _diversify_pages. +_FEED_MAX_SHARE = 1 / 3 +_FEED_DIVERSE_PAGES = 8 + + +def _diversify_pages( + items: list[tuple[uuid.UUID, str]], page_size: int, max_share: float +) -> list[list[uuid.UUID]]: + """Rozklada liste (id, zrodlo), juz w kolejnosci czasu, na strony tak, by jedno zrodlo + zajmowalo najwyzej `max_share` strony, dopoki sa sceny z innych zrodel. Strona bierze + najpierw najnowsze sceny w ramach limitu na zrodlo; gdy innych zrodel brakuje, dopelnia + sie najstarszymi z nadmiaru, wiec zadna strona nie zostaje dziurawa i nic nie ginie. + W obrebie strony i w obrebie zrodla kolejnosc czasu zostaje. + + Po co: eporner dokladal ~2 tys. anonimowych uploadow dziennie przy kilkuset nowych + scenach z calej reszty i po odcieciu archiwum (backfill) zajmowal 121 ze 150 pozycji + pierwszych trzech stron (2026-09-19). + """ + cap = max(1, math.ceil(page_size * max_share)) + remaining = list(enumerate(items)) + pages: list[list[uuid.UUID]] = [] + while remaining: + taken: list[tuple[int, tuple[uuid.UUID, str]]] = [] + overflow: list[tuple[int, tuple[uuid.UUID, str]]] = [] + counts: collections.Counter = collections.Counter() + for pos, item in remaining: + if len(taken) < page_size and counts[item[1]] < cap: + taken.append((pos, item)) + counts[item[1]] += 1 + else: + overflow.append((pos, item)) + while len(taken) < page_size and overflow: + taken.append(overflow.pop(0)) + taken.sort(key=lambda t: t[0]) + pages.append([item[0] for _, item in taken]) + used = {pos for pos, _ in taken} + remaining = [(pos, item) for pos, item in remaining if pos not in used] + return pages + + _VALID_SORTS = { "created_at", "release_date", @@ -599,6 +642,21 @@ def list_scenes( # stubem. Filtruje anonymous tube-only sceny z newUrl/categories ingestu. base = base.where(stub_exclusion_clause()) + # Glowny feed "newest ingest" bez zawezen: bez scen z dociagania archiwum (backfill). + # deep-crawl tubepornclassic dokladal 7,9 tys. scen z lat 2016-2018 dziennie, a xhamster, + # xnxx, xvideos i sxyprn kolejne 3,3 tys., wszystkie z dzisiejszym created_at, wiec + # zajmowaly caly szczyt listy (zgloszenie 2026-09-19: "glownie theclassicporn i eporner"). + # Z filtrem (szukanie, tag, obsada, studio, strona) i w sekcjach JAV/Anime zostaja, bo + # tam user chce wszystkiego z danego zakresu, a nie przegladu nowosci. + _plain_feed = ( + sort == "created_at" + and not (q or tag_slug_list or perf_id_strings or studio_slug_list or origin) + and vertical is None + and not jav + ) + if _plain_feed: + base = base.where(Scene.backfill.is_(False)) + _is_pure_default = ( not include_stubs and not q and not studio_slug_list and not tag_slug_list and not perf_id_strings and origin is None and has_playback is None @@ -759,17 +817,40 @@ def list_scenes( # psuje early-termination i przy filtrach EXISTS planer robi gather-all+sort (sekundy) # zamiast limit-aware index-walk po `ix_scenes_created_at_desc`. page/per_page to # walidowane inty (Query ge=1, le=200), więc literal_column jest bezpieczne. - rows = ( - studio_rows - if studio_rows is not None - else session.execute( - ordered.offset(literal_column(str(_off))).limit(literal_column(str(_want))) + if _plain_feed and page <= _FEED_DIVERSE_PAGES: + # Kandydatki (id + zrodlo) w kolejnosci czasu, ulozone przez _diversify_pages. Pula + # ma STALA wielkosc (2x tyle, ile stron ukladamy), bo inaczej kazda strona liczylaby + # uklad z innej puli i sceny przeskakiwalyby miedzy stronami. Pelne wiersze ladujemy + # tylko dla zwracanej strony. + cand_n = _FEED_DIVERSE_PAGES * per_page * 2 + origin_col = ( + select(func.min(PlaybackSource.origin)) + .where(PlaybackSource.scene_id == Scene.id, PlaybackSource.dead_at.is_(None)) + .correlate(Scene) + .scalar_subquery() ) - .scalars() - .all() - ) - has_more = len(rows) > per_page - rows = rows[:per_page] + cand = session.execute( + ordered.with_only_columns(Scene.id, origin_col).limit(literal_column(str(cand_n))) + ).all() + feed_pages = _diversify_pages([(r[0], r[1] or "") for r in cand], per_page, _FEED_MAX_SHARE) + page_ids = feed_pages[page - 1] if page <= len(feed_pages) else [] + by_id = { + s.id: s for s in session.execute(select(Scene).where(Scene.id.in_(page_ids))).scalars() + } if page_ids else {} + rows = [by_id[i] for i in page_ids if i in by_id] + has_more = page < len(feed_pages) or len(cand) >= cand_n + else: + rows = ( + studio_rows + if studio_rows is not None + else session.execute( + ordered.offset(literal_column(str(_off))).limit(literal_column(str(_want))) + ) + .scalars() + .all() + ) + has_more = len(rows) > per_page + rows = rows[:per_page] # Filtrowane listy: total = dolna granica z dotychczas-widzianych wierszy, a # total_capped=has_more daje UI "N+" (jest kolejna strona). Bez osobnego count query. diff --git a/tests/test_feed_diversify.py b/tests/test_feed_diversify.py new file mode 100644 index 0000000..3fd988b --- /dev/null +++ b/tests/test_feed_diversify.py @@ -0,0 +1,51 @@ +import uuid + +from app.api.scenes import _diversify_pages + + +def _ids(n, origin): + return [(uuid.uuid4(), origin) for _ in range(n)] + + +def _mixed(flood_share=0.7, n=300, others=("b", "c", "d")): + """Jak feed 2026-09-19: jedno zrodlo zalewa, reszta kapie, wszystko w kolejnosci czasu.""" + items, k = [], 0 + for i in range(n): + if (i % 10) < flood_share * 10: + items.append((uuid.uuid4(), "flood")) + else: + items.append((uuid.uuid4(), others[k % len(others)])) + k += 1 + return items + + +def test_flood_is_capped_while_other_sources_exist(): + items = _mixed() + pages = _diversify_pages(items, 10, 1 / 3) + origin = dict(items) + # 90 scen z innych zrodel wystarcza na 15 stron po 6; tyle stron trzyma limit + for pg in pages[:15]: + assert len(pg) == 10 + assert sum(origin[i] == "flood" for i in pg) <= 4 + + +def test_nothing_is_lost_and_pages_are_full(): + items = _mixed() + pages = _diversify_pages(items, 10, 1 / 3) + assert sorted(i for pg in pages for i in pg) == sorted(i for i, _ in items) + assert all(len(pg) == 10 for pg in pages[:-1]) + + +def test_two_sources_still_fill_pages(): + items = _ids(40, "a") + _ids(40, "b") + pages = _diversify_pages(items, 10, 1 / 3) + assert all(len(pg) == 10 for pg in pages) + + +def test_order_within_source_is_kept(): + items = _mixed() + flat = [i for pg in _diversify_pages(items, 10, 1 / 3) for i in pg] + for src in ("flood", "b", "c", "d"): + mine = [i for i, o in items if o == src] + mine_set = set(mine) + assert [i for i in flat if i in mine_set] == mine