refactor(review): dedup favorites/list visibility + SQL-aggregate count, share mobile isNew
Follow-up cleanup from the ultra-review (behaviour-preserving, verified equivalent): - scenes.py: extract live_playback_exists / blacklist_clauses / stub_exclusion_clause; list_scenes and favorites now share ONE definition of "visible scene" (was a hand-kept copy in favorites -> the count-vs-list drift class). Verified identical: helper vs old inline both count 1,853,327 scenes. - favorites.py: replace the two copy-pasted count blocks with one _new_counts(kind=...) that counts in SQL (count(*) FILTER over the windowed subquery) instead of streaming up to N*200 rows to Python; joins the favorite table for per-row last_seen. Deployed _new_counts verified == hand SQL (studios 364). Dropped now-unused imports. - mobile: extract lib/newScenes.ts (isNewScene / sortNewFirst); SceneTile + Performer/ StudioScenes use it (was triplicated, already drifted once in the deleted FavoriteSceneRow). - SceneDetail: fold tube:pornxpph into the phoneResolver map (was a verbatim-duplicated block). - deep_crawl: persist the cursor after every completed page (was once at run end), so a mid-page hard-kill past the soft budget can't lose progress (GOON-V hardening). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
15e7c1646d
commit
cb0b843f48
8 changed files with 211 additions and 288 deletions
|
|
@ -38,8 +38,7 @@ from app.models.favorite_performer import FavoritePerformer
|
||||||
from app.models.favorite_studio import FavoriteStudio
|
from app.models.favorite_studio import FavoriteStudio
|
||||||
from app.models.movie import Movie
|
from app.models.movie import Movie
|
||||||
from app.models.performer import Performer
|
from app.models.performer import Performer
|
||||||
from app.models.playback_source import PlaybackSource
|
from app.models.scene import Scene, ScenePerformer
|
||||||
from app.models.scene import Scene, ScenePerformer, SceneTag
|
|
||||||
from app.models.studio import Studio
|
from app.models.studio import Studio
|
||||||
|
|
||||||
router = APIRouter(
|
router = APIRouter(
|
||||||
|
|
@ -57,79 +56,76 @@ router = APIRouter(
|
||||||
_FAVORITES_PAGE_CAP = 200 # == per_page w PerformerScenesScreen/StudioScenesScreen
|
_FAVORITES_PAGE_CAP = 200 # == per_page w PerformerScenesScreen/StudioScenesScreen
|
||||||
|
|
||||||
|
|
||||||
def _visible_scene_clauses(session: Session, device_id: str, *, apply_stub: bool) -> list:
|
def _new_counts(session: Session, device_id: str, *, kind: str) -> dict:
|
||||||
"""Klauzule WHERE = filtry widoczności listy scen: żywy playback_source +
|
"""Policz per-favorite "+N nowych" = sceny created_at > last_seen_at, ale TYLKO wśród
|
||||||
blacklisty device + (opcjonalnie) odsianie stub-scen. Aplikowane na zapytanie
|
tych które user zobaczy na liście: te same filtry widoczności co list_scenes (żywy
|
||||||
z Scene w FROM."""
|
playback + blacklist + stub) i to samo okno (top-_FAVORITES_PAGE_CAP pod sortem
|
||||||
from sqlalchemy import exists
|
release_date desc), z pominięciem backfillu. Liczy w SQL (count(*) FILTER), zwraca
|
||||||
|
{group_id: n} — bez streamowania N*200 wierszy do Pythona.
|
||||||
|
|
||||||
clauses = [
|
kind="performer": grupuje po ScenePerformer.performer_id, join favorite_performers.
|
||||||
exists(
|
kind="studio": grupuje po Scene.studio_id, join favorite_studios. Studia potrzebują
|
||||||
select(1).where(
|
odsiewu stub (mogą nie mieć performera); performerki nie (mają).
|
||||||
PlaybackSource.scene_id == Scene.id,
|
"""
|
||||||
PlaybackSource.dead_at.is_(None),
|
from sqlalchemy import and_, func
|
||||||
)
|
|
||||||
)
|
|
||||||
]
|
|
||||||
from app.api.scenes import _blacklists_empty
|
|
||||||
|
|
||||||
if not _blacklists_empty(session, device_id):
|
from app.api.scenes import blacklist_clauses, live_playback_exists, stub_exclusion_clause
|
||||||
from app.models.blacklist import (
|
|
||||||
BlacklistedPerformer,
|
|
||||||
BlacklistedStudio,
|
|
||||||
BlacklistedTag,
|
|
||||||
)
|
|
||||||
|
|
||||||
clauses.append(
|
clauses = [live_playback_exists(), *blacklist_clauses(session, device_id)]
|
||||||
~exists(
|
|
||||||
select(1)
|
|
||||||
.select_from(ScenePerformer)
|
|
||||||
.join(
|
|
||||||
BlacklistedPerformer,
|
|
||||||
(BlacklistedPerformer.performer_id == ScenePerformer.performer_id)
|
|
||||||
& (BlacklistedPerformer.device_id == device_id),
|
|
||||||
)
|
|
||||||
.where(ScenePerformer.scene_id == Scene.id)
|
|
||||||
)
|
|
||||||
)
|
|
||||||
clauses.append(
|
|
||||||
~Scene.studio_id.in_(
|
|
||||||
select(BlacklistedStudio.studio_id).where(
|
|
||||||
BlacklistedStudio.device_id == device_id
|
|
||||||
)
|
|
||||||
)
|
|
||||||
)
|
|
||||||
clauses.append(
|
|
||||||
~exists(
|
|
||||||
select(1)
|
|
||||||
.select_from(SceneTag)
|
|
||||||
.join(
|
|
||||||
BlacklistedTag,
|
|
||||||
(BlacklistedTag.tag_id == SceneTag.tag_id)
|
|
||||||
& (BlacklistedTag.device_id == device_id),
|
|
||||||
)
|
|
||||||
.where(SceneTag.scene_id == Scene.id)
|
|
||||||
)
|
|
||||||
)
|
|
||||||
if apply_stub:
|
|
||||||
# Stub = tube-only scena bez release_date AND bez canonical (TPDB/StashDB) AND
|
|
||||||
# bez performera. Dla widoku performerki nigdy nie zachodzi (ma performera), więc
|
|
||||||
# apply_stub=False tam; dla studiów tak. Lustro scenes.py:348-367.
|
|
||||||
from app.models.scene import SceneExternalRef
|
|
||||||
from app.models.source import Source, SourceKind
|
|
||||||
|
|
||||||
canonical_exists = exists(
|
if kind == "performer":
|
||||||
select(1)
|
gid = ScenePerformer.performer_id
|
||||||
.select_from(SceneExternalRef)
|
last_seen = FavoritePerformer.last_seen_at
|
||||||
.join(Source, Source.id == SceneExternalRef.source_id)
|
base = (
|
||||||
.where(SceneExternalRef.scene_id == Scene.id)
|
select(
|
||||||
.where(Source.kind.in_([SourceKind.tpdb, SourceKind.stashdb]))
|
gid.label("gid"),
|
||||||
|
Scene.created_at.label("created_at"),
|
||||||
|
Scene.backfill.label("backfill"),
|
||||||
|
last_seen.label("last_seen"),
|
||||||
)
|
)
|
||||||
has_performer = exists(select(1).where(ScenePerformer.scene_id == Scene.id))
|
.select_from(FavoritePerformer)
|
||||||
clauses.append(
|
.join(ScenePerformer, ScenePerformer.performer_id == FavoritePerformer.performer_id)
|
||||||
Scene.release_date.is_not(None) | canonical_exists | has_performer
|
.join(Scene, Scene.id == ScenePerformer.scene_id)
|
||||||
|
.where(FavoritePerformer.device_id == device_id)
|
||||||
)
|
)
|
||||||
return clauses
|
partition = ScenePerformer.performer_id
|
||||||
|
else:
|
||||||
|
clauses.append(stub_exclusion_clause())
|
||||||
|
gid = Scene.studio_id
|
||||||
|
last_seen = FavoriteStudio.last_seen_at
|
||||||
|
base = (
|
||||||
|
select(
|
||||||
|
gid.label("gid"),
|
||||||
|
Scene.created_at.label("created_at"),
|
||||||
|
Scene.backfill.label("backfill"),
|
||||||
|
last_seen.label("last_seen"),
|
||||||
|
)
|
||||||
|
.select_from(FavoriteStudio)
|
||||||
|
.join(Scene, Scene.studio_id == FavoriteStudio.studio_id)
|
||||||
|
.where(FavoriteStudio.device_id == device_id)
|
||||||
|
)
|
||||||
|
partition = Scene.studio_id
|
||||||
|
|
||||||
|
rn = func.row_number().over(
|
||||||
|
partition_by=partition,
|
||||||
|
order_by=(Scene.release_date.desc().nullslast(), Scene.created_at.desc()),
|
||||||
|
).label("rn")
|
||||||
|
inner = base.add_columns(rn).where(*clauses).subquery()
|
||||||
|
rows = session.execute(
|
||||||
|
select(
|
||||||
|
inner.c.gid,
|
||||||
|
func.count()
|
||||||
|
.filter(
|
||||||
|
and_(
|
||||||
|
inner.c.rn <= _FAVORITES_PAGE_CAP,
|
||||||
|
inner.c.backfill.is_(False),
|
||||||
|
inner.c.created_at > inner.c.last_seen,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
.label("n"),
|
||||||
|
).group_by(inner.c.gid)
|
||||||
|
).all()
|
||||||
|
return {gid_val: int(n) for gid_val, n in rows}
|
||||||
|
|
||||||
|
|
||||||
class FavoriteOut(BaseModel):
|
class FavoriteOut(BaseModel):
|
||||||
|
|
@ -162,48 +158,11 @@ def list_favorites(
|
||||||
if not rows:
|
if not rows:
|
||||||
return FavoriteListOut(items=[], total=0, new_total=0)
|
return FavoriteListOut(items=[], total=0, new_total=0)
|
||||||
|
|
||||||
perf_ids = [perf.id for _, perf in rows]
|
|
||||||
last_seen_by_perf = {fav.performer_id: fav.last_seen_at for fav, _ in rows}
|
|
||||||
|
|
||||||
# scene_count: czytamy zdenormalizowany Performer.scene_count (refresh w tle przez
|
# scene_count: czytamy zdenormalizowany Performer.scene_count (refresh w tle przez
|
||||||
# _job_refresh_taxonomy_counts) — ta sama definicja co przed (sceny z żywym
|
# _job_refresh_taxonomy_counts) — ta sama definicja co przed (sceny z żywym
|
||||||
# playback). Wcześniej grouped count z EXISTS playback per-request. Migracja 0019.
|
# playback). Wcześniej grouped count z EXISTS playback per-request. Migracja 0019.
|
||||||
scene_counts: dict = {perf.id: perf.scene_count for _, perf in rows}
|
scene_counts: dict = {perf.id: perf.scene_count for _, perf in rows}
|
||||||
|
new_counts = _new_counts(session, device_id, kind="performer")
|
||||||
# new_count per performer = sceny created_at > last_seen_at, ale liczone TYLKO wśród
|
|
||||||
# tych, które user faktycznie zobaczy na liście: te same filtry (żywy playback +
|
|
||||||
# blacklist) i to samo okno (top-_FAVORITES_PAGE_CAP pod release_date desc) co
|
|
||||||
# PerformerScenes. apply_stub=False, widok performerki i tak ma performera (nie-stub).
|
|
||||||
new_counts: dict = {}
|
|
||||||
if perf_ids:
|
|
||||||
from sqlalchemy import func
|
|
||||||
|
|
||||||
clauses = _visible_scene_clauses(session, device_id, apply_stub=False)
|
|
||||||
rn = func.row_number().over(
|
|
||||||
partition_by=ScenePerformer.performer_id,
|
|
||||||
order_by=(Scene.release_date.desc().nullslast(), Scene.created_at.desc()),
|
|
||||||
).label("rn")
|
|
||||||
inner = (
|
|
||||||
select(
|
|
||||||
ScenePerformer.performer_id.label("gid"),
|
|
||||||
Scene.created_at.label("created_at"),
|
|
||||||
Scene.backfill.label("backfill"),
|
|
||||||
rn,
|
|
||||||
)
|
|
||||||
.join(Scene, Scene.id == ScenePerformer.scene_id)
|
|
||||||
.where(ScenePerformer.performer_id.in_(perf_ids))
|
|
||||||
.where(*clauses)
|
|
||||||
.subquery()
|
|
||||||
)
|
|
||||||
for gid, created_at, backfill in session.execute(
|
|
||||||
select(inner.c.gid, inner.c.created_at, inner.c.backfill).where(
|
|
||||||
inner.c.rn <= _FAVORITES_PAGE_CAP
|
|
||||||
)
|
|
||||||
):
|
|
||||||
ls = last_seen_by_perf.get(gid)
|
|
||||||
# backfill (masowy import katalogu) NIE liczy się jako nowość, patrz Scene.backfill.
|
|
||||||
if not backfill and created_at is not None and ls is not None and created_at > ls:
|
|
||||||
new_counts[gid] = new_counts.get(gid, 0) + 1
|
|
||||||
|
|
||||||
items: list[FavoriteOut] = []
|
items: list[FavoriteOut] = []
|
||||||
new_total = 0
|
new_total = 0
|
||||||
|
|
@ -316,44 +275,9 @@ def list_favorite_studios(
|
||||||
if not rows:
|
if not rows:
|
||||||
return FavoriteStudioListOut(items=[], total=0, new_total=0)
|
return FavoriteStudioListOut(items=[], total=0, new_total=0)
|
||||||
|
|
||||||
studio_ids = [st.id for _, st in rows]
|
|
||||||
last_seen_by_studio = {fav.studio_id: fav.last_seen_at for fav, _ in rows}
|
|
||||||
|
|
||||||
# scene_count: zdenormalizowany Studio.scene_count (refresh w tle, migracja 0019).
|
# scene_count: zdenormalizowany Studio.scene_count (refresh w tle, migracja 0019).
|
||||||
scene_counts: dict = {st.id: st.scene_count for _, st in rows}
|
scene_counts: dict = {st.id: st.scene_count for _, st in rows}
|
||||||
|
new_counts = _new_counts(session, device_id, kind="studio")
|
||||||
# new_count per studio, jak dla performerów (patrz list_favorites): te same filtry
|
|
||||||
# + okno co lista StudioScenes. apply_stub=True: studio-scena bez performera/release/
|
|
||||||
# canonical to stub który lista odsiewa, więc licznik też musi.
|
|
||||||
new_counts: dict = {}
|
|
||||||
if studio_ids:
|
|
||||||
from sqlalchemy import func
|
|
||||||
|
|
||||||
clauses = _visible_scene_clauses(session, device_id, apply_stub=True)
|
|
||||||
rn = func.row_number().over(
|
|
||||||
partition_by=Scene.studio_id,
|
|
||||||
order_by=(Scene.release_date.desc().nullslast(), Scene.created_at.desc()),
|
|
||||||
).label("rn")
|
|
||||||
inner = (
|
|
||||||
select(
|
|
||||||
Scene.studio_id.label("gid"),
|
|
||||||
Scene.created_at.label("created_at"),
|
|
||||||
Scene.backfill.label("backfill"),
|
|
||||||
rn,
|
|
||||||
)
|
|
||||||
.where(Scene.studio_id.in_(studio_ids))
|
|
||||||
.where(*clauses)
|
|
||||||
.subquery()
|
|
||||||
)
|
|
||||||
for gid, created_at, backfill in session.execute(
|
|
||||||
select(inner.c.gid, inner.c.created_at, inner.c.backfill).where(
|
|
||||||
inner.c.rn <= _FAVORITES_PAGE_CAP
|
|
||||||
)
|
|
||||||
):
|
|
||||||
ls = last_seen_by_studio.get(gid)
|
|
||||||
# backfill (masowy import katalogu) NIE liczy się jako nowość, patrz Scene.backfill.
|
|
||||||
if not backfill and created_at is not None and ls is not None and created_at > ls:
|
|
||||||
new_counts[gid] = new_counts.get(gid, 0) + 1
|
|
||||||
|
|
||||||
items: list[FavoriteStudioOut] = []
|
items: list[FavoriteStudioOut] = []
|
||||||
new_total = 0
|
new_total = 0
|
||||||
|
|
|
||||||
|
|
@ -108,6 +108,73 @@ def _split_csv(raw: str | None) -> list[str]:
|
||||||
return [s.strip() for s in raw.split(",") if s.strip()]
|
return [s.strip() for s in raw.split(",") if s.strip()]
|
||||||
|
|
||||||
|
|
||||||
|
# ---- Współdzielone klauzule widoczności sceny ----------------------------------
|
||||||
|
# Definicja "co user widzi na liście" żyje TU i jest reużywana przez list_scenes ORAZ
|
||||||
|
# licznik "+N nowych" w app/api/favorites.py. Wcześniej favorites miał ręczną kopię tych
|
||||||
|
# klauzul (dryf: zmiana filtra listy nie trafiała do licznika → "+6 a nic nowego").
|
||||||
|
|
||||||
|
|
||||||
|
def live_playback_exists():
|
||||||
|
"""EXISTS: scena ma choć jeden żywy playback_source."""
|
||||||
|
return exists(
|
||||||
|
select(1).where(
|
||||||
|
PlaybackSource.scene_id == Scene.id,
|
||||||
|
PlaybackSource.dead_at.is_(None),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def blacklist_clauses(session: Session, device_id: str) -> list:
|
||||||
|
"""NOT-EXISTS klauzule blacklist device (performer/studio/tag). [] gdy wszystkie puste."""
|
||||||
|
if _blacklists_empty(session, device_id):
|
||||||
|
return []
|
||||||
|
from app.models.blacklist import (
|
||||||
|
BlacklistedPerformer,
|
||||||
|
BlacklistedStudio,
|
||||||
|
BlacklistedTag,
|
||||||
|
)
|
||||||
|
|
||||||
|
return [
|
||||||
|
~exists(
|
||||||
|
select(1)
|
||||||
|
.select_from(ScenePerformer)
|
||||||
|
.join(
|
||||||
|
BlacklistedPerformer,
|
||||||
|
(BlacklistedPerformer.performer_id == ScenePerformer.performer_id)
|
||||||
|
& (BlacklistedPerformer.device_id == device_id),
|
||||||
|
)
|
||||||
|
.where(ScenePerformer.scene_id == Scene.id)
|
||||||
|
),
|
||||||
|
~Scene.studio_id.in_(
|
||||||
|
select(BlacklistedStudio.studio_id).where(BlacklistedStudio.device_id == device_id)
|
||||||
|
),
|
||||||
|
~exists(
|
||||||
|
select(1)
|
||||||
|
.select_from(SceneTag)
|
||||||
|
.join(
|
||||||
|
BlacklistedTag,
|
||||||
|
(BlacklistedTag.tag_id == SceneTag.tag_id)
|
||||||
|
& (BlacklistedTag.device_id == device_id),
|
||||||
|
)
|
||||||
|
.where(SceneTag.scene_id == Scene.id)
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def stub_exclusion_clause():
|
||||||
|
"""Odsiew stub-scen: tube-only bez release_date AND bez canonical (TPDB/StashDB) AND
|
||||||
|
bez performera. NOT stub gdy ma release_date OR canonical OR performera."""
|
||||||
|
canonical_exists = exists(
|
||||||
|
select(1)
|
||||||
|
.select_from(SceneExternalRef)
|
||||||
|
.join(Source, Source.id == SceneExternalRef.source_id)
|
||||||
|
.where(SceneExternalRef.scene_id == Scene.id)
|
||||||
|
.where(Source.kind.in_([SourceKind.tpdb, SourceKind.stashdb]))
|
||||||
|
)
|
||||||
|
has_performer = exists(select(1).where(ScenePerformer.scene_id == Scene.id))
|
||||||
|
return Scene.release_date.is_not(None) | canonical_exists | has_performer
|
||||||
|
|
||||||
|
|
||||||
@router.get("", response_model=SceneListOut)
|
@router.get("", response_model=SceneListOut)
|
||||||
def list_scenes(
|
def list_scenes(
|
||||||
session: Annotated[Session, Depends(get_session)],
|
session: Annotated[Session, Depends(get_session)],
|
||||||
|
|
@ -239,24 +306,9 @@ def list_scenes(
|
||||||
)
|
)
|
||||||
|
|
||||||
if has_playback is True:
|
if has_playback is True:
|
||||||
# Tylko sceny z choć jednym ŻYWYM playback_source.
|
base = base.where(live_playback_exists())
|
||||||
base = base.where(
|
|
||||||
exists(
|
|
||||||
select(1).where(
|
|
||||||
PlaybackSource.scene_id == Scene.id,
|
|
||||||
PlaybackSource.dead_at.is_(None),
|
|
||||||
)
|
|
||||||
)
|
|
||||||
)
|
|
||||||
elif has_playback is False:
|
elif has_playback is False:
|
||||||
base = base.where(
|
base = base.where(~live_playback_exists())
|
||||||
~exists(
|
|
||||||
select(1).where(
|
|
||||||
PlaybackSource.scene_id == Scene.id,
|
|
||||||
PlaybackSource.dead_at.is_(None),
|
|
||||||
)
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
if origin:
|
if origin:
|
||||||
# Substring match na origin — 'hqporner' złapie 'tube:hqpornercom'.
|
# Substring match na origin — 'hqporner' złapie 'tube:hqpornercom'.
|
||||||
|
|
@ -270,46 +322,10 @@ def list_scenes(
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
# Blacklisty — globalne wykluczenia. Jeśli scena ma JAKIEGOKOLWIEK blacklisted
|
# Blacklisty device (performer/studio/tag) — globalne wykluczenia, współdzielone z
|
||||||
# performera, jest na blacklisted studio, lub ma JAKIKOLWIEK blacklisted tag → out.
|
# licznikiem +N ulubionych. Puste blacklisty → [] (typowy single-user, zero kosztu).
|
||||||
# Pomijamy gdy wszystkie 3 blacklisty puste (typowy stan single-user) — te NOT EXISTS
|
for _bl_clause in blacklist_clauses(session, device_id):
|
||||||
# ewaluują się per-row na ~176k scen przy mega-tagu i kosztowały ~3.4s za nic.
|
base = base.where(_bl_clause)
|
||||||
if not _blacklists_empty(session, device_id):
|
|
||||||
from app.models.blacklist import (
|
|
||||||
BlacklistedPerformer,
|
|
||||||
BlacklistedStudio,
|
|
||||||
BlacklistedTag,
|
|
||||||
)
|
|
||||||
base = base.where(
|
|
||||||
~exists(
|
|
||||||
select(1)
|
|
||||||
.select_from(ScenePerformer)
|
|
||||||
.join(
|
|
||||||
BlacklistedPerformer,
|
|
||||||
(BlacklistedPerformer.performer_id == ScenePerformer.performer_id)
|
|
||||||
& (BlacklistedPerformer.device_id == device_id),
|
|
||||||
)
|
|
||||||
.where(ScenePerformer.scene_id == Scene.id)
|
|
||||||
)
|
|
||||||
)
|
|
||||||
base = base.where(
|
|
||||||
~Scene.studio_id.in_(
|
|
||||||
select(BlacklistedStudio.studio_id).where(BlacklistedStudio.device_id == device_id)
|
|
||||||
)
|
|
||||||
)
|
|
||||||
base = base.where(
|
|
||||||
~exists(
|
|
||||||
select(1)
|
|
||||||
.select_from(SceneTag)
|
|
||||||
.join(
|
|
||||||
BlacklistedTag,
|
|
||||||
(BlacklistedTag.tag_id == SceneTag.tag_id)
|
|
||||||
& (BlacklistedTag.device_id == device_id),
|
|
||||||
)
|
|
||||||
.where(SceneTag.scene_id == Scene.id)
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
if min_duration_sec is not None:
|
if min_duration_sec is not None:
|
||||||
base = base.where(Scene.duration_sec >= min_duration_sec)
|
base = base.where(Scene.duration_sec >= min_duration_sec)
|
||||||
|
|
@ -346,25 +362,11 @@ def list_scenes(
|
||||||
)
|
)
|
||||||
|
|
||||||
if not include_stubs:
|
if not include_stubs:
|
||||||
# Stub scene heuristic: tube-only scena BEZ release_date AND BEZ canonical
|
# Stub scene heuristic (współdzielona z licznikiem +N ulubionych): tube-only scena
|
||||||
# (TPDB/StashDB) ref AND BEZ żadnego ScenePerformer linka. ScenePerformer
|
# bez release_date AND bez canonical (TPDB/StashDB) AND bez performera. Continuous
|
||||||
# dodaje continuous worker (search-by-name → wymusza link), więc per-performer
|
# worker dodaje ScenePerformer (search-by-name), więc per-performer wynik nie jest
|
||||||
# search-result NIGDY nie jest stub. To filtruje tylko anonymous tube-only
|
# stubem. Filtruje anonymous tube-only sceny z newUrl/categories ingestu.
|
||||||
# sceny z newUrl/categories ingestu które nie zostały zsyntowane z performerem.
|
base = base.where(stub_exclusion_clause())
|
||||||
canonical_exists = exists(
|
|
||||||
select(1)
|
|
||||||
.select_from(SceneExternalRef)
|
|
||||||
.join(Source, Source.id == SceneExternalRef.source_id)
|
|
||||||
.where(SceneExternalRef.scene_id == Scene.id)
|
|
||||||
.where(Source.kind.in_([SourceKind.tpdb, SourceKind.stashdb]))
|
|
||||||
)
|
|
||||||
has_performer = exists(
|
|
||||||
select(1).where(ScenePerformer.scene_id == Scene.id)
|
|
||||||
)
|
|
||||||
# NOT stub gdy: ma canonical_ref OR ma release_date OR ma performera
|
|
||||||
base = base.where(
|
|
||||||
Scene.release_date.is_not(None) | canonical_exists | has_performer
|
|
||||||
)
|
|
||||||
|
|
||||||
_is_pure_default = (
|
_is_pure_default = (
|
||||||
not include_stubs and not q and not studio_slug_list and not tag_slug_list
|
not include_stubs and not q and not studio_slug_list and not tag_slug_list
|
||||||
|
|
|
||||||
|
|
@ -140,6 +140,20 @@ def run_deep_crawl(*, pages_per_run: int = 60, sitetags: list[str] | None = None
|
||||||
exhausted = False
|
exhausted = False
|
||||||
budget_hit = False
|
budget_hit = False
|
||||||
|
|
||||||
|
st = state.setdefault(sitetag, {})
|
||||||
|
|
||||||
|
def _persist() -> None:
|
||||||
|
# Zapisz kursor po KAŻDEJ skończonej stronie. Bez tego stan leciał raz na końcu
|
||||||
|
# runu, więc hard-timeout ubijający run mid-page (mimo miękkiego budżetu, gdy
|
||||||
|
# pojedyncza strona przez wolne proxy przekroczy margines) gubił cały postęp
|
||||||
|
# (orphan thread, GOON-V). Per-page persist = najgorszy przypadek to powtórka
|
||||||
|
# jednej strony. Zapis jest atomowy (tmp+replace) i tani względem fetcha strony.
|
||||||
|
st["last_page"] = last_done
|
||||||
|
st["exhausted"] = exhausted
|
||||||
|
st["swept_once"] = swept_once or exhausted
|
||||||
|
st["updated_at"] = int(time.time())
|
||||||
|
_save_state(state)
|
||||||
|
|
||||||
if cap is not None and start > cap:
|
if cap is not None and start > cap:
|
||||||
# kursor osiągnął per-tube cap → traktuj jak koniec katalogu (reset re-sweepuje od 1)
|
# kursor osiągnął per-tube cap → traktuj jak koniec katalogu (reset re-sweepuje od 1)
|
||||||
exhausted = True
|
exhausted = True
|
||||||
|
|
@ -167,8 +181,9 @@ def run_deep_crawl(*, pages_per_run: int = 60, sitetags: list[str] | None = None
|
||||||
except Exception:
|
except Exception:
|
||||||
counters["errors"] += 1
|
counters["errors"] += 1
|
||||||
last_done = page
|
last_done = page
|
||||||
# Miękki budżet: stop po skończonej stronie (kursor=last_done zapisany niżej),
|
_persist() # kursor po każdej stronie — mid-page hard-kill nie gubi postępu
|
||||||
# zanim hard-timeout ubije run mid-page (orphan thread, kursor zgubiony, GOON-V).
|
# Miękki budżet: stop po skończonej stronie (kursor już zapisany), zanim
|
||||||
|
# hard-timeout ubije run mid-page (orphan thread, kursor zgubiony, GOON-V).
|
||||||
if time.time() - t0 > _RUN_BUDGET_SEC:
|
if time.time() - t0 > _RUN_BUDGET_SEC:
|
||||||
budget_hit = True
|
budget_hit = True
|
||||||
log.warning(
|
log.warning(
|
||||||
|
|
@ -181,14 +196,10 @@ def run_deep_crawl(*, pages_per_run: int = 60, sitetags: list[str] | None = None
|
||||||
log.info("deep-crawl %s: reached page cap %d (exhausted)", sitetag, cap)
|
log.info("deep-crawl %s: reached page cap %d (exhausted)", sitetag, cap)
|
||||||
exhausted = True
|
exhausted = True
|
||||||
|
|
||||||
st = state.setdefault(sitetag, {})
|
# Zapis terminalny: utrwala finalne flagi (exhausted z empty-page/cap, swept_once).
|
||||||
st["last_page"] = last_done
|
# swept_once=True gdy tube dobił do końca katalogu — kolejne przejścia (po resecie
|
||||||
st["exhausted"] = exhausted
|
# kursora) to re-sweep, gdzie nowe sceny są genuine, nie backfill.
|
||||||
# Gdy tube dobił do końca katalogu (empty page albo cap), zapamiętaj to na stałe -
|
_persist()
|
||||||
# kolejne przejścia (po resecie kursora) to re-sweep, gdzie nowe sceny są genuine.
|
|
||||||
st["swept_once"] = swept_once or exhausted
|
|
||||||
st["updated_at"] = int(time.time())
|
|
||||||
_save_state(state)
|
|
||||||
|
|
||||||
log.info(
|
log.info(
|
||||||
"deep-crawl %s pages %d-%d: %s exhausted=%s budget_hit=%s (%.0fs)",
|
"deep-crawl %s pages %d-%d: %s exhausted=%s budget_hit=%s (%.0fs)",
|
||||||
|
|
|
||||||
|
|
@ -26,6 +26,7 @@ import React from 'react';
|
||||||
import { Pressable, StyleSheet, Text, View } from 'react-native';
|
import { Pressable, StyleSheet, Text, View } from 'react-native';
|
||||||
|
|
||||||
import { useSceneActions } from '../SceneActionsContext';
|
import { useSceneActions } from '../SceneActionsContext';
|
||||||
|
import { isNewScene } from '../lib/newScenes';
|
||||||
import type { RootStackParamList } from '../navigation';
|
import type { RootStackParamList } from '../navigation';
|
||||||
import { fonts, theme } from '../theme';
|
import { fonts, theme } from '../theme';
|
||||||
import type { SceneOut } from '../types';
|
import type { SceneOut } from '../types';
|
||||||
|
|
@ -80,9 +81,7 @@ function SceneTileBase({ scene, secondLine = 'studio', seenSince, onLongPress }:
|
||||||
};
|
};
|
||||||
|
|
||||||
const dim = scene.finished === true;
|
const dim = scene.finished === true;
|
||||||
// NEW = dodane od ostatniej wizyty, ale NIE backfill (masowy import starego katalogu -
|
const isNew = isNewScene(scene, seenSince);
|
||||||
// tube podaje datę importu jako release_date, więc udawałby świeżość). Spójne z licznikiem +N.
|
|
||||||
const isNew = !!(seenSince && scene.created_at && scene.created_at > seenSince && !scene.backfill);
|
|
||||||
const dur = scene.duration_sec;
|
const dur = scene.duration_sec;
|
||||||
const durLabel =
|
const durLabel =
|
||||||
dur && dur > 0
|
dur && dur > 0
|
||||||
|
|
|
||||||
26
mobile/src/lib/newScenes.ts
Normal file
26
mobile/src/lib/newScenes.ts
Normal file
|
|
@ -0,0 +1,26 @@
|
||||||
|
// Jedna definicja "co jest NOWĄ sceną" dla favorite-driven widoków (PerformerScenes,
|
||||||
|
// StudioScenes, SceneTile). Wcześniej reguła była skopiowana w 3-4 miejscach i dryfowała
|
||||||
|
// (martwy FavoriteSceneRow miał starą wersję bez backfillu). Trzymaj tu.
|
||||||
|
import type { SceneOut } from '../types';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* NEW = dodane od ostatniej wizyty (created_at > seenSince), ale NIE backfill (masowy
|
||||||
|
* import starego katalogu, tube podaje datę importu jako release_date, więc udawałby
|
||||||
|
* świeżość). Spójne z licznikiem "+N" w backendzie (app/api/favorites.py _new_counts).
|
||||||
|
*/
|
||||||
|
export function isNewScene(scene: SceneOut, seenSince: string | undefined): boolean {
|
||||||
|
return !!(seenSince && scene.created_at && scene.created_at > seenSince && !scene.backfill);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** NEW-first: nowe sceny na górę (zachowując kolejność), reszta bez zmian. Bez seenSince
|
||||||
|
* zwraca listę jak jest. */
|
||||||
|
export function sortNewFirst(items: SceneOut[], seenSince: string | undefined): SceneOut[] {
|
||||||
|
if (!seenSince) return items;
|
||||||
|
const newOnes: SceneOut[] = [];
|
||||||
|
const rest: SceneOut[] = [];
|
||||||
|
for (const s of items) {
|
||||||
|
if (isNewScene(s, seenSince)) newOnes.push(s);
|
||||||
|
else rest.push(s);
|
||||||
|
}
|
||||||
|
return [...newOnes, ...rest];
|
||||||
|
}
|
||||||
|
|
@ -21,6 +21,7 @@ import {
|
||||||
import { useClient } from '../ClientContext';
|
import { useClient } from '../ClientContext';
|
||||||
import { MoviePosterCard } from '../components/MoviePosterCard';
|
import { MoviePosterCard } from '../components/MoviePosterCard';
|
||||||
import { SceneTile, sceneGridProps } from '../components/SceneTile';
|
import { SceneTile, sceneGridProps } from '../components/SceneTile';
|
||||||
|
import { sortNewFirst } from '../lib/newScenes';
|
||||||
import { usePreferences } from '../PreferencesContext';
|
import { usePreferences } from '../PreferencesContext';
|
||||||
import { ErrorBoundary } from '../ErrorBoundary';
|
import { ErrorBoundary } from '../ErrorBoundary';
|
||||||
import type { RootStackParamList } from '../navigation';
|
import type { RootStackParamList } from '../navigation';
|
||||||
|
|
@ -170,21 +171,10 @@ export function PerformerScenesScreen() {
|
||||||
|
|
||||||
// Sortowanie: NEW (created_at > seenSince) na górze; reszta po release_date desc
|
// Sortowanie: NEW (created_at > seenSince) na górze; reszta po release_date desc
|
||||||
// jak zwracane z backendu. Bez `seenSince` (entry spoza Favorites) — kolejność nie zmieniana.
|
// jak zwracane z backendu. Bez `seenSince` (entry spoza Favorites) — kolejność nie zmieniana.
|
||||||
const sortedScenes = React.useMemo<SceneOut[]>(() => {
|
const sortedScenes = React.useMemo<SceneOut[]>(
|
||||||
const items = scenesQuery.data?.items ?? [];
|
() => sortNewFirst(scenesQuery.data?.items ?? [], seenSince),
|
||||||
if (!seenSince) return items;
|
[scenesQuery.data?.items, seenSince],
|
||||||
const newOnes: SceneOut[] = [];
|
);
|
||||||
const rest: SceneOut[] = [];
|
|
||||||
for (const s of items) {
|
|
||||||
// NEW-first pomija backfill (masowy import katalogu), spójne z badge + licznikiem +N.
|
|
||||||
if (s.created_at && s.created_at > seenSince && !s.backfill) {
|
|
||||||
newOnes.push(s);
|
|
||||||
} else {
|
|
||||||
rest.push(s);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return [...newOnes, ...rest];
|
|
||||||
}, [scenesQuery.data?.items, seenSince]);
|
|
||||||
|
|
||||||
const movies = moviesQuery.data?.items ?? [];
|
const movies = moviesQuery.data?.items ?? [];
|
||||||
const scenesTotal = scenesQuery.data?.total ?? 0;
|
const scenesTotal = scenesQuery.data?.total ?? 0;
|
||||||
|
|
|
||||||
|
|
@ -599,32 +599,11 @@ function PlaybackButton({
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
// pornxp.ph: CDN token IP-bound (backend 403 cross-IP) → backend oddaje WebView
|
// Phone-side resolve dla IP-bound tubów: CDN token jest bound do IP które POBRAŁO
|
||||||
// fallback który czarno-ekranił (bug-report 2026-06-07, fd06cd86). Telefon sam
|
// stronę (audit 2026-06-11). Backend resolvuje z VPS → telefon dostaje URL bound do IP
|
||||||
// pobiera stronę (phone-IP-bound mp4) → natywne multi-quality, zero WebView/reklam.
|
// VPS → direct 403 / 10B placeholder. Telefon sam pobiera stronę (phone IP) → token
|
||||||
if (source.origin === 'tube:pornxpph') {
|
// bound do telefonu → gra direct, zero VPS. pornxp.ph analogicznie (backend WebView
|
||||||
setResolving(true);
|
// fallback czarno-ekranił, fd06cd86). Pusto → spadnij na backend resolve niżej.
|
||||||
try {
|
|
||||||
const links = await resolvePornxpPage(source.page_url);
|
|
||||||
if (links.length > 0) {
|
|
||||||
markStarted();
|
|
||||||
const pick = pickAuto(links);
|
|
||||||
if (pick) await openAsVideo(pick, source.page_url);
|
|
||||||
else setQualityLinks(links);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
// pusto → spadnij na backend resolve (WebView) poniżej
|
|
||||||
} catch {
|
|
||||||
// ignore → backend fallback
|
|
||||||
} finally {
|
|
||||||
setResolving(false);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// sxyprn / eporner: CDN token IP-bound do tego KTO POBRAŁ STRONĘ (audit 2026-06-11).
|
|
||||||
// Backend resolvuje z VPS → telefon dostaje URL bound do IP VPS → direct daje 403 /
|
|
||||||
// 10B placeholder → fallback na proxy → CAŁE wideo przez Hetzner. Telefon sam pobiera
|
|
||||||
// stronę (phone IP) → token bound do telefonu → gra direct, zero VPS. [] → backend niżej.
|
|
||||||
const phoneResolver =
|
const phoneResolver =
|
||||||
source.origin === 'tube:sxyprncom'
|
source.origin === 'tube:sxyprncom'
|
||||||
? resolveSxyprnPage
|
? resolveSxyprnPage
|
||||||
|
|
@ -632,6 +611,8 @@ function PlaybackButton({
|
||||||
? resolveEpornerPage
|
? resolveEpornerPage
|
||||||
: source.origin === 'tube:fpoxxx'
|
: source.origin === 'tube:fpoxxx'
|
||||||
? resolveFpoxxxPage
|
? resolveFpoxxxPage
|
||||||
|
: source.origin === 'tube:pornxpph'
|
||||||
|
? resolvePornxpPage
|
||||||
: null;
|
: null;
|
||||||
if (phoneResolver) {
|
if (phoneResolver) {
|
||||||
setResolving(true);
|
setResolving(true);
|
||||||
|
|
|
||||||
|
|
@ -16,6 +16,7 @@ import {
|
||||||
View,
|
View,
|
||||||
} from 'react-native';
|
} from 'react-native';
|
||||||
import { SceneTile, sceneGridProps } from '../components/SceneTile';
|
import { SceneTile, sceneGridProps } from '../components/SceneTile';
|
||||||
|
import { sortNewFirst } from '../lib/newScenes';
|
||||||
import { useClient } from '../ClientContext';
|
import { useClient } from '../ClientContext';
|
||||||
import { usePreferences } from '../PreferencesContext';
|
import { usePreferences } from '../PreferencesContext';
|
||||||
import type { RootStackParamList } from '../navigation';
|
import type { RootStackParamList } from '../navigation';
|
||||||
|
|
@ -115,21 +116,10 @@ export function StudioScenesScreen() {
|
||||||
}),
|
}),
|
||||||
});
|
});
|
||||||
|
|
||||||
const sortedItems = React.useMemo<SceneOut[]>(() => {
|
const sortedItems = React.useMemo<SceneOut[]>(
|
||||||
const items = data?.items ?? [];
|
() => sortNewFirst(data?.items ?? [], seenSince),
|
||||||
if (!seenSince) return items;
|
[data?.items, seenSince],
|
||||||
const newOnes: SceneOut[] = [];
|
);
|
||||||
const rest: SceneOut[] = [];
|
|
||||||
for (const s of items) {
|
|
||||||
// NEW-first pomija backfill (masowy import katalogu), spójne z badge + licznikiem +N.
|
|
||||||
if (s.created_at && s.created_at > seenSince && !s.backfill) {
|
|
||||||
newOnes.push(s);
|
|
||||||
} else {
|
|
||||||
rest.push(s);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return [...newOnes, ...rest];
|
|
||||||
}, [data?.items, seenSince]);
|
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<View style={styles.container}>
|
<View style={styles.container}>
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue