diff --git a/alembic/versions/20260702_0026_scene_backfill_flag.py b/alembic/versions/20260702_0026_scene_backfill_flag.py new file mode 100644 index 0000000..5613761 --- /dev/null +++ b/alembic/versions/20260702_0026_scene_backfill_flag.py @@ -0,0 +1,39 @@ +"""scene backfill flag: exclude bulk catalog imports from "new" + +Revision ID: 0026_scene_backfill_flag +Revises: 0025_source_ranking +Create Date: 2026-07-02 + +Tube'y podają datę importu jako release_date, więc masowy backfill starego katalogu +(deep-crawl głębokie strony) udawał świeżość i zawyżał licznik "+N nowych" w ulubionych. +`scenes.backfill` oznacza takie sceny — pozostają widoczne, ale nie liczą się jako nowe. +Ustawiane przy tworzeniu (deep_crawl page > próg); świeży latest-crawl + delta TPDB/ +StashDB → False. Patrz app/scheduler/deep_crawl.py, app/resolve/scene_resolver.py. +""" +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +revision: str = "0026_scene_backfill_flag" +down_revision: str | None = "0025_source_ranking" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + op.add_column( + "scenes", + sa.Column( + "backfill", + sa.Boolean(), + nullable=False, + server_default=sa.text("false"), + ), + ) + op.create_index("ix_scenes_backfill", "scenes", ["backfill"]) + + +def downgrade() -> None: + op.drop_index("ix_scenes_backfill", table_name="scenes") + op.drop_column("scenes", "backfill") diff --git a/app/api/favorites.py b/app/api/favorites.py index cd7ab72..4fa03bd 100644 --- a/app/api/favorites.py +++ b/app/api/favorites.py @@ -187,6 +187,7 @@ def list_favorites( select( ScenePerformer.performer_id.label("gid"), Scene.created_at.label("created_at"), + Scene.backfill.label("backfill"), rn, ) .join(Scene, Scene.id == ScenePerformer.scene_id) @@ -194,11 +195,14 @@ def list_favorites( .where(*clauses) .subquery() ) - for gid, created_at in session.execute( - select(inner.c.gid, inner.c.created_at).where(inner.c.rn <= _FAVORITES_PAGE_CAP) + for gid, created_at, backfill in session.execute( + select(inner.c.gid, inner.c.created_at, inner.c.backfill).where( + inner.c.rn <= _FAVORITES_PAGE_CAP + ) ): ls = last_seen_by_perf.get(gid) - if created_at is not None and ls is not None and created_at > ls: + # backfill (masowy import katalogu) NIE liczy się jako nowość — patrz Scene.backfill. + if not backfill and created_at is not None and ls is not None and created_at > ls: new_counts[gid] = new_counts.get(gid, 0) + 1 items: list[FavoriteOut] = [] @@ -334,17 +338,21 @@ def list_favorite_studios( select( Scene.studio_id.label("gid"), Scene.created_at.label("created_at"), + Scene.backfill.label("backfill"), rn, ) .where(Scene.studio_id.in_(studio_ids)) .where(*clauses) .subquery() ) - for gid, created_at in session.execute( - select(inner.c.gid, inner.c.created_at).where(inner.c.rn <= _FAVORITES_PAGE_CAP) + for gid, created_at, backfill in session.execute( + select(inner.c.gid, inner.c.created_at, inner.c.backfill).where( + inner.c.rn <= _FAVORITES_PAGE_CAP + ) ): ls = last_seen_by_studio.get(gid) - if created_at is not None and ls is not None and created_at > ls: + # backfill (masowy import katalogu) NIE liczy się jako nowość — patrz Scene.backfill. + if not backfill and created_at is not None and ls is not None and created_at > ls: new_counts[gid] = new_counts.get(gid, 0) + 1 items: list[FavoriteStudioOut] = [] diff --git a/app/api/scenes.py b/app/api/scenes.py index 3d0b915..844c956 100644 --- a/app/api/scenes.py +++ b/app/api/scenes.py @@ -693,6 +693,7 @@ def _build_scenes_out_batch( external_refs=refs_by_scene.get(scene.id, []), playback_sources=pb_by_scene.get(scene.id, []), created_at=scene.created_at, + backfill=scene.backfill, last_played_at=progress.last_played_at if progress else None, finished=progress.finished if progress else False, position_sec=progress.position_sec if progress else 0, @@ -828,6 +829,7 @@ def _build_scene_out(session: Session, scene: Scene, *, device_id: str = LEGACY_ external_refs=refs_out, playback_sources=playback_out, created_at=scene.created_at, + backfill=scene.backfill, last_played_at=progress.last_played_at if progress else None, finished=progress.finished if progress else False, position_sec=progress.position_sec if progress else 0, diff --git a/app/api/schemas.py b/app/api/schemas.py index 2fdaca2..e8da3ab 100644 --- a/app/api/schemas.py +++ b/app/api/schemas.py @@ -70,6 +70,9 @@ class SceneOut(BaseModel): # "NEW" na karcie scen w PerformerScenesScreen / StudioScenesScreen — gdy # `created_at > last_seen_at` (favorite) → badge. created_at: datetime | None = None + # True = scena z masowego backfillu katalogu (deep-crawl). Mobile NIE pokazuje na niej + # NEW badge nawet gdy created_at > last_seen — to stara treść, nie świeży release. + backfill: bool = False # Watched indicator (z `scene_play_progress`): mobile dim'uje kafelek gdy # `finished=True`, pokazuje progress bar gdy `position_sec > 0`. last_played_at: datetime | None = None diff --git a/app/ingest.py b/app/ingest.py index af4f68c..3fed4dc 100644 --- a/app/ingest.py +++ b/app/ingest.py @@ -267,7 +267,13 @@ def ingest_from_connector( return counters -def _process_scene(*, source_id: uuid.UUID, raw_scene: RawScene, counters: dict[str, int]) -> None: +def _process_scene( + *, + source_id: uuid.UUID, + raw_scene: RawScene, + counters: dict[str, int], + backfill: bool = False, +) -> None: payload = raw_scene.raw or raw_scene.model_dump(mode="json") if _has_nul(payload): # Strip NUL z payloadu (→ external_records.raw JSONB) ORAZ ze structured fields @@ -304,7 +310,7 @@ def _process_scene(*, source_id: uuid.UUID, raw_scene: RawScene, counters: dict[ counters["skipped"] += 1 return - result = resolve_scene(session, norm=norm, source_id=source_id) + result = resolve_scene(session, norm=norm, source_id=source_id, backfill=backfill) if result.was_created: counters["new"] += 1 diff --git a/app/models/scene.py b/app/models/scene.py index 5fc5170..f09d435 100644 --- a/app/models/scene.py +++ b/app/models/scene.py @@ -3,6 +3,7 @@ import uuid from datetime import date, datetime from sqlalchemy import ( + Boolean, Date, DateTime, Enum, @@ -40,6 +41,14 @@ class Scene(UUIDPKMixin, TimestampMixin, Base): description: Mapped[str | None] = mapped_column(Text) code: Mapped[str | None] = mapped_column(String(128), index=True) director: Mapped[str | None] = mapped_column(String(256)) + # True = scena wpadła masowym backfillem katalogu (deep-crawl, głębokie strony), NIE + # świeży ingest. Tube podają datę importu jako release_date, więc stary backfill + # udaje "nowość" (licznik +N w ulubionych). Ten flag pozwala go wykluczyć z "nowych" + # bez chowania sceny. Ustawiany przy TWORZENIU (deep_crawl page > próg); świeży + # latest-crawl + delta TPDB/StashDB → False. Patrz app/scheduler/deep_crawl.py. + backfill: Mapped[bool] = mapped_column( + Boolean, nullable=False, server_default="false", index=True + ) class SceneExternalRef(Base): diff --git a/app/resolve/scene_resolver.py b/app/resolve/scene_resolver.py index 282eef3..ac11a0d 100644 --- a/app/resolve/scene_resolver.py +++ b/app/resolve/scene_resolver.py @@ -69,7 +69,12 @@ def resolve_scene( *, norm: NormalizedScene, source_id: uuid.UUID, + backfill: bool = False, ) -> SceneResolveResult: + """backfill=True → nowo utworzona scena to masowy import katalogu (deep-crawl głębokie + strony), NIE świeży release. Flag ląduje na Scene.backfill i wyklucza scenę z "nowych" + (licznik +N w ulubionych). Dotyczy tylko TWORZENIA — match do istniejącej sceny nie + zmienia jej flagi (istniejąca mogła być świeża).""" studio = resolve_studio(session, norm=norm.studio, source_id=source_id) if norm.studio else None studio_id = studio.id if studio else None @@ -138,7 +143,7 @@ def resolve_scene( scene_match.title_normalized, norm.title_normalized ) if sp_strength >= 1.0: - new_scene = _create_canonical(session, norm=norm, studio_id=studio_id) + new_scene = _create_canonical(session, norm=norm, studio_id=studio_id, backfill=backfill) _attach_external_ref(session, scene_id=new_scene.id, source_id=source_id, norm=norm) _sync_attached_entities(session, scene=new_scene, norm=norm, source_id=source_id) return SceneResolveResult( @@ -155,7 +160,7 @@ def resolve_scene( penalised_score = raw_phash_score * max(dur_prox, 0.1) if 0.0 < sp_strength < 1.0: penalised_score = min(penalised_score, 1.0 - sp_strength) - new_scene = _create_canonical(session, norm=norm, studio_id=studio_id) + new_scene = _create_canonical(session, norm=norm, studio_id=studio_id, backfill=backfill) _attach_external_ref(session, scene_id=new_scene.id, source_id=source_id, norm=norm) _sync_attached_entities(session, scene=new_scene, norm=norm, source_id=source_id) session.add( @@ -189,7 +194,7 @@ def resolve_scene( # ale auto-merge zablokowane: tworzymy nową scenę + pending review. if 0.0 < sp_strength < 1.0: penalised_score = min(raw_phash_score, 1.0 - sp_strength) - new_scene = _create_canonical(session, norm=norm, studio_id=studio_id) + new_scene = _create_canonical(session, norm=norm, studio_id=studio_id, backfill=backfill) _attach_external_ref(session, scene_id=new_scene.id, source_id=source_id, norm=norm) _sync_attached_entities(session, scene=new_scene, norm=norm, source_id=source_id) session.add( @@ -342,7 +347,7 @@ def resolve_scene( ) if decision == "review": - new_scene = _create_canonical(session, norm=norm, studio_id=studio_id) + new_scene = _create_canonical(session, norm=norm, studio_id=studio_id, backfill=backfill) _attach_external_ref(session, scene_id=new_scene.id, source_id=source_id, norm=norm) _sync_performers(session, scene_id=new_scene.id, resolved=resolved_performers) _sync_tags(session, scene_id=new_scene.id, norm=norm, source_id=source_id) @@ -369,7 +374,7 @@ def resolve_scene( ) # Brak żadnego sensownego dopasowania → nowa kanoniczna - new_scene = _create_canonical(session, norm=norm, studio_id=studio_id) + new_scene = _create_canonical(session, norm=norm, studio_id=studio_id, backfill=backfill) _attach_external_ref(session, scene_id=new_scene.id, source_id=source_id, norm=norm) _sync_performers(session, scene_id=new_scene.id, resolved=resolved_performers) _sync_tags(session, scene_id=new_scene.id, norm=norm, source_id=source_id) @@ -403,7 +408,11 @@ def _cap(s: str | None, n: int) -> str | None: def _create_canonical( - session: Session, *, norm: NormalizedScene, studio_id: uuid.UUID | None + session: Session, + *, + norm: NormalizedScene, + studio_id: uuid.UUID | None, + backfill: bool = False, ) -> Scene: scene = Scene( title=norm.title, @@ -415,6 +424,7 @@ def _create_canonical( description=norm.description, code=_cap(norm.code, 128), director=_cap(norm.director, 256), + backfill=backfill, ) session.add(scene) session.flush() diff --git a/app/scheduler/deep_crawl.py b/app/scheduler/deep_crawl.py index 2ebe9f0..15bbcf6 100644 --- a/app/scheduler/deep_crawl.py +++ b/app/scheduler/deep_crawl.py @@ -46,6 +46,13 @@ _PAGE_CAP: dict[str, int] = { # wracamy czysto — następny run kontynuuje. Margines 600s na dokończenie strony w toku. _RUN_BUDGET_SEC = 3000 +# Strony <= tego progu = "latest" (świeże posty tube'a) → nowe sceny stąd to genuine +# nowości. Głębsze strony = backfill starego katalogu → nowe sceny stąd dostają +# Scene.backfill=True i NIE liczą się jako "nowe" w ulubionych (tube podaje datę importu +# jako release_date, więc stary backfill inaczej udaje świeżość). Reset kursora re-sweepuje +# od strony 1, więc genuinnie nowe zawsze łapią się na stronach 1-2. +_LATEST_PAGE_THRESHOLD = 2 + def _state_path() -> Path: return Path(getattr(get_settings(), "deepcrawl_state_path", None) or _DEFAULT_STATE) @@ -142,10 +149,16 @@ def run_deep_crawl(*, pages_per_run: int = 60, sitetags: list[str] | None = None exhausted = True last_done = page break + page_is_backfill = page > _LATEST_PAGE_THRESHOLD for raw in scenes: counters["seen"] += 1 try: - _process_scene(source_id=source_id, raw_scene=raw, counters=counters) + _process_scene( + source_id=source_id, + raw_scene=raw, + counters=counters, + backfill=page_is_backfill, + ) except Exception: counters["errors"] += 1 last_done = page diff --git a/mobile/src/changelog.ts b/mobile/src/changelog.ts index 026eb95..3d324f8 100644 --- a/mobile/src/changelog.ts +++ b/mobile/src/changelog.ts @@ -16,6 +16,13 @@ export type ChangelogEntry = { }; export const CHANGELOG: ChangelogEntry[] = [ + { + id: '2026-07-02', + date: 'July 2026', + items: [ + 'Favorites "+N new" now counts genuinely new scenes only — bulk back-catalog imports (old content a site re-dated to today) no longer inflate the badge or show a NEW tag.', + ], + }, { id: '2026-07-01b', date: 'July 2026', diff --git a/mobile/src/components/SceneTile.tsx b/mobile/src/components/SceneTile.tsx index 5d4f380..c9149ae 100644 --- a/mobile/src/components/SceneTile.tsx +++ b/mobile/src/components/SceneTile.tsx @@ -80,7 +80,9 @@ function SceneTileBase({ scene, secondLine = 'studio', seenSince, onLongPress }: }; const dim = scene.finished === true; - const isNew = !!(seenSince && scene.created_at && scene.created_at > seenSince); + // NEW = dodane od ostatniej wizyty, ale NIE backfill (masowy import starego katalogu — + // tube podaje datę importu jako release_date, więc udawałby świeżość). Spójne z licznikiem +N. + const isNew = !!(seenSince && scene.created_at && scene.created_at > seenSince && !scene.backfill); const dur = scene.duration_sec; const durLabel = dur && dur > 0 diff --git a/mobile/src/screens/PerformerScenesScreen.tsx b/mobile/src/screens/PerformerScenesScreen.tsx index 6d8d521..078b315 100644 --- a/mobile/src/screens/PerformerScenesScreen.tsx +++ b/mobile/src/screens/PerformerScenesScreen.tsx @@ -177,7 +177,8 @@ export function PerformerScenesScreen() { const newOnes: SceneOut[] = []; const rest: SceneOut[] = []; for (const s of items) { - if (s.created_at && s.created_at > seenSince) { + // NEW-first pomija backfill (masowy import katalogu) — spójne z badge + licznikiem +N. + if (s.created_at && s.created_at > seenSince && !s.backfill) { newOnes.push(s); } else { rest.push(s); diff --git a/mobile/src/screens/StudioScenesScreen.tsx b/mobile/src/screens/StudioScenesScreen.tsx index 31cb352..03a693a 100644 --- a/mobile/src/screens/StudioScenesScreen.tsx +++ b/mobile/src/screens/StudioScenesScreen.tsx @@ -122,7 +122,8 @@ export function StudioScenesScreen() { const newOnes: SceneOut[] = []; const rest: SceneOut[] = []; for (const s of items) { - if (s.created_at && s.created_at > seenSince) { + // NEW-first pomija backfill (masowy import katalogu) — spójne z badge + licznikiem +N. + if (s.created_at && s.created_at > seenSince && !s.backfill) { newOnes.push(s); } else { rest.push(s); diff --git a/mobile/src/types.ts b/mobile/src/types.ts index 0542e6c..1774866 100644 --- a/mobile/src/types.ts +++ b/mobile/src/types.ts @@ -173,6 +173,9 @@ export interface SceneOut { // Kiedy scena trafiła do bazy (ingest). Używane do oznaczenia "NEW" — gdy // `created_at > favoriteSeenSince` (param przekazany z FavoritesScreen). created_at?: string | null; + // True = scena z masowego backfillu katalogu (deep-crawl) — stara treść, nie świeży + // release. NIE pokazujemy na niej NEW badge nawet gdy created_at > seenSince. + backfill?: boolean; // Watched indicator + favorite state. Backend dolicza z scene_play_progress + favorite_scenes. last_played_at?: string | null; finished?: boolean;