fix(favorites): exclude bulk-backfill scenes from "+N new" (tube fake dates)

Browse scrapers backfilling old catalogs stamp the tube's import/post date as
release_date, so old content (e.g. 83 MissaX classics via perverzija, ~3600/3 days
across eporner/youporn/etc.) fake-ranked as newest and flooded the favorites "+N".
NULLing the dates was a non-starter — the stub filter would hide 251k performer-less
scenes. Instead: a Scene.backfill flag marks bulk catalog imports; they stay visible but
never count as "new".

- scenes.backfill column (+ index, migration 0026); resolve_scene/_process_scene thread it.
- deep_crawl tags scenes from pages beyond the "latest" threshold (>2) as backfill;
  latest pages + TPDB/StashDB delta stay genuine. Cursor reset re-sweeps page 1 so real
  new content is always caught fresh.
- favorites +N (performers + studios) excludes backfill within the top-200 window.
- SceneOut exposes `backfill`; mobile NEW badge + NEW-first re-sort skip it (badge==count).
- Retroactive: tagged 439k existing scenes in bulk (>10 non-canonical / studio / day)
  clusters. Device check: favorite-studios +N 11626 (naive) -> 236.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

alembic/versions/20260702_0026_scene_backfill_flag.py

@@ -0,0 +1,39 @@
+"""scene backfill flag: exclude bulk catalog imports from "new"
+
+Revision ID: 0026_scene_backfill_flag
+Revises: 0025_source_ranking
+Create Date: 2026-07-02
+
+Tube'y podają datę importu jako release_date, więc masowy backfill starego katalogu
+(deep-crawl głębokie strony) udawał świeżość i zawyżał licznik "+N nowych" w ulubionych.
+`scenes.backfill` oznacza takie sceny — pozostają widoczne, ale nie liczą się jako nowe.
+Ustawiane przy tworzeniu (deep_crawl page > próg); świeży latest-crawl + delta TPDB/
+StashDB → False. Patrz app/scheduler/deep_crawl.py, app/resolve/scene_resolver.py.
+"""
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "0026_scene_backfill_flag"
+down_revision: str | None = "0025_source_ranking"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "scenes",
+        sa.Column(
+            "backfill",
+            sa.Boolean(),
+            nullable=False,
+            server_default=sa.text("false"),
+        ),
+    )
+    op.create_index("ix_scenes_backfill", "scenes", ["backfill"])
+
+
+def downgrade() -> None:
+    op.drop_index("ix_scenes_backfill", table_name="scenes")
+    op.drop_column("scenes", "backfill")

app/api/favorites.py

@@ -187,6 +187,7 @@ def list_favorites(
             select(
                 ScenePerformer.performer_id.label("gid"),
                 Scene.created_at.label("created_at"),
+                Scene.backfill.label("backfill"),
                 rn,
             )
             .join(Scene, Scene.id == ScenePerformer.scene_id)
@@ -194,11 +195,14 @@ def list_favorites(
             .where(*clauses)
             .subquery()
         )
-        for gid, created_at in session.execute(
-            select(inner.c.gid, inner.c.created_at).where(inner.c.rn <= _FAVORITES_PAGE_CAP)
+        for gid, created_at, backfill in session.execute(
+            select(inner.c.gid, inner.c.created_at, inner.c.backfill).where(
+                inner.c.rn <= _FAVORITES_PAGE_CAP
+            )
         ):
             ls = last_seen_by_perf.get(gid)
-            if created_at is not None and ls is not None and created_at > ls:
+            # backfill (masowy import katalogu) NIE liczy się jako nowość — patrz Scene.backfill.
+            if not backfill and created_at is not None and ls is not None and created_at > ls:
                 new_counts[gid] = new_counts.get(gid, 0) + 1
 
     items: list[FavoriteOut] = []
@@ -334,17 +338,21 @@ def list_favorite_studios(
             select(
                 Scene.studio_id.label("gid"),
                 Scene.created_at.label("created_at"),
+                Scene.backfill.label("backfill"),
                 rn,
             )
             .where(Scene.studio_id.in_(studio_ids))
             .where(*clauses)
             .subquery()
         )
-        for gid, created_at in session.execute(
-            select(inner.c.gid, inner.c.created_at).where(inner.c.rn <= _FAVORITES_PAGE_CAP)
+        for gid, created_at, backfill in session.execute(
+            select(inner.c.gid, inner.c.created_at, inner.c.backfill).where(
+                inner.c.rn <= _FAVORITES_PAGE_CAP
+            )
         ):
             ls = last_seen_by_studio.get(gid)
-            if created_at is not None and ls is not None and created_at > ls:
+            # backfill (masowy import katalogu) NIE liczy się jako nowość — patrz Scene.backfill.
+            if not backfill and created_at is not None and ls is not None and created_at > ls:
                 new_counts[gid] = new_counts.get(gid, 0) + 1
 
     items: list[FavoriteStudioOut] = []

app/api/scenes.py

@@ -693,6 +693,7 @@ def _build_scenes_out_batch(
                 external_refs=refs_by_scene.get(scene.id, []),
                 playback_sources=pb_by_scene.get(scene.id, []),
                 created_at=scene.created_at,
+                backfill=scene.backfill,
                 last_played_at=progress.last_played_at if progress else None,
                 finished=progress.finished if progress else False,
                 position_sec=progress.position_sec if progress else 0,
@@ -828,6 +829,7 @@ def _build_scene_out(session: Session, scene: Scene, *, device_id: str = LEGACY_
         external_refs=refs_out,
         playback_sources=playback_out,
         created_at=scene.created_at,
+        backfill=scene.backfill,
         last_played_at=progress.last_played_at if progress else None,
         finished=progress.finished if progress else False,
         position_sec=progress.position_sec if progress else 0,

app/api/schemas.py

@@ -70,6 +70,9 @@ class SceneOut(BaseModel):
     # "NEW" na karcie scen w PerformerScenesScreen / StudioScenesScreen — gdy
     # `created_at > last_seen_at` (favorite) → badge.
     created_at: datetime | None = None
+    # True = scena z masowego backfillu katalogu (deep-crawl). Mobile NIE pokazuje na niej
+    # NEW badge nawet gdy created_at > last_seen — to stara treść, nie świeży release.
+    backfill: bool = False
     # Watched indicator (z `scene_play_progress`): mobile dim'uje kafelek gdy
     # `finished=True`, pokazuje progress bar gdy `position_sec > 0`.
     last_played_at: datetime | None = None

app/ingest.py

@@ -267,7 +267,13 @@ def ingest_from_connector(
     return counters
 
 
-def _process_scene(*, source_id: uuid.UUID, raw_scene: RawScene, counters: dict[str, int]) -> None:
+def _process_scene(
+    *,
+    source_id: uuid.UUID,
+    raw_scene: RawScene,
+    counters: dict[str, int],
+    backfill: bool = False,
+) -> None:
     payload = raw_scene.raw or raw_scene.model_dump(mode="json")
     if _has_nul(payload):
         # Strip NUL z payloadu (→ external_records.raw JSONB) ORAZ ze structured fields
@@ -304,7 +310,7 @@ def _process_scene(*, source_id: uuid.UUID, raw_scene: RawScene, counters: dict[
             counters["skipped"] += 1
             return
 
-        result = resolve_scene(session, norm=norm, source_id=source_id)
+        result = resolve_scene(session, norm=norm, source_id=source_id, backfill=backfill)
 
         if result.was_created:
             counters["new"] += 1

app/models/scene.py

@@ -3,6 +3,7 @@ import uuid
 from datetime import date, datetime
 
 from sqlalchemy import (
+    Boolean,
     Date,
     DateTime,
     Enum,
@@ -40,6 +41,14 @@ class Scene(UUIDPKMixin, TimestampMixin, Base):
     description: Mapped[str | None] = mapped_column(Text)
     code: Mapped[str | None] = mapped_column(String(128), index=True)
     director: Mapped[str | None] = mapped_column(String(256))
+    # True = scena wpadła masowym backfillem katalogu (deep-crawl, głębokie strony), NIE
+    # świeży ingest. Tube podają datę importu jako release_date, więc stary backfill
+    # udaje "nowość" (licznik +N w ulubionych). Ten flag pozwala go wykluczyć z "nowych"
+    # bez chowania sceny. Ustawiany przy TWORZENIU (deep_crawl page > próg); świeży
+    # latest-crawl + delta TPDB/StashDB → False. Patrz app/scheduler/deep_crawl.py.
+    backfill: Mapped[bool] = mapped_column(
+        Boolean, nullable=False, server_default="false", index=True
+    )
 
 
 class SceneExternalRef(Base):

app/resolve/scene_resolver.py

@@ -69,7 +69,12 @@ def resolve_scene(
     *,
     norm: NormalizedScene,
     source_id: uuid.UUID,
+    backfill: bool = False,
 ) -> SceneResolveResult:
+    """backfill=True → nowo utworzona scena to masowy import katalogu (deep-crawl głębokie
+    strony), NIE świeży release. Flag ląduje na Scene.backfill i wyklucza scenę z "nowych"
+    (licznik +N w ulubionych). Dotyczy tylko TWORZENIA — match do istniejącej sceny nie
+    zmienia jej flagi (istniejąca mogła być świeża)."""
     studio = resolve_studio(session, norm=norm.studio, source_id=source_id) if norm.studio else None
     studio_id = studio.id if studio else None
 
@@ -138,7 +143,7 @@ def resolve_scene(
                 scene_match.title_normalized, norm.title_normalized
             )
             if sp_strength >= 1.0:
-                new_scene = _create_canonical(session, norm=norm, studio_id=studio_id)
+                new_scene = _create_canonical(session, norm=norm, studio_id=studio_id, backfill=backfill)
                 _attach_external_ref(session, scene_id=new_scene.id, source_id=source_id, norm=norm)
                 _sync_attached_entities(session, scene=new_scene, norm=norm, source_id=source_id)
                 return SceneResolveResult(
@@ -155,7 +160,7 @@ def resolve_scene(
                 penalised_score = raw_phash_score * max(dur_prox, 0.1)
                 if 0.0 < sp_strength < 1.0:
                     penalised_score = min(penalised_score, 1.0 - sp_strength)
-                new_scene = _create_canonical(session, norm=norm, studio_id=studio_id)
+                new_scene = _create_canonical(session, norm=norm, studio_id=studio_id, backfill=backfill)
                 _attach_external_ref(session, scene_id=new_scene.id, source_id=source_id, norm=norm)
                 _sync_attached_entities(session, scene=new_scene, norm=norm, source_id=source_id)
                 session.add(
@@ -189,7 +194,7 @@ def resolve_scene(
             # ale auto-merge zablokowane: tworzymy nową scenę + pending review.
             if 0.0 < sp_strength < 1.0:
                 penalised_score = min(raw_phash_score, 1.0 - sp_strength)
-                new_scene = _create_canonical(session, norm=norm, studio_id=studio_id)
+                new_scene = _create_canonical(session, norm=norm, studio_id=studio_id, backfill=backfill)
                 _attach_external_ref(session, scene_id=new_scene.id, source_id=source_id, norm=norm)
                 _sync_attached_entities(session, scene=new_scene, norm=norm, source_id=source_id)
                 session.add(
@@ -342,7 +347,7 @@ def resolve_scene(
             )
 
         if decision == "review":
-            new_scene = _create_canonical(session, norm=norm, studio_id=studio_id)
+            new_scene = _create_canonical(session, norm=norm, studio_id=studio_id, backfill=backfill)
             _attach_external_ref(session, scene_id=new_scene.id, source_id=source_id, norm=norm)
             _sync_performers(session, scene_id=new_scene.id, resolved=resolved_performers)
             _sync_tags(session, scene_id=new_scene.id, norm=norm, source_id=source_id)
@@ -369,7 +374,7 @@ def resolve_scene(
             )
 
     # Brak żadnego sensownego dopasowania → nowa kanoniczna
-    new_scene = _create_canonical(session, norm=norm, studio_id=studio_id)
+    new_scene = _create_canonical(session, norm=norm, studio_id=studio_id, backfill=backfill)
     _attach_external_ref(session, scene_id=new_scene.id, source_id=source_id, norm=norm)
     _sync_performers(session, scene_id=new_scene.id, resolved=resolved_performers)
     _sync_tags(session, scene_id=new_scene.id, norm=norm, source_id=source_id)
@@ -403,7 +408,11 @@ def _cap(s: str | None, n: int) -> str | None:
 
 
 def _create_canonical(
-    session: Session, *, norm: NormalizedScene, studio_id: uuid.UUID | None
+    session: Session,
+    *,
+    norm: NormalizedScene,
+    studio_id: uuid.UUID | None,
+    backfill: bool = False,
 ) -> Scene:
     scene = Scene(
         title=norm.title,
@@ -415,6 +424,7 @@ def _create_canonical(
         description=norm.description,
         code=_cap(norm.code, 128),
         director=_cap(norm.director, 256),
+        backfill=backfill,
     )
     session.add(scene)
     session.flush()

app/scheduler/deep_crawl.py

@@ -46,6 +46,13 @@ _PAGE_CAP: dict[str, int] = {
 # wracamy czysto — następny run kontynuuje. Margines 600s na dokończenie strony w toku.
 _RUN_BUDGET_SEC = 3000
 
+# Strony <= tego progu = "latest" (świeże posty tube'a) → nowe sceny stąd to genuine
+# nowości. Głębsze strony = backfill starego katalogu → nowe sceny stąd dostają
+# Scene.backfill=True i NIE liczą się jako "nowe" w ulubionych (tube podaje datę importu
+# jako release_date, więc stary backfill inaczej udaje świeżość). Reset kursora re-sweepuje
+# od strony 1, więc genuinnie nowe zawsze łapią się na stronach 1-2.
+_LATEST_PAGE_THRESHOLD = 2
+
 
 def _state_path() -> Path:
     return Path(getattr(get_settings(), "deepcrawl_state_path", None) or _DEFAULT_STATE)
@@ -142,10 +149,16 @@ def run_deep_crawl(*, pages_per_run: int = 60, sitetags: list[str] | None = None
                 exhausted = True
                 last_done = page
                 break
+            page_is_backfill = page > _LATEST_PAGE_THRESHOLD
             for raw in scenes:
                 counters["seen"] += 1
                 try:
-                    _process_scene(source_id=source_id, raw_scene=raw, counters=counters)
+                    _process_scene(
+                        source_id=source_id,
+                        raw_scene=raw,
+                        counters=counters,
+                        backfill=page_is_backfill,
+                    )
                 except Exception:
                     counters["errors"] += 1
             last_done = page

mobile/src/changelog.ts

@@ -16,6 +16,13 @@ export type ChangelogEntry = {
 };
 
 export const CHANGELOG: ChangelogEntry[] = [
+  {
+    id: '2026-07-02',
+    date: 'July 2026',
+    items: [
+      'Favorites "+N new" now counts genuinely new scenes only — bulk back-catalog imports (old content a site re-dated to today) no longer inflate the badge or show a NEW tag.',
+    ],
+  },
   {
     id: '2026-07-01b',
     date: 'July 2026',

mobile/src/components/SceneTile.tsx

@@ -80,7 +80,9 @@ function SceneTileBase({ scene, secondLine = 'studio', seenSince, onLongPress }:
   };
 
   const dim = scene.finished === true;
-  const isNew = !!(seenSince && scene.created_at && scene.created_at > seenSince);
+  // NEW = dodane od ostatniej wizyty, ale NIE backfill (masowy import starego katalogu —
+  // tube podaje datę importu jako release_date, więc udawałby świeżość). Spójne z licznikiem +N.
+  const isNew = !!(seenSince && scene.created_at && scene.created_at > seenSince && !scene.backfill);
   const dur = scene.duration_sec;
   const durLabel =
     dur && dur > 0

mobile/src/screens/PerformerScenesScreen.tsx

@@ -177,7 +177,8 @@ export function PerformerScenesScreen() {
     const newOnes: SceneOut[] = [];
     const rest: SceneOut[] = [];
     for (const s of items) {
-      if (s.created_at && s.created_at > seenSince) {
+      // NEW-first pomija backfill (masowy import katalogu) — spójne z badge + licznikiem +N.
+      if (s.created_at && s.created_at > seenSince && !s.backfill) {
         newOnes.push(s);
       } else {
         rest.push(s);

mobile/src/screens/StudioScenesScreen.tsx

@@ -122,7 +122,8 @@ export function StudioScenesScreen() {
     const newOnes: SceneOut[] = [];
     const rest: SceneOut[] = [];
     for (const s of items) {
-      if (s.created_at && s.created_at > seenSince) {
+      // NEW-first pomija backfill (masowy import katalogu) — spójne z badge + licznikiem +N.
+      if (s.created_at && s.created_at > seenSince && !s.backfill) {
         newOnes.push(s);
       } else {
         rest.push(s);

mobile/src/types.ts

@@ -173,6 +173,9 @@ export interface SceneOut {
   // Kiedy scena trafiła do bazy (ingest). Używane do oznaczenia "NEW" — gdy
   // `created_at > favoriteSeenSince` (param przekazany z FavoritesScreen).
   created_at?: string | null;
+  // True = scena z masowego backfillu katalogu (deep-crawl) — stara treść, nie świeży
+  // release. NIE pokazujemy na niej NEW badge nawet gdy created_at > seenSince.
+  backfill?: boolean;
   // Watched indicator + favorite state. Backend dolicza z scene_play_progress + favorite_scenes.
   last_played_at?: string | null;
   finished?: boolean;