feat(dedup): kopie tej samej sceny z tubow o roznym formacie tytulu
Kazdy tube zapisuje tytul po swojemu ("[Milfy] Koda Monroe (...)", "Milfy 26 09 16 ...",
"... 4K"), a miniatury sa z innych klatek, wiec ani phash, ani title_duration_dedup
(identyczny tytul) ich nie lapaly. Jedna scena Milfy wisiala w katalogu 8 razy.
Nowy job porownuje rdzen tytulu w obrebie studia: bez nazwy studia, dat, znacznikow
jakosci i nazwisk obsady. Numery czesci zostaja ("Part 1" to nie "Part 2"). Straznicy:
dlugosc, data premiery, wspolna obsada, dluzszy rdzen gdy brak dlugosci. Kopie bez studia
dolaczaja tylko przy zgodnej co do 3 s dlugosci. Kategorie wpisane przez tube jako obsada
nie skracaja rdzenia. Cyklicznie co 12 h z oknem 30 dni.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
c59ad6eb73
commit
7fea132668
5 changed files with 402 additions and 0 deletions
|
|
@ -116,6 +116,12 @@ class Settings(BaseSettings):
|
||||||
sched_title_dedup_hours: int = Field(
|
sched_title_dedup_hours: int = Field(
|
||||||
default=12, validation_alias="GOON_SCHED_TITLE_DEDUP_HOURS"
|
default=12, validation_alias="GOON_SCHED_TITLE_DEDUP_HOURS"
|
||||||
)
|
)
|
||||||
|
# Dedup w obrebie studia po rdzeniu tytulu: ta sama scena z tubow, z ktorych kazdy pisze
|
||||||
|
# tytul po swojemu ("[Milfy] Koda Monroe (...)", "Milfy 26 09 16 ..."), 8 kopii jednej
|
||||||
|
# sceny w zgloszeniu bb1a357d. Okno 30 dni wstecz, bo kopie splywaja w ciagu dni. 0 = off.
|
||||||
|
sched_studio_title_dedup_hours: int = Field(
|
||||||
|
default=12, validation_alias="GOON_SCHED_STUDIO_TITLE_DEDUP_HOURS"
|
||||||
|
)
|
||||||
# Blacklista zdegenerowanych phashy, wartości dzielonych przez ≥10 scen (zaślepka,
|
# Blacklista zdegenerowanych phashy, wartości dzielonych przez ≥10 scen (zaślepka,
|
||||||
# czarna klatka, intro studia). Audyt 2026-07-27: 73 takie wartości przy 4375
|
# czarna klatka, intro studia). Audyt 2026-07-27: 73 takie wartości przy 4375
|
||||||
# scenach, rekordzistka przy 892 scenach o 892 różnych tytułach. Tube może zacząć
|
# scenach, rekordzistka przy 892 scenach o 892 różnych tytułach. Tube może zacząć
|
||||||
|
|
|
||||||
|
|
@ -399,6 +399,23 @@ def _job_title_duration_dedup() -> None:
|
||||||
log.exception("[scheduler] title-duration dedup failed")
|
log.exception("[scheduler] title-duration dedup failed")
|
||||||
|
|
||||||
|
|
||||||
|
def _job_studio_title_dedup() -> None:
|
||||||
|
"""Scal kopie tej samej sceny z tubow, ktore zapisuja tytul kazdy po swojemu (to samo
|
||||||
|
studio + ten sam rdzen tytulu, report bb1a357d). title_duration_dedup wymaga
|
||||||
|
identycznego tytulu, a phash nie lapie, bo miniatury sa z roznych klatek."""
|
||||||
|
log.info("[scheduler] studio-title dedup starting")
|
||||||
|
try:
|
||||||
|
from app.scheduler.studio_title_dedup import run_studio_title_dedup
|
||||||
|
|
||||||
|
_run_with_timeout(
|
||||||
|
lambda: run_studio_title_dedup(days=30, commit=True),
|
||||||
|
label="studio-title-dedup",
|
||||||
|
)
|
||||||
|
log.info("[scheduler] studio-title dedup done")
|
||||||
|
except Exception:
|
||||||
|
log.exception("[scheduler] studio-title dedup failed")
|
||||||
|
|
||||||
|
|
||||||
def _job_ingest_watchdog(
|
def _job_ingest_watchdog(
|
||||||
max_age_hours: int, search_max_age_hours: int, movie_max_age_hours: int
|
max_age_hours: int, search_max_age_hours: int, movie_max_age_hours: int
|
||||||
) -> None:
|
) -> None:
|
||||||
|
|
@ -723,6 +740,17 @@ def build_scheduler(cfg: dict[str, Any]) -> BlockingScheduler:
|
||||||
)
|
)
|
||||||
log.info("scheduler: title-duration dedup every %dh", cfg["title_dedup_hours"])
|
log.info("scheduler: title-duration dedup every %dh", cfg["title_dedup_hours"])
|
||||||
|
|
||||||
|
if cfg.get("studio_title_dedup_hours"):
|
||||||
|
sched.add_job(
|
||||||
|
_job_studio_title_dedup,
|
||||||
|
IntervalTrigger(hours=cfg["studio_title_dedup_hours"], start_date=INTERVAL_ANCHOR),
|
||||||
|
id="studio_title_dedup",
|
||||||
|
replace_existing=True,
|
||||||
|
max_instances=1,
|
||||||
|
coalesce=True,
|
||||||
|
)
|
||||||
|
log.info("scheduler: studio-title dedup every %dh", cfg["studio_title_dedup_hours"])
|
||||||
|
|
||||||
if cfg.get("ingest_watchdog_hours"):
|
if cfg.get("ingest_watchdog_hours"):
|
||||||
wd_max_age = cfg.get("ingest_watchdog_max_age_hours") or 48
|
wd_max_age = cfg.get("ingest_watchdog_max_age_hours") or 48
|
||||||
wd_search_max_age = cfg.get("ingest_watchdog_search_max_age_hours") or 168
|
wd_search_max_age = cfg.get("ingest_watchdog_search_max_age_hours") or 168
|
||||||
|
|
|
||||||
273
app/scheduler/studio_title_dedup.py
Normal file
273
app/scheduler/studio_title_dedup.py
Normal file
|
|
@ -0,0 +1,273 @@
|
||||||
|
"""Dedup w obrebie studia po rdzeniu tytulu.
|
||||||
|
|
||||||
|
Kazdy tube zapisuje tytul tej samej sceny po swojemu: "[Milfy] Koda Monroe (Voluptuous
|
||||||
|
Therapist Fucks Prisoner)", "Milfy 26 09 16 Koda Monroe Voluptuous Therapist Fucks Prisoner",
|
||||||
|
"Koda Monroe Voluptuous Therapist Fucks Prisoner 4K". Miniatury sa z innych klatek, wiec
|
||||||
|
phash sie nie pokrywa, a title_duration_dedup wymaga identycznego tytulu. Skutek: ta sama
|
||||||
|
scena Milfy wisiala w katalogu 8 razy (zgloszenie bb1a357d, 2026-09-18).
|
||||||
|
|
||||||
|
Rdzen tytulu = slowa bez nazwy studia, dat, znacznikow jakosci, lacznikow i nazwisk obsady
|
||||||
|
(suma obsady obu porownywanych scen, bo tube czesto nie ma obsady, a nazwisko ma w tytule).
|
||||||
|
Pojedyncze liczby ZOSTAJA: to numery czesci ("Whip 2", "Part 1"), ktore pierwsza wersja
|
||||||
|
reguly sklejala w jedno (suchy przebieg 2026-09-18).
|
||||||
|
|
||||||
|
Para jest ta sama scena, gdy to samo studio, rowne rdzenie i nic jej nie wyklucza:
|
||||||
|
- dlugosci (gdy obie znane) roznia sie o <= 5 s,
|
||||||
|
- daty premiery (gdy obie znane i zadna nie jest samym rokiem, czyli 1 stycznia) o <= 7 dni,
|
||||||
|
- bez obu dlugosci rdzen musi miec >= 4 slowa, bo zostaje sam tytul,
|
||||||
|
- obsady (gdy obie niepuste) maja wspolna osobe.
|
||||||
|
Grupy skladamy przez sume par, ale scalamy tylko sceny zgodne BEZPOSREDNIO z zostawiana,
|
||||||
|
zeby lancuch A~B~C nie wciagnal C, ktore z A sie nie zgadza (inna dlugosc przez scene bez
|
||||||
|
dlugosci w srodku).
|
||||||
|
|
||||||
|
Kopie bez studia (tube nie podal wytworni, 3 z 8 kopii Milfy) dopinamy do scen studyjnych
|
||||||
|
ostrzej: obie dlugosci znane i rowne co do 3 s, rdzen >= 4 slowa. Dwie sceny bez studia nie
|
||||||
|
maja wspolnej kotwicy, wiec tu sie nie lacza (identyczne tytuly zalatwia title_duration_dedup).
|
||||||
|
|
||||||
|
Okno czasowe, bo dupy splywaja z tubow w ciagu dni od premiery; cyklicznie w schedulerze.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import collections
|
||||||
|
import logging
|
||||||
|
import re
|
||||||
|
import unicodedata
|
||||||
|
import uuid as _u
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import date, datetime
|
||||||
|
|
||||||
|
from sqlalchemy import text
|
||||||
|
|
||||||
|
from app.db import session_scope
|
||||||
|
from app.resolve.scene_merge import merge_scenes
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
_QUALITY = {"4k", "8k", "uhd", "fhd", "hd", "sd", "2160p", "1440p", "1080p", "720p", "480p", "360p", "full", "mp4", "vr"}
|
||||||
|
_CONNECT = {"and", "x", "ft", "feat", "featuring", "starring", "with"}
|
||||||
|
# Data wpisana w tytul przez tube: "26 09 16", "(17/9/26)", "21-08-2026", "2026.09.16".
|
||||||
|
_DATE_RE = re.compile(r"(?<![0-9])[0-9]{1,4}[ ./_-][0-9]{1,2}[ ./_-][0-9]{1,4}(?![0-9])")
|
||||||
|
_MAX_DURATION_DIFF = 5
|
||||||
|
_MAX_RELEASE_DIFF_DAYS = 7
|
||||||
|
# Kubelek (studio + ostatnie slowo rdzenia) wiekszy od tego to generyczny tytul w studiu-
|
||||||
|
# zbieraczu ("Only Fans"), gdzie porownanie parami jest drogie i ryzykowne.
|
||||||
|
_MAX_BUCKET = 400
|
||||||
|
_JUNK_TAG_MIN_SCENES = 1000
|
||||||
|
_ORPHAN_MAX_DURATION_DIFF = 3
|
||||||
|
_ORPHAN_MIN_CORE = 4
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class Cand:
|
||||||
|
id: str
|
||||||
|
title: str
|
||||||
|
studio_id: str
|
||||||
|
studio_name: str
|
||||||
|
studio_slug: str
|
||||||
|
duration_sec: int | None
|
||||||
|
release_date: date | None
|
||||||
|
created_at: datetime
|
||||||
|
perfs: tuple[str, ...]
|
||||||
|
canon: bool
|
||||||
|
live: int
|
||||||
|
|
||||||
|
|
||||||
|
def title_words(title: str) -> list[str]:
|
||||||
|
t = unicodedata.normalize("NFKD", title or "").encode("ascii", "ignore").decode().lower()
|
||||||
|
t = _DATE_RE.sub(" ", t)
|
||||||
|
return re.sub(r"[^a-z0-9]+", " ", t).split()
|
||||||
|
|
||||||
|
|
||||||
|
def studio_tokens(name: str, slug: str) -> set[str]:
|
||||||
|
words = title_words(name)
|
||||||
|
# "[BrazzersExxtra]" i "Brazzers Exxtra" to to samo studio.
|
||||||
|
return set(words) | set(title_words(slug)) | {"".join(words)}
|
||||||
|
|
||||||
|
|
||||||
|
def title_core(title: str, st_tokens: set[str], perf_tokens: set[str]) -> tuple[str, ...]:
|
||||||
|
return tuple(
|
||||||
|
w for w in title_words(title)
|
||||||
|
if w not in _QUALITY and w not in _CONNECT and w not in st_tokens and w not in perf_tokens
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _year_only(d: date | None) -> bool:
|
||||||
|
# Czesc tubow podaje sam rok, zapisany jako 1 stycznia. Taka data niczego nie wyklucza.
|
||||||
|
return d is not None and d.month == 1 and d.day == 1
|
||||||
|
|
||||||
|
|
||||||
|
def same_scene(a: Cand, b: Cand, st_tokens: set[str], junk_names: frozenset[str] = frozenset()) -> bool:
|
||||||
|
if a.studio_id and b.studio_id and a.studio_id != b.studio_id:
|
||||||
|
return False
|
||||||
|
if not a.studio_id and not b.studio_id:
|
||||||
|
return False
|
||||||
|
orphan = not (a.studio_id and b.studio_id)
|
||||||
|
perf_tokens = {
|
||||||
|
w for n in a.perfs + b.perfs if n.lower() not in junk_names for w in title_words(n)
|
||||||
|
}
|
||||||
|
ca = title_core(a.title, st_tokens, perf_tokens)
|
||||||
|
if len(ca) < (_ORPHAN_MIN_CORE if orphan else 3) or ca != title_core(b.title, st_tokens, perf_tokens):
|
||||||
|
return False
|
||||||
|
if a.duration_sec and b.duration_sec:
|
||||||
|
if abs(a.duration_sec - b.duration_sec) > (_ORPHAN_MAX_DURATION_DIFF if orphan else _MAX_DURATION_DIFF):
|
||||||
|
return False
|
||||||
|
elif orphan or len(ca) < 4:
|
||||||
|
return False
|
||||||
|
ra, rb = a.release_date, b.release_date
|
||||||
|
if ra and rb and not _year_only(ra) and not _year_only(rb):
|
||||||
|
if abs((ra - rb).days) > _MAX_RELEASE_DIFF_DAYS:
|
||||||
|
return False
|
||||||
|
if a.perfs and b.perfs and not (set(a.perfs) & set(b.perfs)):
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def _keep_rank(c: Cand) -> tuple:
|
||||||
|
# Zostaje scena z kanonem (TPDB/StashDB), potem z najwieksza liczba zywych zrodel.
|
||||||
|
return (not c.canon, not c.studio_id, -c.live, not c.perfs, c.created_at)
|
||||||
|
|
||||||
|
|
||||||
|
def plan_merges(cands: list[Cand], junk_names: frozenset[str] = frozenset()) -> list[tuple[Cand, list[Cand]]]:
|
||||||
|
"""Zwraca [(zostaje, [do_scalenia...])]. Czysta funkcja, bez bazy."""
|
||||||
|
by_studio: dict[str, list[Cand]] = collections.defaultdict(list)
|
||||||
|
orphans: list[Cand] = []
|
||||||
|
for c in cands:
|
||||||
|
if c.studio_id:
|
||||||
|
by_studio[c.studio_id].append(c)
|
||||||
|
elif c.duration_sec:
|
||||||
|
orphans.append(c)
|
||||||
|
|
||||||
|
parent: dict[str, str] = {}
|
||||||
|
|
||||||
|
def find(x: str) -> str:
|
||||||
|
while parent.setdefault(x, x) != x:
|
||||||
|
parent[x] = parent[parent[x]]
|
||||||
|
x = parent[x]
|
||||||
|
return x
|
||||||
|
|
||||||
|
tokens_by_studio: dict[str, set[str]] = {}
|
||||||
|
by_id: dict[str, Cand] = {}
|
||||||
|
for sid, items in by_studio.items():
|
||||||
|
st = studio_tokens(items[0].studio_name, items[0].studio_slug)
|
||||||
|
tokens_by_studio[sid] = st
|
||||||
|
buckets: dict[str, list[Cand]] = collections.defaultdict(list)
|
||||||
|
for c in items:
|
||||||
|
core = title_core(c.title, st, set())
|
||||||
|
if core:
|
||||||
|
buckets[core[-1]].append(c)
|
||||||
|
for bucket in buckets.values():
|
||||||
|
if len(bucket) < 2 or len(bucket) > _MAX_BUCKET:
|
||||||
|
continue
|
||||||
|
for i, a in enumerate(bucket):
|
||||||
|
for b in bucket[i + 1:]:
|
||||||
|
if same_scene(a, b, st, junk_names):
|
||||||
|
parent[find(a.id)] = find(b.id)
|
||||||
|
by_id[a.id], by_id[b.id] = a, b
|
||||||
|
|
||||||
|
# Kopie bez studia: kandydaci z tym samym ostatnim slowem rdzenia i dlugoscia w tym samym
|
||||||
|
# albo sasiednim przedziale 10 s. Scena studyjna trafia pod oba warianty rdzenia, bo nazwa
|
||||||
|
# studia bywa na koncu tytulu ("Hotwife Hall Pass - TouchMyWife").
|
||||||
|
index: dict[tuple[str, int], list[Cand]] = collections.defaultdict(list)
|
||||||
|
for sid, items in by_studio.items():
|
||||||
|
st = tokens_by_studio[sid]
|
||||||
|
for c in items:
|
||||||
|
if not c.duration_sec:
|
||||||
|
continue
|
||||||
|
for core in {title_core(c.title, st, set()), title_core(c.title, set(), set())}:
|
||||||
|
if core:
|
||||||
|
index[(core[-1], c.duration_sec // 10)].append(c)
|
||||||
|
for x in orphans:
|
||||||
|
core = title_core(x.title, set(), set())
|
||||||
|
if not core:
|
||||||
|
continue
|
||||||
|
seen: set[str] = set()
|
||||||
|
slot = x.duration_sec // 10
|
||||||
|
for key in ((core[-1], slot - 1), (core[-1], slot), (core[-1], slot + 1)):
|
||||||
|
for y in index.get(key, ()):
|
||||||
|
if y.id in seen:
|
||||||
|
continue
|
||||||
|
seen.add(y.id)
|
||||||
|
if same_scene(x, y, tokens_by_studio[y.studio_id], junk_names):
|
||||||
|
parent[find(x.id)] = find(y.id)
|
||||||
|
by_id[x.id], by_id[y.id] = x, y
|
||||||
|
|
||||||
|
groups: dict[str, list[Cand]] = collections.defaultdict(list)
|
||||||
|
for cid, c in by_id.items():
|
||||||
|
groups[find(cid)].append(c)
|
||||||
|
|
||||||
|
plan: list[tuple[Cand, list[Cand]]] = []
|
||||||
|
for members in groups.values():
|
||||||
|
keep = min(members, key=_keep_rank)
|
||||||
|
st = tokens_by_studio[keep.studio_id]
|
||||||
|
drops = [
|
||||||
|
m for m in members
|
||||||
|
if m.id != keep.id
|
||||||
|
and (m.live or keep.live) # dwie niewidoczne w apce sceny zostawiamy bulk_dedup
|
||||||
|
and same_scene(keep, m, st, junk_names)
|
||||||
|
]
|
||||||
|
if drops:
|
||||||
|
plan.append((keep, drops))
|
||||||
|
return plan
|
||||||
|
|
||||||
|
|
||||||
|
def _load(days: int) -> tuple[list[Cand], frozenset[str]]:
|
||||||
|
sql = text("""
|
||||||
|
SELECT s.id::text, s.title, s.studio_id::text, st.name, st.slug, s.duration_sec,
|
||||||
|
s.release_date, s.created_at,
|
||||||
|
COALESCE((SELECT array_agg(p.canonical_name) FROM scene_performers sp
|
||||||
|
JOIN performers p ON p.id = sp.performer_id
|
||||||
|
WHERE sp.scene_id = s.id), '{}') AS perfs,
|
||||||
|
EXISTS (SELECT 1 FROM scene_external_refs r JOIN sources so ON so.id = r.source_id
|
||||||
|
WHERE r.scene_id = s.id AND so.kind IN ('tpdb', 'stashdb')) AS canon,
|
||||||
|
(SELECT count(*) FROM playback_sources p
|
||||||
|
WHERE p.scene_id = s.id AND p.dead_at IS NULL) AS live
|
||||||
|
FROM scenes s LEFT JOIN studios st ON st.id = s.studio_id
|
||||||
|
WHERE (s.created_at > now() - make_interval(days => :d) OR s.release_date > current_date - :d)
|
||||||
|
AND (s.studio_id IS NOT NULL OR s.duration_sec IS NOT NULL)
|
||||||
|
""")
|
||||||
|
with session_scope() as s:
|
||||||
|
rows = s.execute(sql, {"d": days}).all()
|
||||||
|
# Tube wpisuja czasem kategorie jako obsade ("Big Tits", "Deep Throat"). Takie
|
||||||
|
# "nazwiska" wycinalyby z rdzenia zwykle slowa tytulu, wiec ich nie uzywamy. Sama
|
||||||
|
# obecnosc wsrod tagow nie wystarcza: hashtagi z nazwiskami tez sa tagami ("Koda
|
||||||
|
# Monroe" 51 scen, "Angela White" 239), a kategorie maja ich tysiace ("Ball Licking"
|
||||||
|
# 11 tys., "Big Tits" 685 tys.). Pomiar 2026-09-18.
|
||||||
|
junk = frozenset(
|
||||||
|
n.lower()
|
||||||
|
for (n,) in s.execute(text("SELECT name FROM tags WHERE scene_count >= :m"), {"m": _JUNK_TAG_MIN_SCENES}).all()
|
||||||
|
if n
|
||||||
|
)
|
||||||
|
cands = [
|
||||||
|
Cand(
|
||||||
|
id=r[0], title=r[1] or "", studio_id=r[2] or "", studio_name=r[3] or "", studio_slug=r[4] or "",
|
||||||
|
duration_sec=r[5], release_date=r[6], created_at=r[7], perfs=tuple(r[8] or ()),
|
||||||
|
canon=bool(r[9]), live=int(r[10] or 0),
|
||||||
|
)
|
||||||
|
for r in rows
|
||||||
|
]
|
||||||
|
return cands, junk
|
||||||
|
|
||||||
|
|
||||||
|
def run_studio_title_dedup(*, days: int = 45, commit: bool = True) -> dict[str, int]:
|
||||||
|
"""Scal dupy w obrebie studia po rdzeniu tytulu. Zwraca {groups, merges, merged, errors}."""
|
||||||
|
cands, junk = _load(days)
|
||||||
|
plan = plan_merges(cands, junk)
|
||||||
|
merges = sum(len(d) for _, d in plan)
|
||||||
|
merged = errors = 0
|
||||||
|
if commit:
|
||||||
|
for keep, drops in plan:
|
||||||
|
for drop in drops:
|
||||||
|
try:
|
||||||
|
with session_scope() as s:
|
||||||
|
merge_scenes(
|
||||||
|
s, keep_id=_u.UUID(keep.id), drop_id=_u.UUID(drop.id),
|
||||||
|
resolved_by="studio_title_dedup",
|
||||||
|
)
|
||||||
|
merged += 1
|
||||||
|
except Exception as e: # pragma: no cover - obronnie
|
||||||
|
errors += 1
|
||||||
|
if errors <= 20:
|
||||||
|
log.warning("studio_title_dedup keep %s drop %s: %s", keep.id[:8], drop.id[:8], str(e)[:120])
|
||||||
|
result = {"candidates": len(cands), "groups": len(plan), "merges": merges, "merged": merged, "errors": errors}
|
||||||
|
log.info("studio_title_dedup: %s", result)
|
||||||
|
return result
|
||||||
|
|
@ -216,6 +216,8 @@ def run_forever() -> int:
|
||||||
"thumb_dedup_hours": getattr(settings, "sched_thumb_dedup_hours", 12) or None,
|
"thumb_dedup_hours": getattr(settings, "sched_thumb_dedup_hours", 12) or None,
|
||||||
# Title+duration dedup, missing-merge tube-dupy (reports 28fe8181/32df33b1).
|
# Title+duration dedup, missing-merge tube-dupy (reports 28fe8181/32df33b1).
|
||||||
"title_dedup_hours": getattr(settings, "sched_title_dedup_hours", 12) or None,
|
"title_dedup_hours": getattr(settings, "sched_title_dedup_hours", 12) or None,
|
||||||
|
# Studio + rdzen tytulu, kopie z tubow o roznym formacie tytulu (report bb1a357d).
|
||||||
|
"studio_title_dedup_hours": getattr(settings, "sched_studio_title_dedup_hours", 12) or None,
|
||||||
# Blacklista zdegenerowanych phashy, zaślepki dzielone przez setki scen (audyt
|
# Blacklista zdegenerowanych phashy, zaślepki dzielone przez setki scen (audyt
|
||||||
# 2026-07-27). Bez tego zaślepka jest zawsze najbliższym trafieniem w Path 3.
|
# 2026-07-27). Bez tego zaślepka jest zawsze najbliższym trafieniem w Path 3.
|
||||||
"phash_blacklist_hours": getattr(settings, "sched_phash_blacklist_hours", 24) or None,
|
"phash_blacklist_hours": getattr(settings, "sched_phash_blacklist_hours", 24) or None,
|
||||||
|
|
|
||||||
93
tests/test_studio_title_dedup.py
Normal file
93
tests/test_studio_title_dedup.py
Normal file
|
|
@ -0,0 +1,93 @@
|
||||||
|
from datetime import date, datetime
|
||||||
|
|
||||||
|
from app.scheduler.studio_title_dedup import Cand, plan_merges, same_scene, studio_tokens
|
||||||
|
|
||||||
|
_T0 = datetime(2026, 9, 16, 23, 0)
|
||||||
|
_MILFY = studio_tokens("Milfy", "milfy")
|
||||||
|
|
||||||
|
|
||||||
|
def _c(cid, title, dur=None, rel=None, perfs=(), canon=False, live=1, studio="milfy"):
|
||||||
|
return Cand(
|
||||||
|
id=cid, title=title, studio_id=studio, studio_name="Milfy", studio_slug="milfy",
|
||||||
|
duration_sec=dur, release_date=rel, created_at=_T0, perfs=tuple(perfs), canon=canon, live=live,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# Zgloszenie bb1a357d: ta sama scena z osmiu tubow, kazdy z innym formatem tytulu.
|
||||||
|
_MILFY_COPIES = [
|
||||||
|
_c("canon", "Voluptuous Therapist Fucks Prisoner", 2218, date(2026, 9, 16),
|
||||||
|
("Troy Francisco", "Koda Monroe"), canon=True),
|
||||||
|
_c("a", "MILFY – Koda Monroe – Voluptuous Therapist Fucks Prisoner", 2218, date(2026, 9, 16),
|
||||||
|
("Troy Francisco", "Koda Monroe"), live=4),
|
||||||
|
_c("b", "[Milfy] Koda Monroe (Voluptuous Therapist Fucks Prisoner)", None, date(2026, 9, 16)),
|
||||||
|
_c("c", "[MILFY] Koda Monroe – Voluptuous Therapist Fucks Prisoner", perfs=("Koda Monroe",)),
|
||||||
|
_c("d", "Koda Monroe - Voluptuous Therapist Fucks Prisoner (17/9/26)", 2218),
|
||||||
|
_c("e", "Koda Monroe Voluptuous Therapist Fucks Prisoner 4K", 2218),
|
||||||
|
_c("f", "Milfy 26 09 16 Koda Monroe Voluptuous Therapist Fucks Prisoner", None, date(2026, 9, 16)),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def test_milfy_copies_collapse_into_canonical_scene():
|
||||||
|
plan = plan_merges(_MILFY_COPIES)
|
||||||
|
assert len(plan) == 1
|
||||||
|
keep, drops = plan[0]
|
||||||
|
assert keep.id == "canon"
|
||||||
|
assert {d.id for d in drops} == {"a", "b", "c", "d", "e", "f"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_part_numbers_are_not_dates():
|
||||||
|
one = _c("1", "Ariel Swings Her Whip", 600, date(2026, 9, 16))
|
||||||
|
two = _c("2", "Ariel Swings Her Whip 2", 600, date(2026, 9, 16))
|
||||||
|
assert not same_scene(one, two, _MILFY)
|
||||||
|
p1 = _c("p1", "Jewell - Punishment Stool - Part 1", 232, date(2003, 11, 13))
|
||||||
|
p2 = _c("p2", "Jewell - Punishment Stool - Part 2", 231, date(2003, 11, 13))
|
||||||
|
assert not same_scene(p1, p2, _MILFY)
|
||||||
|
|
||||||
|
|
||||||
|
def test_guards():
|
||||||
|
base = _c("x", "Voluptuous Therapist Fucks Prisoner", 2218, date(2026, 9, 16), ("Koda Monroe",))
|
||||||
|
longer = _c("y", "Voluptuous Therapist Fucks Prisoner", 2600, date(2026, 9, 16), ("Koda Monroe",))
|
||||||
|
assert not same_scene(base, longer, _MILFY)
|
||||||
|
later = _c("z", "Voluptuous Therapist Fucks Prisoner", 2218, date(2026, 10, 1), ("Koda Monroe",))
|
||||||
|
assert not same_scene(base, later, _MILFY)
|
||||||
|
year_only = _c("j", "Voluptuous Therapist Fucks Prisoner", 2218, date(2026, 1, 1), ("Koda Monroe",))
|
||||||
|
assert same_scene(base, year_only, _MILFY)
|
||||||
|
other_cast = _c("o", "Voluptuous Therapist Fucks Prisoner", 2218, date(2026, 9, 16), ("Someone Else",))
|
||||||
|
assert not same_scene(base, other_cast, _MILFY)
|
||||||
|
other_studio = _c("s", "Voluptuous Therapist Fucks Prisoner", 2218, date(2026, 9, 16), studio="blacked")
|
||||||
|
assert not same_scene(base, other_studio, _MILFY)
|
||||||
|
# Bez dlugosci po obu stronach trzy slowa to za malo.
|
||||||
|
short_a = _c("sa", "[Milfy] Hot Stepmom Fun")
|
||||||
|
short_b = _c("sb", "Milfy 26 09 16 Hot Stepmom Fun")
|
||||||
|
assert not same_scene(short_a, short_b, _MILFY)
|
||||||
|
|
||||||
|
|
||||||
|
def test_chain_does_not_pull_in_incompatible_scene():
|
||||||
|
keep = _c("k", "Hotel Vixen Season 3 Episode 4 Soft Launch Hard Times", 2608, canon=True)
|
||||||
|
bridge = _c("m", "[Milfy] Hotel Vixen Season 3 Episode 4 Soft Launch Hard Times")
|
||||||
|
other = _c("l", "Milfy Hotel Vixen Season 3 Episode 4 Soft Launch Hard Times", 2896)
|
||||||
|
plan = plan_merges([keep, bridge, other])
|
||||||
|
assert [(k.id, [d.id for d in ds]) for k, ds in plan] == [("k", ["m"])]
|
||||||
|
|
||||||
|
|
||||||
|
def test_category_named_as_performer_does_not_shorten_core():
|
||||||
|
# Tube wpisal kategorie jako obsade. Bez listy smieci "big", "ass" i "tits" wypadlyby
|
||||||
|
# z obu tytulow i dwie rozne sceny mialyby ten sam rdzen "blonde fucks jason luv".
|
||||||
|
a = _c("a", "Big Ass Blonde Fucks Jason Luv", perfs=("Big Ass", "Big Tits"))
|
||||||
|
b = _c("b", "Big Tits Blonde Fucks Jason Luv")
|
||||||
|
assert not same_scene(a, b, _MILFY, junk_names=frozenset({"big ass", "big tits"}))
|
||||||
|
|
||||||
|
|
||||||
|
def test_copies_without_studio_join_only_with_matching_duration():
|
||||||
|
orphan = _c("o1", "Koda Monroe - Voluptuous Therapist Fucks Prisoner (17/9/26)", 2218, studio="")
|
||||||
|
orphan_4k = _c("o2", "Koda Monroe Voluptuous Therapist Fucks Prisoner 4K", 2217, studio="")
|
||||||
|
other_cut = _c("o3", "Koda Monroe Voluptuous Therapist Fucks Prisoner", 2600, studio="")
|
||||||
|
no_duration = _c("o4", "Koda Monroe Voluptuous Therapist Fucks Prisoner", None, studio="")
|
||||||
|
plan = plan_merges(_MILFY_COPIES + [orphan, orphan_4k, other_cut, no_duration])
|
||||||
|
keep, drops = plan[0]
|
||||||
|
assert keep.id == "canon"
|
||||||
|
ids = {d.id for d in drops}
|
||||||
|
assert {"o1", "o2"} <= ids
|
||||||
|
assert not ids & {"o3", "o4"}
|
||||||
|
# Dwie sceny bez studia nie maja kotwicy.
|
||||||
|
assert not same_scene(orphan, orphan_4k, _MILFY)
|
||||||
Loading…
Add table
Reference in a new issue