fix(movies): kandydaci do dopasowania po podobienstwie tytulu
Some checks are pending
Backend tests / test (push) Waiting to run
Some checks are pending
Backend tests / test (push) Waiting to run
find_movie_candidates bral dowolne 50 wierszy z sumy trigramu i bloku studio+rok. Duze studio ma kilkaset filmow w trzech latach, wiec wierny duplikat wypadal z puli i powstawal nowy film. Teraz kolejnosc po similarity(). Skrypt remerge dostal tryb --same-title-studio (scalono 930 par o tym samym tytule i studiu). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
37f8eb1762
commit
e0f70f2574
2 changed files with 63 additions and 1 deletions
|
|
@ -32,8 +32,13 @@ def find_movie_candidates(
|
||||||
net dla SEO tytułów (np. mangoporn często ma "Watch Cece adult..." vs paradisehill
|
net dla SEO tytułów (np. mangoporn często ma "Watch Cece adult..." vs paradisehill
|
||||||
"Cece", trigram złapie, ale gdyby nie, studio+year wystarczają).
|
"Cece", trigram złapie, ale gdyby nie, studio+year wystarczają).
|
||||||
"""
|
"""
|
||||||
from sqlalchemy import literal
|
from sqlalchemy import func, literal
|
||||||
|
|
||||||
|
# Kolejnosc po podobienstwie tytulu: bez niej LIMIT bral dowolne 50 z sumy trigramu i
|
||||||
|
# bloku studio+rok, a duze studio ma kilkaset filmow w trzech latach. Wtedy wierny
|
||||||
|
# duplikat (ten sam tytul, rok, studio i obsada) wypadal z puli i powstawal nowy film
|
||||||
|
# (2026-10-16: 13 z 228 nowych filmow z pandamovies, po scaleniu blizniaczych studiow
|
||||||
|
# bloki urosly).
|
||||||
q = session.execute(
|
q = session.execute(
|
||||||
select(Movie)
|
select(Movie)
|
||||||
.where(
|
.where(
|
||||||
|
|
@ -45,6 +50,7 @@ def find_movie_candidates(
|
||||||
_studio_year_block(studio_id, release_year),
|
_studio_year_block(studio_id, release_year),
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
.order_by(func.similarity(Movie.title_normalized, literal(title_normalized)).desc())
|
||||||
.limit(limit)
|
.limit(limit)
|
||||||
).scalars()
|
).scalars()
|
||||||
return list(q)
|
return list(q)
|
||||||
|
|
|
||||||
|
|
@ -66,11 +66,67 @@ def _keep_rank(session, movie: Movie) -> tuple:
|
||||||
return (movie.release_year is not None, cast, -movie.created_at.timestamp())
|
return (movie.release_year is not None, cast, -movie.created_at.timestamp())
|
||||||
|
|
||||||
|
|
||||||
|
def _same_title_studio(apply: bool) -> None:
|
||||||
|
"""Grupy filmow o identycznym tytule i studiu. find_movie_candidates bral dowolne 50
|
||||||
|
kandydatow bez kolejnosci (do 2026-10-16), wiec przy duzych studiach wierny duplikat
|
||||||
|
wypadal z puli. Kazdy czlonek grupy oceniany wzgledem zostajacego; ten sam prog co
|
||||||
|
ingest, wiec wznowienia z innym rokiem (ocena max 0,8) zostaja osobno."""
|
||||||
|
with session_scope() as s:
|
||||||
|
groups = s.execute(text("""
|
||||||
|
SELECT array_agg(id) FROM movies
|
||||||
|
WHERE studio_id IS NOT NULL AND length(title_normalized) >= 4
|
||||||
|
GROUP BY title_normalized, studio_id HAVING count(*) > 1
|
||||||
|
""")).scalars().all()
|
||||||
|
rejected = _rejected_pairs(s)
|
||||||
|
print(f"grup: {len(groups)}", flush=True)
|
||||||
|
merged = 0
|
||||||
|
for ids in groups:
|
||||||
|
with session_scope() as s:
|
||||||
|
members = sorted((s.get(Movie, i) for i in ids), key=lambda x: _keep_rank(s, x), reverse=True)
|
||||||
|
keep = members[0]
|
||||||
|
for m in members[1:]:
|
||||||
|
if frozenset((keep.id, m.id)) in rejected:
|
||||||
|
continue
|
||||||
|
perfs = [
|
||||||
|
r.performer_id
|
||||||
|
for r in s.execute(
|
||||||
|
MoviePerformer.__table__.select().where(MoviePerformer.movie_id == m.id)
|
||||||
|
).fetchall()
|
||||||
|
]
|
||||||
|
b = score_movie_candidate(
|
||||||
|
s,
|
||||||
|
candidate=keep,
|
||||||
|
norm_title_normalized=m.title_normalized,
|
||||||
|
norm_release_year=m.release_year,
|
||||||
|
norm_studio_id=m.studio_id,
|
||||||
|
norm_performer_ids=perfs,
|
||||||
|
)
|
||||||
|
if b.composite < AUTO_THRESHOLD:
|
||||||
|
continue
|
||||||
|
print(
|
||||||
|
f"{b.composite:.2f} ZOSTAJE {keep.title!r} ({keep.release_year})"
|
||||||
|
f" <= {m.title!r} ({m.release_year})",
|
||||||
|
flush=True,
|
||||||
|
)
|
||||||
|
if apply:
|
||||||
|
try:
|
||||||
|
merge_movies(s, keep_id=keep.id, drop_id=m.id, resolved_by="remerge_same_title_studio_20261016")
|
||||||
|
except MovieMergeError as e:
|
||||||
|
print(" blad:", e, flush=True)
|
||||||
|
continue
|
||||||
|
merged += 1
|
||||||
|
print(f"{'scalono' if apply else 'do scalenia'}: {merged}", flush=True)
|
||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
def main() -> None:
|
||||||
ap = argparse.ArgumentParser()
|
ap = argparse.ArgumentParser()
|
||||||
ap.add_argument("--apply", action="store_true")
|
ap.add_argument("--apply", action="store_true")
|
||||||
ap.add_argument("--limit", type=int, default=0)
|
ap.add_argument("--limit", type=int, default=0)
|
||||||
|
ap.add_argument("--same-title-studio", action="store_true")
|
||||||
args = ap.parse_args()
|
args = ap.parse_args()
|
||||||
|
if args.same_title_studio:
|
||||||
|
_same_title_studio(args.apply)
|
||||||
|
return
|
||||||
|
|
||||||
with session_scope() as s:
|
with session_scope() as s:
|
||||||
ids = _affected_ids(s)
|
ids = _affected_ids(s)
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue