fix(performers): skrypt rozbijajacy zlepionych wykonawcow "A, B"

Scraper siski do 494b15a zapisywal kilka osob z alt jako jedna. Skrypt
przepina sceny, filmy i ulubione na prawdziwe osoby (resolve_performer),
czarnych list nie przenosi. "Woods, Monique" rozpoznaje jako jedna osobe
tylko gdy tytul sceny to potwierdza. Na prodzie: 660 zlepkow, 1194
powiazania, 4 nowe osoby, 4 niejasne zostaly.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
goon-foss 2026-09-27 19:37:23 +02:00
parent c97af57979
commit 559c3bf01c

View file

@ -0,0 +1,172 @@
"""Rozbija zlepionych wykonawcow typu "Cory Chase, Gigi Sweets" na prawdziwe osoby.
Scraper siski do 2026-09-27 (494b15a) bral pierwszy czlon alt='A, B - Tytul' jako jedno
nazwisko, wiec powstalo ~660 takich "osob". Kazda czesc przechodzi przez
resolve_performer (ta sama sciezka co ingest: nazwa, potem alias, na koncu nowa osoba),
powiazania ze scenami, filmami i ulubionymi przechodza na czesci, zlepek znika.
Czarne listy NIE przechodza: blokada "A, B" to nie blokada wszystkiego z A i z B.
Pomijamy nazwiska z ref do TPDB/StashDB (kanon nie ma przecinkow, wiec to bylby
swiadomy wyjatek) i takie, ktorych czesci nie wygladaja na nazwiska.
docker compose run --rm worker python3 -u /srv/scripts/split_comma_performers.py # na sucho
docker compose run --rm worker python3 -u /srv/scripts/split_comma_performers.py --commit
--commit najpierw kopiuje zlepki i ich powiazania do tabel *_comma_20260927.
"""
from __future__ import annotations
import collections
import sys
from sqlalchemy import text
from app.connectors.base import RawPerformer
from app.db import session_scope
from app.normalize.scenes import normalize_performer
from app.resolve.performer_resolver import resolve_performer
COMMIT = "--commit" in sys.argv
BACKUP_SUFFIX = "comma_20260927"
# Tabele z (..., performer_id) w kluczu glownym, powiazania przenosimy na kazda czesc.
MOVE_TABLES = ("scene_performers", "movie_performers", "favorite_performers")
DROP_TABLES = ("blacklisted_performers",)
def parts_of(name: str) -> list[str] | None:
parts = [p.strip() for p in name.split(",")]
if len(parts) < 2 or len(parts) > 10 or any(not p for p in parts):
return None
for p in parts:
words = p.split()
# Cyfry tylko w jednowyrazowym nicku ("Subgirl0831"), nie w "Scene 2".
if len(p) < 2 or len(words) > 4 or (len(words) > 1 and any(ch.isdigit() for ch in p)):
return None
# "Merce Palau aka Merce", "Ruby Reign. Yoya Grey": inny separator, zostawiamy.
if "." in p or " aka " in f" {p.lower()} ":
return None
return parts
def two_single_words(parts: list[str]) -> bool:
return len(parts) == 2 and all(len(p.split()) == 1 for p in parts)
def reversed_person(session, pid, parts: list[str]) -> str | None:
""""Woods, Monique" to jedna osoba w zapisie nazwisko, imie, gdy "Monique Woods" jest
w bazie i stoi w tytule ktorejs z jej scen. Samo istnienie nazwiska nie wystarcza:
"Jenna, Chloe" przy "Wild Brunette Threesome" to raczej dwie osoby."""
if not two_single_words(parts):
return None
name = f"{parts[1]} {parts[0]}"
norm = normalize_performer(RawPerformer(name=name)).name_normalized
hit = session.execute(text("""
SELECT 1 FROM performers WHERE name_normalized = :n
AND EXISTS (SELECT 1 FROM scene_performers sp JOIN scenes sc ON sc.id = sp.scene_id
WHERE sp.performer_id = :pid AND sc.title ILIKE :pat)
LIMIT 1
"""), {"n": norm, "pid": pid, "pat": f"%{name}%"}).first()
return name if hit else None
def columns(session, table: str) -> list[str]:
return list(session.execute(text(
"SELECT column_name FROM information_schema.columns "
"WHERE table_schema = 'public' AND table_name = :t ORDER BY ordinal_position"
), {"t": table}).scalars())
with session_scope() as s:
tube_source = s.execute(text("SELECT id FROM sources WHERE name = 'tube-scraper'")).scalar_one()
canon_sources = list(s.execute(text(
"SELECT id FROM sources WHERE kind IN ('tpdb', 'stashdb')"
)).scalars())
junk = s.execute(text("""
SELECT p.id, p.canonical_name FROM performers p
WHERE p.canonical_name LIKE '%,%'
AND NOT EXISTS (
SELECT 1 FROM performer_external_refs r
WHERE r.performer_id = p.id AND r.source_id = ANY(:canon))
ORDER BY p.canonical_name
"""), {"canon": canon_sources}).all()
print(f"zlepkow z przecinkiem bez refu kanonu: {len(junk)}", flush=True)
usage = collections.Counter()
for t in MOVE_TABLES + DROP_TABLES:
usage[t] = s.execute(text(
f"SELECT count(*) FROM {t} WHERE performer_id = ANY(:ids)"
), {"ids": [j.id for j in junk]}).scalar()
print("powiazania:", dict(usage), flush=True)
plan: list[tuple] = []
skipped: list[str] = []
for pid, name in junk:
parts = parts_of(name)
if parts is None:
skipped.append(name)
continue
whole = reversed_person(s, pid, parts)
if whole:
print(f" nazwisko, imie: {name!r} -> {whole!r}")
parts = [whole]
elif two_single_words(parts) and s.execute(
text("SELECT 1 FROM performers WHERE name_normalized = :n LIMIT 1"),
{"n": normalize_performer(RawPerformer(name=f"{parts[1]} {parts[0]}")).name_normalized},
).first():
# Odwrocone nazwisko istnieje, ale tytul go nie potwierdza: nie zgadujemy.
skipped.append(name)
continue
plan.append((pid, name, parts))
print(f"do rozbicia: {len(plan)}, pominiete: {len(skipped)}", flush=True)
for name in skipped[:30]:
print(" pomijam:", name)
existing = new = 0
samples = []
targets: dict = {}
for pid, name, parts in plan:
ids = []
for part in parts:
norm = normalize_performer(RawPerformer(name=part))
hit = s.execute(text(
"SELECT id FROM performers WHERE name_normalized = :n "
"UNION ALL SELECT performer_id FROM performer_aliases WHERE alias_normalized = :n LIMIT 1"
), {"n": norm.name_normalized}).scalar()
if hit is not None and hit != pid:
existing += 1
else:
new += 1
if COMMIT:
ids.append(resolve_performer(s, norm=norm, source_id=tube_source).id)
samples.append((name, part, "jest" if hit is not None and hit != pid else "NOWA"))
targets[pid] = ids
print(f"czesci: {existing} juz w bazie, {new} nowych osob", flush=True)
for name, part, st in samples[:10] + [x for x in samples if x[2] == "NOWA"]:
print(f" {name!r} -> {part!r} ({st})")
if not COMMIT:
print("NA SUCHO, nic nie zmienione. --commit zeby wykonac.")
sys.exit(0)
ids = [pid for pid, _, _ in plan]
s.execute(text(f"DROP TABLE IF EXISTS performers_{BACKUP_SUFFIX}"))
s.execute(text(f"CREATE TABLE performers_{BACKUP_SUFFIX} AS SELECT * FROM performers WHERE id = ANY(:ids)"), {"ids": ids})
for t in MOVE_TABLES + DROP_TABLES:
s.execute(text(f"DROP TABLE IF EXISTS {t}_{BACKUP_SUFFIX}"))
s.execute(text(f"CREATE TABLE {t}_{BACKUP_SUFFIX} AS SELECT * FROM {t} WHERE performer_id = ANY(:ids)"), {"ids": ids})
moved = collections.Counter()
for t in MOVE_TABLES:
cols = columns(s, t)
sel = ", ".join(":new" if c == "performer_id" else c for c in cols)
for pid, _, _ in plan:
for new_id in targets[pid]:
if new_id == pid:
continue
moved[t] += s.execute(text(
f"INSERT INTO {t} ({', '.join(cols)}) SELECT {sel} FROM {t} "
f"WHERE performer_id = :old ON CONFLICT DO NOTHING"
), {"new": new_id, "old": pid}).rowcount or 0
deleted = s.execute(text("DELETE FROM performers WHERE id = ANY(:ids)"), {"ids": ids}).rowcount
s.commit()
print(f"przeniesione powiazania: {dict(moved)}, skasowane zlepki: {deleted}", flush=True)