chore(studios): skrypt scalajacy blizniacze studia
Some checks are pending
Backend tests / test (push) Waiting to run
Some checks are pending
Backend tests / test (push) Waiting to run
Studia o tej samej nazwie bez spacji ("EvilAngel" i "Evil Angel") scalane
w jedno: sceny, filmy, refy, aliasy, ulubione i ukrycia przechodza do studia
z najwiekszym katalogiem, nazwa scalanego zostaje aliasem.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
f167c491c8
commit
f58ae3ea2d
1 changed files with 138 additions and 0 deletions
138
scripts/merge_twin_studios.py
Normal file
138
scripts/merge_twin_studios.py
Normal file
|
|
@ -0,0 +1,138 @@
|
|||
"""Scala blizniacze studia: te same nazwy po usunieciu spacji ("EvilAngel" = "Evil Angel",
|
||||
"Let's Post It" = "Lets PostIt"). 2026-10-16: ok. 2 tys. grup z tubow, TPDB i stashdb;
|
||||
rozdzielaly katalog studia na kilka stron i psuly dopasowanie filmow (zgloszenie f8c9486e).
|
||||
|
||||
Zostaje studio z najwiekszym katalogiem (scene_count + filmy), potem z ref TPDB/stashdb,
|
||||
potem zapis ze spacja, potem starsze. Do niego przechodza sceny, filmy, refy zewnetrzne,
|
||||
aliasy, studia podrzedne, ulubione i ukrycia uzytkownikow; nazwa scalanego zostaje
|
||||
aliasem. Ulubione na obu: zostaje jeden wpis z wczesniejszym created_at i pozniejszym
|
||||
last_seen_at (bez zalewu "+N"). Kazda grupa w osobnej transakcji.
|
||||
|
||||
python3 /srv/scripts/merge_twin_studios.py # dry-run, plan i liczby
|
||||
python3 /srv/scripts/merge_twin_studios.py --apply
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
|
||||
from sqlalchemy import text
|
||||
|
||||
from app.db import session_scope
|
||||
|
||||
_GROUPS_SQL = """
|
||||
WITH s AS (
|
||||
SELECT id, replace(name_normalized, ' ', '') AS k FROM studios
|
||||
WHERE length(replace(name_normalized, ' ', '')) >= 4
|
||||
)
|
||||
SELECT k, array_agg(id) FROM s
|
||||
WHERE k IN (SELECT k FROM s GROUP BY k HAVING count(*) > 1)
|
||||
GROUP BY k ORDER BY k
|
||||
"""
|
||||
|
||||
_MEMBERS_SQL = """
|
||||
SELECT st.id, st.name, st.name_normalized, st.parent_studio_id, st.scene_count,
|
||||
(SELECT count(*) FROM movies m WHERE m.studio_id = st.id) AS movies,
|
||||
EXISTS (SELECT 1 FROM studio_external_refs r JOIN sources so ON so.id = r.source_id
|
||||
WHERE r.studio_id = st.id AND so.name IN ('tpdb', 'stashdb')) AS canonical,
|
||||
st.created_at
|
||||
FROM studios st WHERE st.id = ANY(:ids)
|
||||
"""
|
||||
|
||||
|
||||
def _rank(m) -> tuple:
|
||||
return (m.scene_count + m.movies, m.canonical, " " in m.name_normalized, -m.created_at.timestamp())
|
||||
|
||||
|
||||
def _merge(s, keep, drop) -> int:
|
||||
p = {"keep": keep.id, "drop": drop.id}
|
||||
s.execute(text(
|
||||
"UPDATE studios SET parent_studio_id = :keep WHERE parent_studio_id = :drop AND id <> :keep"
|
||||
), p)
|
||||
# Rodzic zatrzymanego: nie moze wskazywac na siebie; brakujacy dziedziczy po scalanym.
|
||||
keep_parent = keep.parent_studio_id
|
||||
if keep_parent == drop.id or keep_parent is None:
|
||||
keep_parent = drop.parent_studio_id if drop.parent_studio_id not in (keep.id, drop.id) else None
|
||||
s.execute(text("""
|
||||
UPDATE studios k SET parent_studio_id = :parent,
|
||||
network = coalesce(k.network, d.network),
|
||||
homepage_url = coalesce(k.homepage_url, d.homepage_url),
|
||||
scene_count = k.scene_count + d.scene_count,
|
||||
updated_at = now()
|
||||
FROM studios d WHERE k.id = :keep AND d.id = :drop
|
||||
"""), {**p, "parent": keep_parent})
|
||||
|
||||
moved = s.execute(text("UPDATE scenes SET studio_id = :keep WHERE studio_id = :drop"), p).rowcount
|
||||
s.execute(text("UPDATE movies SET studio_id = :keep WHERE studio_id = :drop"), p)
|
||||
s.execute(text("UPDATE studio_external_refs SET studio_id = :keep WHERE studio_id = :drop"), p)
|
||||
|
||||
s.execute(text("""
|
||||
DELETE FROM studio_aliases a WHERE a.studio_id = :drop AND EXISTS (
|
||||
SELECT 1 FROM studio_aliases b WHERE b.studio_id = :keep AND b.alias_normalized = a.alias_normalized)
|
||||
"""), p)
|
||||
s.execute(text("UPDATE studio_aliases SET studio_id = :keep WHERE studio_id = :drop"), p)
|
||||
s.execute(text("""
|
||||
INSERT INTO studio_aliases (studio_id, alias, alias_normalized)
|
||||
SELECT :keep, d.name, d.name_normalized FROM studios d, studios k
|
||||
WHERE d.id = :drop AND k.id = :keep AND d.name_normalized <> k.name_normalized
|
||||
AND NOT EXISTS (SELECT 1 FROM studio_aliases b
|
||||
WHERE b.studio_id = :keep AND b.alias_normalized = d.name_normalized)
|
||||
"""), p)
|
||||
|
||||
s.execute(text("""
|
||||
UPDATE favorite_studios k SET created_at = least(k.created_at, d.created_at),
|
||||
last_seen_at = greatest(k.last_seen_at, d.last_seen_at)
|
||||
FROM favorite_studios d
|
||||
WHERE k.studio_id = :keep AND d.studio_id = :drop AND d.device_id = k.device_id
|
||||
"""), p)
|
||||
s.execute(text("""
|
||||
DELETE FROM favorite_studios d WHERE d.studio_id = :drop AND EXISTS (
|
||||
SELECT 1 FROM favorite_studios k WHERE k.studio_id = :keep AND k.device_id = d.device_id)
|
||||
"""), p)
|
||||
s.execute(text("UPDATE favorite_studios SET studio_id = :keep WHERE studio_id = :drop"), p)
|
||||
s.execute(text("""
|
||||
DELETE FROM blacklisted_studios d WHERE d.studio_id = :drop AND EXISTS (
|
||||
SELECT 1 FROM blacklisted_studios k WHERE k.studio_id = :keep AND k.device_id = d.device_id)
|
||||
"""), p)
|
||||
s.execute(text("UPDATE blacklisted_studios SET studio_id = :keep WHERE studio_id = :drop"), p)
|
||||
|
||||
s.execute(text("DELETE FROM studios WHERE id = :drop"), p)
|
||||
return moved
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--apply", action="store_true")
|
||||
ap.add_argument("--limit", type=int, default=0)
|
||||
args = ap.parse_args()
|
||||
|
||||
with session_scope() as s:
|
||||
groups = s.execute(text(_GROUPS_SQL)).all()
|
||||
if args.limit:
|
||||
groups = groups[: args.limit]
|
||||
print(f"grup: {len(groups)}", flush=True)
|
||||
|
||||
dropped = scenes = 0
|
||||
for n, (key, ids) in enumerate(groups, 1):
|
||||
with session_scope() as s:
|
||||
members = sorted(s.execute(text(_MEMBERS_SQL), {"ids": list(ids)}).all(), key=_rank, reverse=True)
|
||||
if len(members) < 2:
|
||||
continue
|
||||
keep, rest = members[0], members[1:]
|
||||
print(
|
||||
f"ZOSTAJE {keep.name!r} ({keep.scene_count}/{keep.movies}) <= "
|
||||
+ ", ".join(f"{m.name!r} ({m.scene_count}/{m.movies})" for m in rest),
|
||||
flush=True,
|
||||
)
|
||||
if args.apply:
|
||||
for d in rest:
|
||||
# swiezo z bazy: poprzednie scalenie w tej grupie moglo zmienic rodzica
|
||||
fresh = {r.id: r for r in s.execute(text(_MEMBERS_SQL), {"ids": [keep.id, d.id]}).all()}
|
||||
scenes += _merge(s, fresh[keep.id], fresh[d.id])
|
||||
dropped += len(rest)
|
||||
if n % 200 == 0:
|
||||
print(f"... {n}/{len(groups)}, studiow scalonych: {dropped}, scen przepietych: {scenes}", flush=True)
|
||||
print(f"{'scalono' if args.apply else 'do scalenia'}: {dropped} studiow, scen przepietych: {scenes}", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
Add table
Reference in a new issue