goon/scripts/fix_title_normalized.py

"""Przelicza `title_normalized` scen i filmow tam, gdzie rozjechal sie z `normalize(title)`.

Ingest liczy `title_normalized = normalize(title)`, ale stare skrypty naprawcze
(`bulk_rescrape_hqporner.py`, `repair_truncated_titles.py`) zapisywaly samo `lower(title)`.
Takie sceny maja w kluczu interpunkcje (`cindy hope - horny boss`, `matty's virgin`), wiec
nie trafiaja w dokladne dopasowanie tytulu w resolverze ani w dedup. Pelny skan z 2026-09-15:
6005 scen, wszystkie w formie `lower(title)`, innych rozjazdow nie bylo.

Skrypt czyta cala tabele strumieniowo (4,3 mln scen to ok. 1,5 minuty), zbiera wiersze,
w ktorych klucz sie nie zgadza, i zapisuje partiami. Zapis jest warunkowy
(`WHERE title = stary AND title_normalized = stary`), wiec wiersz zmieniony w miedzyczasie
jest pomijany. `title` i `slug` zostaja bez zmian.

Uzycie:
    python3 -u /srv/scripts/fix_title_normalized.py --dry-run
    python3 -u /srv/scripts/fix_title_normalized.py
"""
from __future__ import annotations

import argparse
import collections
import logging
import random
import sys
import time

import sqlalchemy as sa

from app.db import engine, session_scope
from app.normalize.text import normalize

log = logging.getLogger("fix_title_normalized")

_TABLES = ("scenes", "movies")


def _plan(table: str) -> tuple[list[dict], collections.Counter]:
    t0 = time.monotonic()
    plan: list[dict] = []
    kinds: collections.Counter = collections.Counter()
    scanned = 0
    with engine.connect().execution_options(stream_results=True, yield_per=20000) as conn:
        for row_id, title, title_norm in conn.execute(sa.text(f"SELECT id, title, title_normalized FROM {table}")):
            scanned += 1
            new = normalize(title)
            if new == title_norm:
                continue
            kinds["lower" if title_norm == (title or "").lower() else "inne"] += 1
            plan.append({"id": row_id, "title": title, "old": title_norm, "new": new})
    log.info(
        "%s: przejrzane %d w %ds, do zmiany %d %s",
        table, scanned, time.monotonic() - t0, len(plan), dict(kinds),
    )
    return plan, kinds


def _apply(table: str, plan: list[dict], *, batch_size: int) -> int:
    stmt = sa.text(
        f"UPDATE {table} t SET title_normalized = v.new "
        "FROM unnest(CAST(:ids AS uuid[]), CAST(:titles AS text[]), CAST(:olds AS text[]), "
        "CAST(:news AS text[])) AS v(id, title, old, new) "
        "WHERE t.id = v.id AND t.title = v.title AND t.title_normalized = v.old"
    )
    updated = 0
    for start in range(0, len(plan), batch_size):
        batch = plan[start : start + batch_size]
        params = {
            "ids": [str(item["id"]) for item in batch],
            "titles": [item["title"] for item in batch],
            "olds": [item["old"] for item in batch],
            "news": [item["new"] for item in batch],
        }
        with session_scope() as session:
            updated += session.execute(stmt, params).rowcount
        log.info("%s: %d/%d przejrzane, %d zapisane", table, start + len(batch), len(plan), updated)
    return updated


def main() -> int:
    p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    p.add_argument("--dry-run", action="store_true", help="tylko policz i pokaz probke")
    p.add_argument("--batch-size", type=int, default=500)
    p.add_argument("--sample", type=int, default=20, help="ile losowych zmian pokazac na tabele")
    p.add_argument(
        "--only-lower",
        action="store_true",
        help="zapisz tylko rozjazdy w formie lower(title), inne tylko pokaz",
    )
    args = p.parse_args()
    logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")

    total_planned = total_updated = 0
    for table in _TABLES:
        plan, kinds = _plan(table)
        if kinds.get("inne"):
            others = [item for item in plan if item["old"] != (item["title"] or "").lower()]
            for item in random.sample(others, min(args.sample, len(others))):
                print(f"[{table} inne] {item['title']!r}: {item['old']!r} -> {item['new']!r}")
            if args.only_lower:
                plan = [item for item in plan if item["old"] == (item["title"] or "").lower()]
        if not plan:
            continue
        for item in random.sample(plan, min(args.sample, len(plan))):
            print(f"[{table}] {item['title']!r}")
            print(f"    {item['old']!r}")
            print(f" -> {item['new']!r}")
        sys.stdout.flush()
        total_planned += len(plan)
        if not args.dry_run:
            total_updated += _apply(table, plan, batch_size=args.batch_size)

    if args.dry_run:
        log.info("DRY RUN: %d wierszy do zmiany, nic nie zapisane", total_planned)
    else:
        log.info("zapisane %d z %d (reszta zmieniona w miedzyczasie)", total_updated, total_planned)
    return 0


if __name__ == "__main__":
    sys.exit(main())