"""Przelicza `title_normalized` scen i filmow tam, gdzie rozjechal sie z `normalize(title)`. Ingest liczy `title_normalized = normalize(title)`, ale stare skrypty naprawcze (`bulk_rescrape_hqporner.py`, `repair_truncated_titles.py`) zapisywaly samo `lower(title)`. Takie sceny maja w kluczu interpunkcje (`cindy hope - horny boss`, `matty's virgin`), wiec nie trafiaja w dokladne dopasowanie tytulu w resolverze ani w dedup. Pelny skan z 2026-09-15: 6005 scen, wszystkie w formie `lower(title)`, innych rozjazdow nie bylo. Skrypt czyta cala tabele strumieniowo (4,3 mln scen to ok. 1,5 minuty), zbiera wiersze, w ktorych klucz sie nie zgadza, i zapisuje partiami. Zapis jest warunkowy (`WHERE title = stary AND title_normalized = stary`), wiec wiersz zmieniony w miedzyczasie jest pomijany. `title` i `slug` zostaja bez zmian. Uzycie: python3 -u /srv/scripts/fix_title_normalized.py --dry-run python3 -u /srv/scripts/fix_title_normalized.py """ from __future__ import annotations import argparse import collections import logging import random import sys import time import sqlalchemy as sa from app.db import engine, session_scope from app.normalize.text import normalize log = logging.getLogger("fix_title_normalized") _TABLES = ("scenes", "movies") def _plan(table: str) -> tuple[list[dict], collections.Counter]: t0 = time.monotonic() plan: list[dict] = [] kinds: collections.Counter = collections.Counter() scanned = 0 with engine.connect().execution_options(stream_results=True, yield_per=20000) as conn: for row_id, title, title_norm in conn.execute(sa.text(f"SELECT id, title, title_normalized FROM {table}")): scanned += 1 new = normalize(title) if new == title_norm: continue kinds["lower" if title_norm == (title or "").lower() else "inne"] += 1 plan.append({"id": row_id, "title": title, "old": title_norm, "new": new}) log.info( "%s: przejrzane %d w %ds, do zmiany %d %s", table, scanned, time.monotonic() - t0, len(plan), dict(kinds), ) return plan, kinds def _apply(table: str, plan: list[dict], *, batch_size: int) -> int: stmt = sa.text( f"UPDATE {table} t SET title_normalized = v.new " "FROM unnest(CAST(:ids AS uuid[]), CAST(:titles AS text[]), CAST(:olds AS text[]), " "CAST(:news AS text[])) AS v(id, title, old, new) " "WHERE t.id = v.id AND t.title = v.title AND t.title_normalized = v.old" ) updated = 0 for start in range(0, len(plan), batch_size): batch = plan[start : start + batch_size] params = { "ids": [str(item["id"]) for item in batch], "titles": [item["title"] for item in batch], "olds": [item["old"] for item in batch], "news": [item["new"] for item in batch], } with session_scope() as session: updated += session.execute(stmt, params).rowcount log.info("%s: %d/%d przejrzane, %d zapisane", table, start + len(batch), len(plan), updated) return updated def main() -> int: p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) p.add_argument("--dry-run", action="store_true", help="tylko policz i pokaz probke") p.add_argument("--batch-size", type=int, default=500) p.add_argument("--sample", type=int, default=20, help="ile losowych zmian pokazac na tabele") p.add_argument( "--only-lower", action="store_true", help="zapisz tylko rozjazdy w formie lower(title), inne tylko pokaz", ) args = p.parse_args() logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") total_planned = total_updated = 0 for table in _TABLES: plan, kinds = _plan(table) if kinds.get("inne"): others = [item for item in plan if item["old"] != (item["title"] or "").lower()] for item in random.sample(others, min(args.sample, len(others))): print(f"[{table} inne] {item['title']!r}: {item['old']!r} -> {item['new']!r}") if args.only_lower: plan = [item for item in plan if item["old"] == (item["title"] or "").lower()] if not plan: continue for item in random.sample(plan, min(args.sample, len(plan))): print(f"[{table}] {item['title']!r}") print(f" {item['old']!r}") print(f" -> {item['new']!r}") sys.stdout.flush() total_planned += len(plan) if not args.dry_run: total_updated += _apply(table, plan, batch_size=args.batch_size) if args.dry_run: log.info("DRY RUN: %d wierszy do zmiany, nic nie zapisane", total_planned) else: log.info("zapisane %d z %d (reszta zmieniona w miedzyczasie)", total_updated, total_planned) return 0 if __name__ == "__main__": sys.exit(main())