fix(ingest): czysc opisy z encji i znacznikow HTML, title_normalized zawsze przez normalize

Opisy scen szly do bazy z `&#34;`, `&lt;br&gt;` i surowymi `<br />`, a apka i strona
SEO pokazuja opis jako tekst, wiec bylo to widac doslownie. clean_description
dekoduje encje, `<br>` i koniec akapitu zamienia na nowa linie, a znane znaczniki
wycina (`<3` zostaje tekstem). Wpiete w normalize_scene i normalize_movie.

bulk_rescrape_hqporner.py i repair_truncated_titles.py liczyly title_normalized
jako lower(title), przez co 6005 scen mialo interpunkcje w kluczu dopasowania.
Oba licza teraz normalize() i dekoduja encje w tytule.

Skrypty: fix_html_entity_titles.py przemianowany na fix_html_entities.py z
przebiegiem po opisach; nowy fix_title_normalized.py przelicza klucz tam, gdzie
rozjechal sie z normalize(title).

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

app/normalize/movies.py

@@ -13,7 +13,7 @@ from app.normalize.scenes import (
     normalize_studio,
     normalize_tag,
 )
-from app.normalize.text import normalize, slugify, unescape_html
+from app.normalize.text import clean_description, normalize, slugify, unescape_html
 
 
 @dataclass
@@ -50,7 +50,7 @@ def normalize_movie(raw: RawMovie) -> NormalizedMovie:
         slug=slugify(title),
         release_year=raw.release_year,
         release_date=raw.release_date,
-        description=raw.description,
+        description=clean_description(raw.description),
         duration_sec=raw.duration_sec,
         director=raw.director,
         country=raw.country,

app/normalize/scenes.py

@@ -16,7 +16,13 @@ from app.connectors.base import (
     RawTag,
 )
 from app.normalize.tag_inference import infer_tag_slugs
-from app.normalize.text import normalize, normalize_person, slugify, unescape_html
+from app.normalize.text import (
+    clean_description,
+    normalize,
+    normalize_person,
+    slugify,
+    unescape_html,
+)
 
 
 @dataclass
@@ -194,7 +200,7 @@ def normalize_scene(raw: RawScene) -> NormalizedScene:
         title_normalized=normalize(title),
         slug=slugify(title),
         release_date=_sane_release_date(raw.release_date),
-        description=raw.description,
+        description=clean_description(raw.description),
         duration_sec=raw.duration_sec,
         code=raw.code,
         director=raw.director,

app/normalize/text.py

@@ -62,14 +62,47 @@ def unescape_html(value: str | None) -> str:
     """
     if not value:
         return ""
-    text = value
+    return _decode_entities(value).strip()
+
+
+def _decode_entities(text: str) -> str:
     for _ in range(_UNESCAPE_MAX_PASSES):
         new = _HTML_ENTITY_RE.sub(_decode_entity, text)
         if new == text:
             break
         text = new
-    # ` ` i `` (BOM) zostawiaja niewidoczne znaki, w tytule sa zbedne.
+    # ` ` i `` (BOM) zostawiaja niewidoczne znaki, w tekscie sa zbedne.
-    return text.replace("\xa0", " ").replace("", "").strip()
+    return text.replace("\N{NO-BREAK SPACE}", " ").replace("\N{ZERO WIDTH NO-BREAK SPACE}", "")
+
+
+# Znaczniki, ktore tuby zostawiaja w opisach, surowe albo jako `<br>`. Lista nazw
+# jest zamknieta, a atrybuty musza miec postac `nazwa=wartosc`, bo `<3`, `<Yokomiya-san`
+# czy `<a lot of fun>` w opisach to tekst, nie znacznik.
+_BREAK_TAG_RE = re.compile(r"<br\s*/?>|</(?:br|p|div|li|h[1-6])\s*>", re.IGNORECASE)
+_INLINE_TAG_RE = re.compile(
+    r"</?(?:a|b|i|u|em|strong|span|div|p|font|small|big|sup|sub|h[1-6]|ul|ol|li|center|blockquote)"
+    r"""(?:\s+[\w:-]+\s*=\s*(?:"[^"]*"|'[^']*'|[^\s"'<>]+))*\s*/?>""",
+    re.IGNORECASE,
+)
+_TRAILING_WS_RE = re.compile(r"[ \t]+\n")
+_MANY_NEWLINES_RE = re.compile(r"\n{3,}")
+
+
+def clean_description(value: str | None) -> str | None:
+    """Opis ze scrapera jako zwykly tekst: encje odkodowane, znaczniki HTML usuniete.
+
+    Apka pokazuje opis w `Text`, strona SEO przez autoescape, wiec `&#34;` i `<br>`
+    widac doslownie. Encje ida pierwsze, bo czesc tubow koduje tez znaczniki
+    (`&lt;br&gt;`). `<br>` i koniec akapitu zamieniamy na nowa linie, reszte wycinamy.
+    """
+    if not value:
+        return None
+    text = _decode_entities(value)
+    text = _BREAK_TAG_RE.sub("\n", text)
+    text = _INLINE_TAG_RE.sub("", text)
+    text = _TRAILING_WS_RE.sub("\n", text)
+    text = _MANY_NEWLINES_RE.sub("\n\n", text)
+    return text.strip() or None
 
 
 def strip_accents(value: str) -> str:

scripts/bulk_rescrape_hqporner.py

@@ -28,7 +28,7 @@ from app.extractors.thumb_extract import extract_thumbnail_url
 from app.models.playback_source import PlaybackSource
 from app.models.scene import Scene, SceneTag
 from app.normalize.scenes import NormalizedTag
-from app.normalize.text import slugify
+from app.normalize.text import normalize, slugify, unescape_html
 from app.resolve.tag_resolver import resolve_tag
 
 log = logging.getLogger(__name__)
@@ -71,7 +71,7 @@ def rescrape_scene(session, scene_id: uuid.UUID, pb_id: uuid.UUID, page_url: str
 
     # Title, jeśli current ma triple space (slug-derived), zastąp z <title>.
     if (tm := _TITLE_RE.search(html)):
-        new_title = _cleanup_title(tm.group(1))
+        new_title = _cleanup_title(unescape_html(tm.group(1)))
         if new_title:
             scene = session.get(Scene, scene_id)
             if scene and ("   " in scene.title or scene.title.strip() != new_title):
@@ -79,7 +79,7 @@ def rescrape_scene(session, scene_id: uuid.UUID, pb_id: uuid.UUID, page_url: str
                 # Bezpieczna heurystyka: nie nadpisuj jeśli current jest dłuższy.
                 if "   " in scene.title or len(new_title) > len(scene.title):
                     scene.title = new_title
-                    scene.title_normalized = new_title.lower()
+                    scene.title_normalized = normalize(new_title)
                     out["updated_title"] = 1
 
     # Duration

scripts/fix_html_entities.py

@@ -0,0 +1,211 @@
+"""Czysci encje i znaczniki HTML w tytulach i opisach scen i filmow sprzed poprawki normalizera.
+
+Tuby (glownie xvideos, JSON-LD) oddawaly tytuly typu `we're [AI]`, a opisy
+z `"` i `<br>`. Zapisywalismy je tak jak przyszly, wiec `normalize` robil z tytulu
+slowa `apos`, `lpar`, a w apce i na stronie SEO encje widac doslownie. Od poprawki
+`normalize_scene` przepuszcza tytul przez `unescape_html`, a opis przez `clean_description`,
+ale istniejace wiersze trzeba poprawic osobno: scraper pola juz ustawionego nie nadpisuje.
+
+Przebieg `titles` zmienia na wierszu:
+  - `title` na wersje po `unescape_html`,
+  - `title_normalized` na `normalize(nowy tytul)`, tak jak liczy go ingest,
+  - `slug` tylko gdy byl wyliczony ze starego tytulu (`slugify(stary) == slug`).
+Przebieg `descriptions` zmienia `description` na wersje po `clean_description`.
+
+Zapis jest warunkowy (`WHERE pole = stara wartosc`), wiec wiersz zmieniony w miedzyczasie
+przez ingest albo merge jest pomijany. Ponowne odpalenie jest bezpieczne: bierze tylko
+wiersze, w ktorych nadal jest encja albo znacznik.
+
+Uzycie:
+    python3 -u /srv/scripts/fix_html_entities.py --dry-run
+    python3 -u /srv/scripts/fix_html_entities.py --only descriptions
+"""
+from __future__ import annotations
+
+import argparse
+import logging
+import random
+import sys
+
+import sqlalchemy as sa
+
+from app.db import session_scope
+from app.normalize.text import clean_description, normalize, slugify, unescape_html
+
+log = logging.getLogger("fix_html_entities")
+
+# Ten sam zestaw co `_HTML_ENTITY_RE` w app/normalize/text.py, w skladni Postgresa.
+_PG_ENTITY_RE = r"&(#[0-9]{1,7}|#[xX][0-9a-fA-F]{1,6}|[A-Za-z][A-Za-z0-9]{1,31});"
+# Wstepny filtr na znaczniki z `_BREAK_TAG_RE`/`_INLINE_TAG_RE`; decyduje `clean_description`.
+_PG_TAG_RE = (
+    r"</?(a|b|i|u|em|strong|span|div|p|font|small|big|sup|sub|h[1-6]|ul|ol|li|center"
+    r"|blockquote|br)([[:space:]/>])"
+)
+_TABLES = ("scenes", "movies")
+
+
+def _plan_titles(table: str, *, limit: int | None) -> list[dict]:
+    with session_scope() as session:
+        rows = session.execute(
+            sa.text(
+                f"SELECT id, title, title_normalized, slug FROM {table} "
+                "WHERE title ~ :re" + (" LIMIT :limit" if limit else "")
+            ),
+            {"re": _PG_ENTITY_RE, "limit": limit},
+        ).all()
+
+    plan: list[dict] = []
+    for row_id, title, title_norm, slug in rows:
+        new_title = unescape_html(title)
+        if not new_title or new_title == title:
+            continue
+        new_slug = slug
+        if slug is None or slug == slugify(title):
+            new_slug = slugify(new_title) or slug
+        plan.append(
+            {
+                "id": row_id,
+                "old_title": title,
+                "title": new_title,
+                "old_title_normalized": title_norm,
+                "title_normalized": normalize(new_title),
+                "old_slug": slug,
+                "slug": new_slug,
+            }
+        )
+    log.info("%s.title: %d wierszy z encja, %d do zmiany", table, len(rows), len(plan))
+    return plan
+
+
+def _plan_descriptions(table: str, *, limit: int | None) -> list[dict]:
+    with session_scope() as session:
+        rows = session.execute(
+            sa.text(
+                f"SELECT id, description FROM {table} "
+                "WHERE description ~ :ent OR description ~* :tag" + (" LIMIT :limit" if limit else "")
+            ),
+            {"ent": _PG_ENTITY_RE, "tag": _PG_TAG_RE, "limit": limit},
+        ).all()
+
+    plan = [
+        {"id": row_id, "old_description": desc, "description": new}
+        for row_id, desc in rows
+        if (new := clean_description(desc)) != desc
+    ]
+    log.info("%s.description: %d wierszy z encja lub znacznikiem, %d do zmiany", table, len(rows), len(plan))
+    return plan
+
+
+def _apply_titles(table: str, plan: list[dict], *, batch_size: int) -> int:
+    # Jedno UPDATE na partie przez unnest: rowcount jest wtedy wiarygodny.
+    stmt = sa.text(
+        f"UPDATE {table} t SET title = v.title, title_normalized = v.title_normalized, slug = v.slug "
+        "FROM unnest(CAST(:ids AS uuid[]), CAST(:old_titles AS text[]), CAST(:titles AS text[]), "
+        "CAST(:norms AS text[]), CAST(:slugs AS text[])) "
+        "AS v(id, old_title, title, title_normalized, slug) "
+        "WHERE t.id = v.id AND t.title = v.old_title"
+    )
+    return _run_batches(
+        f"{table}.title",
+        stmt,
+        plan,
+        batch_size=batch_size,
+        columns={"ids": "id", "old_titles": "old_title", "titles": "title", "norms": "title_normalized", "slugs": "slug"},
+    )
+
+
+def _apply_descriptions(table: str, plan: list[dict], *, batch_size: int) -> int:
+    stmt = sa.text(
+        f"UPDATE {table} t SET description = v.description "
+        "FROM unnest(CAST(:ids AS uuid[]), CAST(:old AS text[]), CAST(:new AS text[])) "
+        "AS v(id, old_description, description) "
+        "WHERE t.id = v.id AND t.description = v.old_description"
+    )
+    return _run_batches(
+        f"{table}.description",
+        stmt,
+        plan,
+        batch_size=batch_size,
+        columns={"ids": "id", "old": "old_description", "new": "description"},
+    )
+
+
+def _run_batches(label: str, stmt, plan: list[dict], *, batch_size: int, columns: dict[str, str]) -> int:
+    updated = 0
+    for start in range(0, len(plan), batch_size):
+        batch = plan[start : start + batch_size]
+        params = {
+            param: [str(item[key]) if key == "id" else item[key] for item in batch]
+            for param, key in columns.items()
+        }
+        with session_scope() as session:
+            updated += session.execute(stmt, params).rowcount
+        log.info("%s: %d/%d przejrzane, %d zapisane", label, start + len(batch), len(plan), updated)
+    return updated
+
+
+def _diff_window(old: str, new: str | None, width: int = 90) -> tuple[str, str]:
+    """Wycinek wokol pierwszej roznicy, bo opisy sa dlugie, a zmiana bywa w srodku."""
+    new = new or ""
+    i = next((k for k, (a, b) in enumerate(zip(old, new)) if a != b), min(len(old), len(new)))
+    start = max(0, i - width // 3)
+    return old[start : start + width], new[start : start + width]
+
+
+def _print_title_sample(table: str, plan: list[dict], n: int) -> None:
+    for item in random.sample(plan, min(n, len(plan))):
+        print(f"[{table}.title] {item['id']}")
+        print(f"  title: {item['old_title']!r}")
+        print(f"      -> {item['title']!r}")
+        print(f"  norm:  {item['old_title_normalized']!r}")
+        print(f"      -> {item['title_normalized']!r}")
+        if item["slug"] != item["old_slug"]:
+            print(f"  slug:  {item['old_slug']!r}")
+            print(f"      -> {item['slug']!r}")
+
+
+def _print_description_sample(table: str, plan: list[dict], n: int) -> None:
+    for item in random.sample(plan, min(n, len(plan))):
+        old, new = _diff_window(item["old_description"], item["description"])
+        print(f"[{table}.description] {item['id']}")
+        print(f"  ...{old!r}")
+        print(f"  ...{new!r}")
+
+
+def main() -> int:
+    p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    p.add_argument("--dry-run", action="store_true", help="tylko policz i pokaz probke")
+    p.add_argument("--only", choices=("titles", "descriptions"), default=None)
+    p.add_argument("--batch-size", type=int, default=500)
+    p.add_argument("--sample", type=int, default=20, help="ile losowych zmian pokazac na przebieg")
+    p.add_argument("--limit", type=int, default=None, help="max wierszy na tabele (test)")
+    args = p.parse_args()
+    logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
+
+    passes = []
+    if args.only in (None, "titles"):
+        passes.append((_plan_titles, _print_title_sample, _apply_titles))
+    if args.only in (None, "descriptions"):
+        passes.append((_plan_descriptions, _print_description_sample, _apply_descriptions))
+
+    total_planned = total_updated = 0
+    for plan_fn, sample_fn, apply_fn in passes:
+        for table in _TABLES:
+            plan = plan_fn(table, limit=args.limit)
+            if not plan:
+                continue
+            total_planned += len(plan)
+            sample_fn(table, plan, args.sample)
+            sys.stdout.flush()
+            if not args.dry_run:
+                total_updated += apply_fn(table, plan, batch_size=args.batch_size)
+
+    if args.dry_run:
+        log.info("DRY RUN: %d zmian w planie, nic nie zapisane", total_planned)
+    else:
+        log.info("zapisane %d z %d (reszta zmieniona w miedzyczasie)", total_updated, total_planned)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/fix_html_entity_titles.py

@@ -1,138 +0,0 @@
-"""Dekoduje encje HTML w tytulach scen i filmow zapisanych przed poprawka normalizera.
-
-Tuby (glownie xvideos, JSON-LD) oddawaly tytuly typu `we're [AI]`,
-a my zapisywalismy je tak jak przyszly. `normalize` robil z tego slowa `apos`, `lpar`,
-wiec tytul nie trafial ani w wyszukiwarke, ani w kanon. Od poprawki `normalize_scene`
-przepuszcza tytul przez `unescape_html`, ale istniejace wiersze trzeba poprawic osobno:
-scraper tytulu juz ustawionego nie nadpisuje.
-
-Co zmienia na wierszu:
-  - `title` na wersje po `unescape_html`,
-  - `title_normalized` na `normalize(nowy tytul)`, tak jak liczy go ingest,
-  - `slug` tylko gdy byl wyliczony ze starego tytulu (`slugify(stary) == slug`); slug
-    ustawiony z innego zrodla zostaje.
-
-Zapis jest warunkowy (`WHERE title = stary`), wiec wiersz zmieniony w miedzyczasie przez
-ingest albo merge jest pomijany. Ponowne odpalenie jest bezpieczne: bierze tylko tytuly,
-w ktorych nadal jest encja.
-
-Uzycie:
-    python3 -u /srv/scripts/fix_html_entity_titles.py --dry-run
-    python3 -u /srv/scripts/fix_html_entity_titles.py
-"""
-from __future__ import annotations
-
-import argparse
-import logging
-import random
-import sys
-
-import sqlalchemy as sa
-
-from app.db import session_scope
-from app.normalize.text import normalize, slugify, unescape_html
-
-log = logging.getLogger("fix_html_entity_titles")
-
-# Ten sam zestaw co `_HTML_ENTITY_RE` w app/normalize/text.py, w skladni Postgresa.
-_PG_ENTITY_RE = r"&(#[0-9]{1,7}|#[xX][0-9a-fA-F]{1,6}|[A-Za-z][A-Za-z0-9]{1,31});"
-_TABLES = ("scenes", "movies")
-
-
-def _plan(table: str, *, limit: int | None) -> list[dict]:
-    with session_scope() as session:
-        rows = session.execute(
-            sa.text(
-                f"SELECT id, title, title_normalized, slug FROM {table} "
-                "WHERE title ~ :re" + (" LIMIT :limit" if limit else "")
-            ),
-            {"re": _PG_ENTITY_RE, "limit": limit},
-        ).all()
-
-    plan: list[dict] = []
-    for row_id, title, title_norm, slug in rows:
-        new_title = unescape_html(title)
-        if not new_title or new_title == title:
-            continue
-        new_slug = slug
-        if slug is None or slug == slugify(title):
-            new_slug = slugify(new_title) or slug
-        plan.append(
-            {
-                "id": row_id,
-                "old_title": title,
-                "title": new_title,
-                "old_title_normalized": title_norm,
-                "title_normalized": normalize(new_title),
-                "old_slug": slug,
-                "slug": new_slug,
-            }
-        )
-    log.info("%s: %d wierszy z encja, %d do zmiany", table, len(rows), len(plan))
-    return plan
-
-
-def _apply(table: str, plan: list[dict], *, batch_size: int) -> int:
-    # Jedno UPDATE na partie przez unnest: rowcount jest wtedy wiarygodny.
-    stmt = sa.text(
-        f"UPDATE {table} t SET title = v.title, title_normalized = v.title_normalized, slug = v.slug "
-        "FROM unnest(CAST(:ids AS uuid[]), CAST(:old_titles AS text[]), CAST(:titles AS text[]), "
-        "CAST(:norms AS text[]), CAST(:slugs AS text[])) "
-        "AS v(id, old_title, title, title_normalized, slug) "
-        "WHERE t.id = v.id AND t.title = v.old_title"
-    )
-    updated = 0
-    for start in range(0, len(plan), batch_size):
-        batch = plan[start : start + batch_size]
-        params = {
-            "ids": [str(item["id"]) for item in batch],
-            "old_titles": [item["old_title"] for item in batch],
-            "titles": [item["title"] for item in batch],
-            "norms": [item["title_normalized"] for item in batch],
-            "slugs": [item["slug"] for item in batch],
-        }
-        with session_scope() as session:
-            updated += session.execute(stmt, params).rowcount
-        log.info("%s: %d/%d przejrzane, %d zapisane", table, start + len(batch), len(plan), updated)
-    return updated
-
-
-def main() -> int:
-    p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
-    p.add_argument("--dry-run", action="store_true", help="tylko policz i pokaz probke")
-    p.add_argument("--batch-size", type=int, default=500)
-    p.add_argument("--sample", type=int, default=20, help="ile losowych zmian pokazac")
-    p.add_argument("--limit", type=int, default=None, help="max wierszy na tabele (test)")
-    args = p.parse_args()
-    logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
-
-    total_planned = total_updated = 0
-    for table in _TABLES:
-        plan = _plan(table, limit=args.limit)
-        if not plan:
-            continue
-        total_planned += len(plan)
-        slug_changes = sum(1 for item in plan if item["slug"] != item["old_slug"])
-        log.info("%s: slug zmieniony w %d z %d", table, slug_changes, len(plan))
-        for item in random.sample(plan, min(args.sample, len(plan))):
-            print(f"[{table}] {item['id']}")
-            print(f"  title: {item['old_title']!r}")
-            print(f"      -> {item['title']!r}")
-            print(f"  norm:  {item['old_title_normalized']!r}")
-            print(f"      -> {item['title_normalized']!r}")
-            if item["slug"] != item["old_slug"]:
-                print(f"  slug:  {item['old_slug']!r}")
-                print(f"      -> {item['slug']!r}")
-        sys.stdout.flush()
-        if not args.dry_run:
-            total_updated += _apply(table, plan, batch_size=args.batch_size)
-
-    if args.dry_run:
-        log.info("DRY RUN: %d wierszy do zmiany, nic nie zapisane", total_planned)
-    else:
-        log.info("zapisane %d z %d (reszta zmieniona w miedzyczasie)", total_updated, total_planned)
-    return 0
-
-
-if __name__ == "__main__":
-    sys.exit(main())

scripts/fix_title_normalized.py

@@ -0,0 +1,120 @@
+"""Przelicza `title_normalized` scen i filmow tam, gdzie rozjechal sie z `normalize(title)`.
+
+Ingest liczy `title_normalized = normalize(title)`, ale stare skrypty naprawcze
+(`bulk_rescrape_hqporner.py`, `repair_truncated_titles.py`) zapisywaly samo `lower(title)`.
+Takie sceny maja w kluczu interpunkcje (`cindy hope - horny boss`, `matty's virgin`), wiec
+nie trafiaja w dokladne dopasowanie tytulu w resolverze ani w dedup. Pelny skan z 2026-09-15:
+6005 scen, wszystkie w formie `lower(title)`, innych rozjazdow nie bylo.
+
+Skrypt czyta cala tabele strumieniowo (4,3 mln scen to ok. 1,5 minuty), zbiera wiersze,
+w ktorych klucz sie nie zgadza, i zapisuje partiami. Zapis jest warunkowy
+(`WHERE title = stary AND title_normalized = stary`), wiec wiersz zmieniony w miedzyczasie
+jest pomijany. `title` i `slug` zostaja bez zmian.
+
+Uzycie:
+    python3 -u /srv/scripts/fix_title_normalized.py --dry-run
+    python3 -u /srv/scripts/fix_title_normalized.py
+"""
+from __future__ import annotations
+
+import argparse
+import collections
+import logging
+import random
+import sys
+import time
+
+import sqlalchemy as sa
+
+from app.db import engine, session_scope
+from app.normalize.text import normalize
+
+log = logging.getLogger("fix_title_normalized")
+
+_TABLES = ("scenes", "movies")
+
+
+def _plan(table: str) -> tuple[list[dict], collections.Counter]:
+    t0 = time.monotonic()
+    plan: list[dict] = []
+    kinds: collections.Counter = collections.Counter()
+    scanned = 0
+    with engine.connect().execution_options(stream_results=True, yield_per=20000) as conn:
+        for row_id, title, title_norm in conn.execute(sa.text(f"SELECT id, title, title_normalized FROM {table}")):
+            scanned += 1
+            new = normalize(title)
+            if new == title_norm:
+                continue
+            kinds["lower" if title_norm == (title or "").lower() else "inne"] += 1
+            plan.append({"id": row_id, "title": title, "old": title_norm, "new": new})
+    log.info(
+        "%s: przejrzane %d w %ds, do zmiany %d %s",
+        table, scanned, time.monotonic() - t0, len(plan), dict(kinds),
+    )
+    return plan, kinds
+
+
+def _apply(table: str, plan: list[dict], *, batch_size: int) -> int:
+    stmt = sa.text(
+        f"UPDATE {table} t SET title_normalized = v.new "
+        "FROM unnest(CAST(:ids AS uuid[]), CAST(:titles AS text[]), CAST(:olds AS text[]), "
+        "CAST(:news AS text[])) AS v(id, title, old, new) "
+        "WHERE t.id = v.id AND t.title = v.title AND t.title_normalized = v.old"
+    )
+    updated = 0
+    for start in range(0, len(plan), batch_size):
+        batch = plan[start : start + batch_size]
+        params = {
+            "ids": [str(item["id"]) for item in batch],
+            "titles": [item["title"] for item in batch],
+            "olds": [item["old"] for item in batch],
+            "news": [item["new"] for item in batch],
+        }
+        with session_scope() as session:
+            updated += session.execute(stmt, params).rowcount
+        log.info("%s: %d/%d przejrzane, %d zapisane", table, start + len(batch), len(plan), updated)
+    return updated
+
+
+def main() -> int:
+    p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    p.add_argument("--dry-run", action="store_true", help="tylko policz i pokaz probke")
+    p.add_argument("--batch-size", type=int, default=500)
+    p.add_argument("--sample", type=int, default=20, help="ile losowych zmian pokazac na tabele")
+    p.add_argument(
+        "--only-lower",
+        action="store_true",
+        help="zapisz tylko rozjazdy w formie lower(title), inne tylko pokaz",
+    )
+    args = p.parse_args()
+    logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
+
+    total_planned = total_updated = 0
+    for table in _TABLES:
+        plan, kinds = _plan(table)
+        if kinds.get("inne"):
+            others = [item for item in plan if item["old"] != (item["title"] or "").lower()]
+            for item in random.sample(others, min(args.sample, len(others))):
+                print(f"[{table} inne] {item['title']!r}: {item['old']!r} -> {item['new']!r}")
+            if args.only_lower:
+                plan = [item for item in plan if item["old"] == (item["title"] or "").lower()]
+        if not plan:
+            continue
+        for item in random.sample(plan, min(args.sample, len(plan))):
+            print(f"[{table}] {item['title']!r}")
+            print(f"    {item['old']!r}")
+            print(f" -> {item['new']!r}")
+        sys.stdout.flush()
+        total_planned += len(plan)
+        if not args.dry_run:
+            total_updated += _apply(table, plan, batch_size=args.batch_size)
+
+    if args.dry_run:
+        log.info("DRY RUN: %d wierszy do zmiany, nic nie zapisane", total_planned)
+    else:
+        log.info("zapisane %d z %d (reszta zmieniona w miedzyczasie)", total_updated, total_planned)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/repair_truncated_titles.py

@@ -18,6 +18,7 @@ import sqlalchemy as sa
 
 from app.connectors.direct_scrapers._browse_base import meta_content
 from app.db import session_scope
+from app.normalize.text import normalize, unescape_html
 from app.extractors._fetch import browser_get
 
 log = logging.getLogger(__name__)
@@ -59,7 +60,7 @@ def main() -> int:
         if r.status_code >= 400:
             continue
 
-        new_title = meta_content(r.text, property="og:title")
+        new_title = unescape_html(meta_content(r.text, property="og:title"))
         if not new_title:
             continue
         # Only update if longer (i.e. parser found full title after fix)
@@ -70,9 +71,9 @@ def main() -> int:
         with session_scope() as session:
             session.execute(
                 sa.text(
-                    "UPDATE scenes SET title = :t, title_normalized = LOWER(:t) WHERE id = :sid"
+                    "UPDATE scenes SET title = :t, title_normalized = :tn WHERE id = :sid"
                 ),
-                {"t": new_title.strip(), "sid": scene_id},
+                {"t": new_title, "tn": normalize(new_title), "sid": scene_id},
             )
         counters["updated"] += 1
         if i % 25 == 0:

tests/test_normalize.py

@@ -1,6 +1,7 @@
 import pytest
 
 from app.normalize.text import (
+    clean_description,
     normalize,
     normalize_person,
     person_named_in,
@@ -107,3 +108,28 @@ def test_person_named_in(title: str, person: str, expected: bool) -> None:
 )
 def test_unescape_html(raw: str | None, expected: str) -> None:
     assert unescape_html(raw) == expected
+
+
+@pytest.mark.parametrize(
+    ("raw", "expected"),
+    [
+        ('Manny in "MANUEL CREAMPIES 2". She loved it.', 'Manny in "MANUEL CREAMPIES 2". She loved it.'),
+        # Znaczniki zakodowane jako encje.
+        ("Is it love?<br>Eveline only ordered", "Is it love?\nEveline only ordered"),
+        ("door.</br></br>Jessica", "door.\n\nJessica"),
+        ("wash.<br><br><br><br>She", "wash.\n\nShe"),
+        # Surowe znaczniki.
+        ("Welcome, Carlo!<br />&nbsp;", "Welcome, Carlo!"),
+        ('<p>See <a href="/x">Bunny</a> &amp; <b>more</b>!</p>', "See Bunny & more!"),
+        # To nie sa znaczniki.
+        ("I miss you so much &lt;3.", "I miss you so much <3."),
+        ("It was <a lot of fun> and <b careful>", "It was <a lot of fun> and <b careful>"),
+        ('<span style="color: red" class=x>Hot</span>', "Hot"),
+        ("&lt;Yokomiya-san. SSSP! No Cuts]", "<Yokomiya-san. SSSP! No Cuts]"),
+        ("lips&tongues; Alix &Amp; Cherie", "lips&tongues; Alix & Cherie"),
+        ("&nbsp;", None),
+        (None, None),
+    ],
+)
+def test_clean_description(raw: str | None, expected: str | None) -> None:
+    assert clean_description(raw) == expected