fix(ingest): dekoduj encje HTML w tytulach scen i filmow

xvideos (JSON-LD) i kilka innych tubow oddaje tytuly z encjami HTML5
(`we're`, `(AI)`), czasem podwojnie zakodowane. normalize
robil z nich slowa `apos`, `lpar`, wiec tytul nie trafial w wyszukiwarke
ani w kanon.

unescape_html w app/normalize/text.py dekoduje encje zakonczone srednikiem
po slowniku HTML5, do skutku (max 3 przebiegi). Wpiete w normalize_scene i
normalize_movie oraz w parsery xvideos_browse i pornxp. Skrypt
fix_html_entity_titles.py poprawia istniejace wiersze (title,
title_normalized, slug gdy byl wyliczony z tytulu).

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

app/connectors/direct_scrapers/pornxp.py

@@ -40,6 +40,7 @@ from app.connectors.direct_scrapers._browse_base import (
     BaseBrowseScraper,
     compute_thumbnail_phash,
 )
+from app.normalize.text import unescape_html
 
 log = logging.getLogger(__name__)
 
@@ -182,7 +183,7 @@ class PornXPScraper(BaseBrowseScraper):
                 ),
                 "thumb": thumb,
                 "duration_sec": _parse_mmss(m.group("dur") or ""),
-                "title": m.group("title").strip(),
+                "title": unescape_html(m.group("title")),
             }
             out.append(url)
         return out
@@ -200,7 +201,7 @@ class PornXPScraper(BaseBrowseScraper):
         # Title: prefer h1 over listing card title (detail h1 jest cleaner)
         title = meta.get("title") or ""
         if (m := _H1_RE.search(detail_html)):
-            title = m.group(1).strip() or title
+            title = unescape_html(m.group(1)) or title
         if not title:
             return None
 

app/connectors/direct_scrapers/xvideos_browse.py

@@ -19,7 +19,7 @@ from datetime import date, datetime
 
 from app.connectors.base import RawPerformer, RawPlaybackSource, RawScene, RawTag
 from app.connectors.direct_scrapers._browse_base import BaseBrowseScraper, meta_content
-from app.normalize.text import slugify
+from app.normalize.text import slugify, unescape_html
 
 log = logging.getLogger(__name__)
 
@@ -99,10 +99,11 @@ class XVideosBrowseScraper(BaseBrowseScraper):
     def _parse_detail(self, scene_url: str, detail_html: str) -> RawScene | None:
         video = _video_object(detail_html) or {}
 
-        title = (video.get("name") or "").strip()
+        # JSON-LD `name` niesie encje HTML5 (`we're`, `[AI]`), og:title tez.
+        title = unescape_html(video.get("name"))
         if not title:
             m = _SETTITLE_RE.search(detail_html)
-            title = m.group(1).strip() if m else (meta_content(detail_html, property="og:title") or "").strip()
+            title = unescape_html(m.group(1) if m else meta_content(detail_html, property="og:title"))
         if not title:
             return None
 

app/normalize/movies.py

@@ -13,7 +13,7 @@ from app.normalize.scenes import (
     normalize_studio,
     normalize_tag,
 )
-from app.normalize.text import normalize, slugify
+from app.normalize.text import normalize, slugify, unescape_html
 
 
 @dataclass
@@ -42,11 +42,12 @@ class NormalizedMovie:
 
 
 def normalize_movie(raw: RawMovie) -> NormalizedMovie:
+    title = unescape_html(raw.title)
     return NormalizedMovie(
         external_id=raw.external_id,
-        title=raw.title,
-        title_normalized=normalize(raw.title),
-        slug=slugify(raw.title),
+        title=title,
+        title_normalized=normalize(title),
+        slug=slugify(title),
         release_year=raw.release_year,
         release_date=raw.release_date,
         description=raw.description,

app/normalize/scenes.py

@@ -16,7 +16,7 @@ from app.connectors.base import (
     RawTag,
 )
 from app.normalize.tag_inference import infer_tag_slugs
-from app.normalize.text import normalize, normalize_person, slugify
+from app.normalize.text import normalize, normalize_person, slugify, unescape_html
 
 
 @dataclass
@@ -186,11 +186,13 @@ def _sane_release_date(value):
 
 
 def normalize_scene(raw: RawScene) -> NormalizedScene:
+    # Tuby oddaja tytuly z encjami HTML (`we're`), patrz `unescape_html`.
+    title = unescape_html(raw.title)
     return NormalizedScene(
         external_id=raw.external_id,
-        title=raw.title,
-        title_normalized=normalize(raw.title),
-        slug=slugify(raw.title),
+        title=title,
+        title_normalized=normalize(title),
+        slug=slugify(title),
         release_date=_sane_release_date(raw.release_date),
         description=raw.description,
         duration_sec=raw.duration_sec,
@@ -199,7 +201,7 @@ def normalize_scene(raw: RawScene) -> NormalizedScene:
         url=raw.url,
         studio=normalize_studio(raw.studio) if raw.studio else None,
         performers=[normalize_performer(p) for p in raw.performers],
-        tags=_merge_inferred_tags(raw.tags, title=raw.title),
+        tags=_merge_inferred_tags(raw.tags, title=title),
         fingerprints=[(fp.kind, fp.value) for fp in raw.fingerprints],
         playback_sources=list(raw.playback_sources),
         cross_source_refs=dict(raw.cross_source_refs),

app/normalize/text.py

@@ -5,8 +5,10 @@ muszą produkować identyczny string po `normalize`. Trgram index na tych polach
 """
 from __future__ import annotations
 
+import html
 import re
 import unicodedata
+from html.entities import html5 as html5_entities
 
 from slugify import slugify as _slugify
 
@@ -30,6 +32,46 @@ _EXTRA_TRANSLIT = str.maketrans(
 )
 
 
+# Encja z sredznikiem: nazwana HTML5 (`(`, `“`) albo numeryczna.
+_HTML_ENTITY_RE = re.compile(r"&(?:#[0-9]{1,7}|#[xX][0-9a-fA-F]{1,6}|[A-Za-z][A-Za-z0-9]{1,31});")
+_UNESCAPE_MAX_PASSES = 3
+
+
+def _decode_entity(m: re.Match[str]) -> str:
+    ent = m.group(0)
+    if ent[1] == "#":
+        return html.unescape(ent)
+    # Tylko pelna nazwa ze slownika HTML5. `html.unescape` na nieznanej nazwie dopasowuje
+    # przedrostek, wiec `&copyright;` zrobiloby sie `©right;`.
+    name = ent[1:]
+    if name in html5_entities:
+        return html5_entities[name]
+    # `&Amp;` i `&Quot;` biora sie z .title() na zakodowanym tytule, HTML5 ich nie zna.
+    return html5_entities.get(name.lower(), ent)
+
+
+def unescape_html(value: str | None) -> str:
+    """Dekoduje encje HTML w tytule ze scrapera, do skutku (max 3 przebiegi).
+
+    xvideos wklada do JSON-LD tytul z encjami HTML5 (`we're`, `(AI)`),
+    czasem podwojnie (`(`). Bez tego `normalize` robi z `'` slowo
+    `apos` i tytul nie trafia w kanon. Dekodujemy tylko encje zakonczone srednikiem,
+    po slowniku HTML5, z ktorego korzysta `html.unescape`. Samo `html.unescape` na
+    calym tekscie zamienia tez stare nazwy bez srednika, wiec `Me&notice` wyszloby
+    jako `Me¬ice`.
+    """
+    if not value:
+        return ""
+    text = value
+    for _ in range(_UNESCAPE_MAX_PASSES):
+        new = _HTML_ENTITY_RE.sub(_decode_entity, text)
+        if new == text:
+            break
+        text = new
+    # ` ` i `` (BOM) zostawiaja niewidoczne znaki, w tytule sa zbedne.
+    return text.replace("\xa0", " ").replace("", "").strip()
+
+
 def strip_accents(value: str) -> str:
     pre = value.translate(_EXTRA_TRANSLIT)
     nfkd = unicodedata.normalize("NFKD", pre)

scripts/fix_html_entity_titles.py

@@ -0,0 +1,138 @@
+"""Dekoduje encje HTML w tytulach scen i filmow zapisanych przed poprawka normalizera.
+
+Tuby (glownie xvideos, JSON-LD) oddawaly tytuly typu `we're [AI]`,
+a my zapisywalismy je tak jak przyszly. `normalize` robil z tego slowa `apos`, `lpar`,
+wiec tytul nie trafial ani w wyszukiwarke, ani w kanon. Od poprawki `normalize_scene`
+przepuszcza tytul przez `unescape_html`, ale istniejace wiersze trzeba poprawic osobno:
+scraper tytulu juz ustawionego nie nadpisuje.
+
+Co zmienia na wierszu:
+  - `title` na wersje po `unescape_html`,
+  - `title_normalized` na `normalize(nowy tytul)`, tak jak liczy go ingest,
+  - `slug` tylko gdy byl wyliczony ze starego tytulu (`slugify(stary) == slug`); slug
+    ustawiony z innego zrodla zostaje.
+
+Zapis jest warunkowy (`WHERE title = stary`), wiec wiersz zmieniony w miedzyczasie przez
+ingest albo merge jest pomijany. Ponowne odpalenie jest bezpieczne: bierze tylko tytuly,
+w ktorych nadal jest encja.
+
+Uzycie:
+    python3 -u /srv/scripts/fix_html_entity_titles.py --dry-run
+    python3 -u /srv/scripts/fix_html_entity_titles.py
+"""
+from __future__ import annotations
+
+import argparse
+import logging
+import random
+import sys
+
+import sqlalchemy as sa
+
+from app.db import session_scope
+from app.normalize.text import normalize, slugify, unescape_html
+
+log = logging.getLogger("fix_html_entity_titles")
+
+# Ten sam zestaw co `_HTML_ENTITY_RE` w app/normalize/text.py, w skladni Postgresa.
+_PG_ENTITY_RE = r"&(#[0-9]{1,7}|#[xX][0-9a-fA-F]{1,6}|[A-Za-z][A-Za-z0-9]{1,31});"
+_TABLES = ("scenes", "movies")
+
+
+def _plan(table: str, *, limit: int | None) -> list[dict]:
+    with session_scope() as session:
+        rows = session.execute(
+            sa.text(
+                f"SELECT id, title, title_normalized, slug FROM {table} "
+                "WHERE title ~ :re" + (" LIMIT :limit" if limit else "")
+            ),
+            {"re": _PG_ENTITY_RE, "limit": limit},
+        ).all()
+
+    plan: list[dict] = []
+    for row_id, title, title_norm, slug in rows:
+        new_title = unescape_html(title)
+        if not new_title or new_title == title:
+            continue
+        new_slug = slug
+        if slug is None or slug == slugify(title):
+            new_slug = slugify(new_title) or slug
+        plan.append(
+            {
+                "id": row_id,
+                "old_title": title,
+                "title": new_title,
+                "old_title_normalized": title_norm,
+                "title_normalized": normalize(new_title),
+                "old_slug": slug,
+                "slug": new_slug,
+            }
+        )
+    log.info("%s: %d wierszy z encja, %d do zmiany", table, len(rows), len(plan))
+    return plan
+
+
+def _apply(table: str, plan: list[dict], *, batch_size: int) -> int:
+    # Jedno UPDATE na partie przez unnest: rowcount jest wtedy wiarygodny.
+    stmt = sa.text(
+        f"UPDATE {table} t SET title = v.title, title_normalized = v.title_normalized, slug = v.slug "
+        "FROM unnest(CAST(:ids AS uuid[]), CAST(:old_titles AS text[]), CAST(:titles AS text[]), "
+        "CAST(:norms AS text[]), CAST(:slugs AS text[])) "
+        "AS v(id, old_title, title, title_normalized, slug) "
+        "WHERE t.id = v.id AND t.title = v.old_title"
+    )
+    updated = 0
+    for start in range(0, len(plan), batch_size):
+        batch = plan[start : start + batch_size]
+        params = {
+            "ids": [str(item["id"]) for item in batch],
+            "old_titles": [item["old_title"] for item in batch],
+            "titles": [item["title"] for item in batch],
+            "norms": [item["title_normalized"] for item in batch],
+            "slugs": [item["slug"] for item in batch],
+        }
+        with session_scope() as session:
+            updated += session.execute(stmt, params).rowcount
+        log.info("%s: %d/%d przejrzane, %d zapisane", table, start + len(batch), len(plan), updated)
+    return updated
+
+
+def main() -> int:
+    p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    p.add_argument("--dry-run", action="store_true", help="tylko policz i pokaz probke")
+    p.add_argument("--batch-size", type=int, default=500)
+    p.add_argument("--sample", type=int, default=20, help="ile losowych zmian pokazac")
+    p.add_argument("--limit", type=int, default=None, help="max wierszy na tabele (test)")
+    args = p.parse_args()
+    logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
+
+    total_planned = total_updated = 0
+    for table in _TABLES:
+        plan = _plan(table, limit=args.limit)
+        if not plan:
+            continue
+        total_planned += len(plan)
+        slug_changes = sum(1 for item in plan if item["slug"] != item["old_slug"])
+        log.info("%s: slug zmieniony w %d z %d", table, slug_changes, len(plan))
+        for item in random.sample(plan, min(args.sample, len(plan))):
+            print(f"[{table}] {item['id']}")
+            print(f"  title: {item['old_title']!r}")
+            print(f"      -> {item['title']!r}")
+            print(f"  norm:  {item['old_title_normalized']!r}")
+            print(f"      -> {item['title_normalized']!r}")
+            if item["slug"] != item["old_slug"]:
+                print(f"  slug:  {item['old_slug']!r}")
+                print(f"      -> {item['slug']!r}")
+        sys.stdout.flush()
+        if not args.dry_run:
+            total_updated += _apply(table, plan, batch_size=args.batch_size)
+
+    if args.dry_run:
+        log.info("DRY RUN: %d wierszy do zmiany, nic nie zapisane", total_planned)
+    else:
+        log.info("zapisane %d z %d (reszta zmieniona w miedzyczasie)", total_updated, total_planned)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tests/test_normalize.py

@@ -6,6 +6,7 @@ from app.normalize.text import (
     person_named_in,
     slugify,
     strip_accents,
+    unescape_html,
 )
 
 
@@ -81,3 +82,28 @@ def test_strip_accents_idempotent() -> None:
 )
 def test_person_named_in(title: str, person: str, expected: bool) -> None:
     assert person_named_in(title, person) is expected
+
+
+@pytest.mark.parametrize(
+    ("raw", "expected"),
+    [
+        # xvideos JSON-LD, encje HTML5.
+        (
+            "Are you sure we're allowed to do this mister Steve?? [AI]",
+            "Are you sure we're allowed to do this mister Steve?? [AI]",
+        ),
+        ("The BEST (( HOT WIFE ))", "The BEST (( HOT WIFE ))"),
+        # Podwojne kodowanie.
+        ("Barbara & Jake (4K)", "Barbara & Jake (4K)"),
+        ("'x' "q"", "'x' \"q\""),
+        # .title() na zakodowanym tytule.
+        ("Alix &Amp; Cherie", "Alix & Cherie"),
+        (" Hot Wife", "Hot Wife"),
+        # Bez srednika albo nieznana nazwa: zostaje jak bylo.
+        ("Me&notice Tom & Jerry &copyright; Q&A", "Me&notice Tom & Jerry &copyright; Q&A"),
+        ("", ""),
+        (None, ""),
+    ],
+)
+def test_unescape_html(raw: str | None, expected: str) -> None:
+    assert unescape_html(raw) == expected

tests/test_normalize_scenes.py

@@ -6,6 +6,7 @@ from pathlib import Path
 
 import pytest
 
+from app.connectors.base import RawScene
 from app.connectors.tpdb import _parse_scene
 from app.normalize.scenes import normalize_performer, normalize_scene, normalize_studio
 from app.normalize.text import normalize, normalize_person
@@ -58,3 +59,11 @@ def test_normalize_scene_preserves_performer_order(tpdb_scene_raw: dict) -> None
     assert raw is not None
     norm = normalize_scene(raw)
     assert [p.canonical_name for p in norm.performers] == ["Mia Malkova", "Johnny Sins"]
+
+
+def test_normalize_scene_unescapes_html_entities_in_title() -> None:
+    raw = RawScene(external_id="xvideoscom:x", title="Mama's boy (4K)")
+    norm = normalize_scene(raw)
+    assert norm.title == "Mama's boy (4K)"
+    assert norm.title_normalized == normalize("Mama's boy (4K)")
+    assert norm.slug == "mama-s-boy-4k"