diff --git a/app/connectors/direct_scrapers/pornxp.py b/app/connectors/direct_scrapers/pornxp.py index 22091cc..ac1990b 100644 --- a/app/connectors/direct_scrapers/pornxp.py +++ b/app/connectors/direct_scrapers/pornxp.py @@ -40,6 +40,7 @@ from app.connectors.direct_scrapers._browse_base import ( BaseBrowseScraper, compute_thumbnail_phash, ) +from app.normalize.text import unescape_html log = logging.getLogger(__name__) @@ -182,7 +183,7 @@ class PornXPScraper(BaseBrowseScraper): ), "thumb": thumb, "duration_sec": _parse_mmss(m.group("dur") or ""), - "title": m.group("title").strip(), + "title": unescape_html(m.group("title")), } out.append(url) return out @@ -200,7 +201,7 @@ class PornXPScraper(BaseBrowseScraper): # Title: prefer h1 over listing card title (detail h1 jest cleaner) title = meta.get("title") or "" if (m := _H1_RE.search(detail_html)): - title = m.group(1).strip() or title + title = unescape_html(m.group(1)) or title if not title: return None diff --git a/app/connectors/direct_scrapers/xvideos_browse.py b/app/connectors/direct_scrapers/xvideos_browse.py index ea2a65f..cdb3386 100644 --- a/app/connectors/direct_scrapers/xvideos_browse.py +++ b/app/connectors/direct_scrapers/xvideos_browse.py @@ -19,7 +19,7 @@ from datetime import date, datetime from app.connectors.base import RawPerformer, RawPlaybackSource, RawScene, RawTag from app.connectors.direct_scrapers._browse_base import BaseBrowseScraper, meta_content -from app.normalize.text import slugify +from app.normalize.text import slugify, unescape_html log = logging.getLogger(__name__) @@ -99,10 +99,11 @@ class XVideosBrowseScraper(BaseBrowseScraper): def _parse_detail(self, scene_url: str, detail_html: str) -> RawScene | None: video = _video_object(detail_html) or {} - title = (video.get("name") or "").strip() + # JSON-LD `name` niesie encje HTML5 (`we're`, `[AI]`), og:title tez. + title = unescape_html(video.get("name")) if not title: m = _SETTITLE_RE.search(detail_html) - title = m.group(1).strip() if m else (meta_content(detail_html, property="og:title") or "").strip() + title = unescape_html(m.group(1) if m else meta_content(detail_html, property="og:title")) if not title: return None diff --git a/app/normalize/movies.py b/app/normalize/movies.py index 7aec668..1f4abb7 100644 --- a/app/normalize/movies.py +++ b/app/normalize/movies.py @@ -13,7 +13,7 @@ from app.normalize.scenes import ( normalize_studio, normalize_tag, ) -from app.normalize.text import normalize, slugify +from app.normalize.text import normalize, slugify, unescape_html @dataclass @@ -42,11 +42,12 @@ class NormalizedMovie: def normalize_movie(raw: RawMovie) -> NormalizedMovie: + title = unescape_html(raw.title) return NormalizedMovie( external_id=raw.external_id, - title=raw.title, - title_normalized=normalize(raw.title), - slug=slugify(raw.title), + title=title, + title_normalized=normalize(title), + slug=slugify(title), release_year=raw.release_year, release_date=raw.release_date, description=raw.description, diff --git a/app/normalize/scenes.py b/app/normalize/scenes.py index e28d290..d157616 100644 --- a/app/normalize/scenes.py +++ b/app/normalize/scenes.py @@ -16,7 +16,7 @@ from app.connectors.base import ( RawTag, ) from app.normalize.tag_inference import infer_tag_slugs -from app.normalize.text import normalize, normalize_person, slugify +from app.normalize.text import normalize, normalize_person, slugify, unescape_html @dataclass @@ -186,11 +186,13 @@ def _sane_release_date(value): def normalize_scene(raw: RawScene) -> NormalizedScene: + # Tuby oddaja tytuly z encjami HTML (`we're`), patrz `unescape_html`. + title = unescape_html(raw.title) return NormalizedScene( external_id=raw.external_id, - title=raw.title, - title_normalized=normalize(raw.title), - slug=slugify(raw.title), + title=title, + title_normalized=normalize(title), + slug=slugify(title), release_date=_sane_release_date(raw.release_date), description=raw.description, duration_sec=raw.duration_sec, @@ -199,7 +201,7 @@ def normalize_scene(raw: RawScene) -> NormalizedScene: url=raw.url, studio=normalize_studio(raw.studio) if raw.studio else None, performers=[normalize_performer(p) for p in raw.performers], - tags=_merge_inferred_tags(raw.tags, title=raw.title), + tags=_merge_inferred_tags(raw.tags, title=title), fingerprints=[(fp.kind, fp.value) for fp in raw.fingerprints], playback_sources=list(raw.playback_sources), cross_source_refs=dict(raw.cross_source_refs), diff --git a/app/normalize/text.py b/app/normalize/text.py index a1086cc..9742566 100644 --- a/app/normalize/text.py +++ b/app/normalize/text.py @@ -5,8 +5,10 @@ muszą produkować identyczny string po `normalize`. Trgram index na tych polach """ from __future__ import annotations +import html import re import unicodedata +from html.entities import html5 as html5_entities from slugify import slugify as _slugify @@ -30,6 +32,46 @@ _EXTRA_TRANSLIT = str.maketrans( ) +# Encja z sredznikiem: nazwana HTML5 (`(`, `“`) albo numeryczna. +_HTML_ENTITY_RE = re.compile(r"&(?:#[0-9]{1,7}|#[xX][0-9a-fA-F]{1,6}|[A-Za-z][A-Za-z0-9]{1,31});") +_UNESCAPE_MAX_PASSES = 3 + + +def _decode_entity(m: re.Match[str]) -> str: + ent = m.group(0) + if ent[1] == "#": + return html.unescape(ent) + # Tylko pelna nazwa ze slownika HTML5. `html.unescape` na nieznanej nazwie dopasowuje + # przedrostek, wiec `©right;` zrobiloby sie `©right;`. + name = ent[1:] + if name in html5_entities: + return html5_entities[name] + # `&Amp;` i `&Quot;` biora sie z .title() na zakodowanym tytule, HTML5 ich nie zna. + return html5_entities.get(name.lower(), ent) + + +def unescape_html(value: str | None) -> str: + """Dekoduje encje HTML w tytule ze scrapera, do skutku (max 3 przebiegi). + + xvideos wklada do JSON-LD tytul z encjami HTML5 (`we're`, `(AI)`), + czasem podwojnie (`(`). Bez tego `normalize` robi z `'` slowo + `apos` i tytul nie trafia w kanon. Dekodujemy tylko encje zakonczone srednikiem, + po slowniku HTML5, z ktorego korzysta `html.unescape`. Samo `html.unescape` na + calym tekscie zamienia tez stare nazwy bez srednika, wiec `Me¬ice` wyszloby + jako `Me¬ice`. + """ + if not value: + return "" + text = value + for _ in range(_UNESCAPE_MAX_PASSES): + new = _HTML_ENTITY_RE.sub(_decode_entity, text) + if new == text: + break + text = new + # ` ` i `` (BOM) zostawiaja niewidoczne znaki, w tytule sa zbedne. + return text.replace("\xa0", " ").replace("", "").strip() + + def strip_accents(value: str) -> str: pre = value.translate(_EXTRA_TRANSLIT) nfkd = unicodedata.normalize("NFKD", pre) diff --git a/scripts/fix_html_entity_titles.py b/scripts/fix_html_entity_titles.py new file mode 100644 index 0000000..a45c4ac --- /dev/null +++ b/scripts/fix_html_entity_titles.py @@ -0,0 +1,138 @@ +"""Dekoduje encje HTML w tytulach scen i filmow zapisanych przed poprawka normalizera. + +Tuby (glownie xvideos, JSON-LD) oddawaly tytuly typu `we're [AI]`, +a my zapisywalismy je tak jak przyszly. `normalize` robil z tego slowa `apos`, `lpar`, +wiec tytul nie trafial ani w wyszukiwarke, ani w kanon. Od poprawki `normalize_scene` +przepuszcza tytul przez `unescape_html`, ale istniejace wiersze trzeba poprawic osobno: +scraper tytulu juz ustawionego nie nadpisuje. + +Co zmienia na wierszu: + - `title` na wersje po `unescape_html`, + - `title_normalized` na `normalize(nowy tytul)`, tak jak liczy go ingest, + - `slug` tylko gdy byl wyliczony ze starego tytulu (`slugify(stary) == slug`); slug + ustawiony z innego zrodla zostaje. + +Zapis jest warunkowy (`WHERE title = stary`), wiec wiersz zmieniony w miedzyczasie przez +ingest albo merge jest pomijany. Ponowne odpalenie jest bezpieczne: bierze tylko tytuly, +w ktorych nadal jest encja. + +Uzycie: + python3 -u /srv/scripts/fix_html_entity_titles.py --dry-run + python3 -u /srv/scripts/fix_html_entity_titles.py +""" +from __future__ import annotations + +import argparse +import logging +import random +import sys + +import sqlalchemy as sa + +from app.db import session_scope +from app.normalize.text import normalize, slugify, unescape_html + +log = logging.getLogger("fix_html_entity_titles") + +# Ten sam zestaw co `_HTML_ENTITY_RE` w app/normalize/text.py, w skladni Postgresa. +_PG_ENTITY_RE = r"&(#[0-9]{1,7}|#[xX][0-9a-fA-F]{1,6}|[A-Za-z][A-Za-z0-9]{1,31});" +_TABLES = ("scenes", "movies") + + +def _plan(table: str, *, limit: int | None) -> list[dict]: + with session_scope() as session: + rows = session.execute( + sa.text( + f"SELECT id, title, title_normalized, slug FROM {table} " + "WHERE title ~ :re" + (" LIMIT :limit" if limit else "") + ), + {"re": _PG_ENTITY_RE, "limit": limit}, + ).all() + + plan: list[dict] = [] + for row_id, title, title_norm, slug in rows: + new_title = unescape_html(title) + if not new_title or new_title == title: + continue + new_slug = slug + if slug is None or slug == slugify(title): + new_slug = slugify(new_title) or slug + plan.append( + { + "id": row_id, + "old_title": title, + "title": new_title, + "old_title_normalized": title_norm, + "title_normalized": normalize(new_title), + "old_slug": slug, + "slug": new_slug, + } + ) + log.info("%s: %d wierszy z encja, %d do zmiany", table, len(rows), len(plan)) + return plan + + +def _apply(table: str, plan: list[dict], *, batch_size: int) -> int: + # Jedno UPDATE na partie przez unnest: rowcount jest wtedy wiarygodny. + stmt = sa.text( + f"UPDATE {table} t SET title = v.title, title_normalized = v.title_normalized, slug = v.slug " + "FROM unnest(CAST(:ids AS uuid[]), CAST(:old_titles AS text[]), CAST(:titles AS text[]), " + "CAST(:norms AS text[]), CAST(:slugs AS text[])) " + "AS v(id, old_title, title, title_normalized, slug) " + "WHERE t.id = v.id AND t.title = v.old_title" + ) + updated = 0 + for start in range(0, len(plan), batch_size): + batch = plan[start : start + batch_size] + params = { + "ids": [str(item["id"]) for item in batch], + "old_titles": [item["old_title"] for item in batch], + "titles": [item["title"] for item in batch], + "norms": [item["title_normalized"] for item in batch], + "slugs": [item["slug"] for item in batch], + } + with session_scope() as session: + updated += session.execute(stmt, params).rowcount + log.info("%s: %d/%d przejrzane, %d zapisane", table, start + len(batch), len(plan), updated) + return updated + + +def main() -> int: + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--dry-run", action="store_true", help="tylko policz i pokaz probke") + p.add_argument("--batch-size", type=int, default=500) + p.add_argument("--sample", type=int, default=20, help="ile losowych zmian pokazac") + p.add_argument("--limit", type=int, default=None, help="max wierszy na tabele (test)") + args = p.parse_args() + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") + + total_planned = total_updated = 0 + for table in _TABLES: + plan = _plan(table, limit=args.limit) + if not plan: + continue + total_planned += len(plan) + slug_changes = sum(1 for item in plan if item["slug"] != item["old_slug"]) + log.info("%s: slug zmieniony w %d z %d", table, slug_changes, len(plan)) + for item in random.sample(plan, min(args.sample, len(plan))): + print(f"[{table}] {item['id']}") + print(f" title: {item['old_title']!r}") + print(f" -> {item['title']!r}") + print(f" norm: {item['old_title_normalized']!r}") + print(f" -> {item['title_normalized']!r}") + if item["slug"] != item["old_slug"]: + print(f" slug: {item['old_slug']!r}") + print(f" -> {item['slug']!r}") + sys.stdout.flush() + if not args.dry_run: + total_updated += _apply(table, plan, batch_size=args.batch_size) + + if args.dry_run: + log.info("DRY RUN: %d wierszy do zmiany, nic nie zapisane", total_planned) + else: + log.info("zapisane %d z %d (reszta zmieniona w miedzyczasie)", total_updated, total_planned) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_normalize.py b/tests/test_normalize.py index 009dd3f..fea0939 100644 --- a/tests/test_normalize.py +++ b/tests/test_normalize.py @@ -6,6 +6,7 @@ from app.normalize.text import ( person_named_in, slugify, strip_accents, + unescape_html, ) @@ -81,3 +82,28 @@ def test_strip_accents_idempotent() -> None: ) def test_person_named_in(title: str, person: str, expected: bool) -> None: assert person_named_in(title, person) is expected + + +@pytest.mark.parametrize( + ("raw", "expected"), + [ + # xvideos JSON-LD, encje HTML5. + ( + "Are you sure we're allowed to do this mister Steve?? [AI]", + "Are you sure we're allowed to do this mister Steve?? [AI]", + ), + ("The BEST (( HOT WIFE ))", "The BEST (( HOT WIFE ))"), + # Podwojne kodowanie. + ("Barbara & Jake (4K)", "Barbara & Jake (4K)"), + ("'x' "q"", "'x' \"q\""), + # .title() na zakodowanym tytule. + ("Alix &Amp; Cherie", "Alix & Cherie"), + (" Hot Wife", "Hot Wife"), + # Bez srednika albo nieznana nazwa: zostaje jak bylo. + ("Me¬ice Tom & Jerry ©right; Q&A", "Me¬ice Tom & Jerry ©right; Q&A"), + ("", ""), + (None, ""), + ], +) +def test_unescape_html(raw: str | None, expected: str) -> None: + assert unescape_html(raw) == expected diff --git a/tests/test_normalize_scenes.py b/tests/test_normalize_scenes.py index f565c74..68f847d 100644 --- a/tests/test_normalize_scenes.py +++ b/tests/test_normalize_scenes.py @@ -6,6 +6,7 @@ from pathlib import Path import pytest +from app.connectors.base import RawScene from app.connectors.tpdb import _parse_scene from app.normalize.scenes import normalize_performer, normalize_scene, normalize_studio from app.normalize.text import normalize, normalize_person @@ -58,3 +59,11 @@ def test_normalize_scene_preserves_performer_order(tpdb_scene_raw: dict) -> None assert raw is not None norm = normalize_scene(raw) assert [p.canonical_name for p in norm.performers] == ["Mia Malkova", "Johnny Sins"] + + +def test_normalize_scene_unescapes_html_entities_in_title() -> None: + raw = RawScene(external_id="xvideoscom:x", title="Mama's boy (4K)") + norm = normalize_scene(raw) + assert norm.title == "Mama's boy (4K)" + assert norm.title_normalized == normalize("Mama's boy (4K)") + assert norm.slug == "mama-s-boy-4k"