diff --git a/app/normalize/movies.py b/app/normalize/movies.py index 1f4abb7..5dc01ca 100644 --- a/app/normalize/movies.py +++ b/app/normalize/movies.py @@ -13,7 +13,7 @@ from app.normalize.scenes import ( normalize_studio, normalize_tag, ) -from app.normalize.text import normalize, slugify, unescape_html +from app.normalize.text import clean_description, normalize, slugify, unescape_html @dataclass @@ -50,7 +50,7 @@ def normalize_movie(raw: RawMovie) -> NormalizedMovie: slug=slugify(title), release_year=raw.release_year, release_date=raw.release_date, - description=raw.description, + description=clean_description(raw.description), duration_sec=raw.duration_sec, director=raw.director, country=raw.country, diff --git a/app/normalize/scenes.py b/app/normalize/scenes.py index d157616..affcc0e 100644 --- a/app/normalize/scenes.py +++ b/app/normalize/scenes.py @@ -16,7 +16,13 @@ from app.connectors.base import ( RawTag, ) from app.normalize.tag_inference import infer_tag_slugs -from app.normalize.text import normalize, normalize_person, slugify, unescape_html +from app.normalize.text import ( + clean_description, + normalize, + normalize_person, + slugify, + unescape_html, +) @dataclass @@ -194,7 +200,7 @@ def normalize_scene(raw: RawScene) -> NormalizedScene: title_normalized=normalize(title), slug=slugify(title), release_date=_sane_release_date(raw.release_date), - description=raw.description, + description=clean_description(raw.description), duration_sec=raw.duration_sec, code=raw.code, director=raw.director, diff --git a/app/normalize/text.py b/app/normalize/text.py index 9742566..91c5f4f 100644 --- a/app/normalize/text.py +++ b/app/normalize/text.py @@ -62,14 +62,47 @@ def unescape_html(value: str | None) -> str: """ if not value: return "" - text = value + return _decode_entities(value).strip() + + +def _decode_entities(text: str) -> str: for _ in range(_UNESCAPE_MAX_PASSES): new = _HTML_ENTITY_RE.sub(_decode_entity, text) if new == text: break text = new - # ` ` i `` (BOM) zostawiaja niewidoczne znaki, w tytule sa zbedne. - return text.replace("\xa0", " ").replace("", "").strip() + # ` ` i `` (BOM) zostawiaja niewidoczne znaki, w tekscie sa zbedne. + return text.replace("\N{NO-BREAK SPACE}", " ").replace("\N{ZERO WIDTH NO-BREAK SPACE}", "") + + +# Znaczniki, ktore tuby zostawiaja w opisach, surowe albo jako `<br>`. Lista nazw +# jest zamknieta, a atrybuty musza miec postac `nazwa=wartosc`, bo `<3`, `` w opisach to tekst, nie znacznik. +_BREAK_TAG_RE = re.compile(r"|", re.IGNORECASE) +_INLINE_TAG_RE = re.compile( + r"]+))*\s*/?>""", + re.IGNORECASE, +) +_TRAILING_WS_RE = re.compile(r"[ \t]+\n") +_MANY_NEWLINES_RE = re.compile(r"\n{3,}") + + +def clean_description(value: str | None) -> str | None: + """Opis ze scrapera jako zwykly tekst: encje odkodowane, znaczniki HTML usuniete. + + Apka pokazuje opis w `Text`, strona SEO przez autoescape, wiec `"` i `
` + widac doslownie. Encje ida pierwsze, bo czesc tubow koduje tez znaczniki + (`<br>`). `
` i koniec akapitu zamieniamy na nowa linie, reszte wycinamy. + """ + if not value: + return None + text = _decode_entities(value) + text = _BREAK_TAG_RE.sub("\n", text) + text = _INLINE_TAG_RE.sub("", text) + text = _TRAILING_WS_RE.sub("\n", text) + text = _MANY_NEWLINES_RE.sub("\n\n", text) + return text.strip() or None def strip_accents(value: str) -> str: diff --git a/scripts/bulk_rescrape_hqporner.py b/scripts/bulk_rescrape_hqporner.py index 5fc5b35..7471523 100644 --- a/scripts/bulk_rescrape_hqporner.py +++ b/scripts/bulk_rescrape_hqporner.py @@ -28,7 +28,7 @@ from app.extractors.thumb_extract import extract_thumbnail_url from app.models.playback_source import PlaybackSource from app.models.scene import Scene, SceneTag from app.normalize.scenes import NormalizedTag -from app.normalize.text import slugify +from app.normalize.text import normalize, slugify, unescape_html from app.resolve.tag_resolver import resolve_tag log = logging.getLogger(__name__) @@ -71,7 +71,7 @@ def rescrape_scene(session, scene_id: uuid.UUID, pb_id: uuid.UUID, page_url: str # Title, jeśli current ma triple space (slug-derived), zastąp z . if (tm := _TITLE_RE.search(html)): - new_title = _cleanup_title(tm.group(1)) + new_title = _cleanup_title(unescape_html(tm.group(1))) if new_title: scene = session.get(Scene, scene_id) if scene and (" " in scene.title or scene.title.strip() != new_title): @@ -79,7 +79,7 @@ def rescrape_scene(session, scene_id: uuid.UUID, pb_id: uuid.UUID, page_url: str # Bezpieczna heurystyka: nie nadpisuj jeśli current jest dłuższy. if " " in scene.title or len(new_title) > len(scene.title): scene.title = new_title - scene.title_normalized = new_title.lower() + scene.title_normalized = normalize(new_title) out["updated_title"] = 1 # Duration diff --git a/scripts/fix_html_entities.py b/scripts/fix_html_entities.py new file mode 100644 index 0000000..de051b6 --- /dev/null +++ b/scripts/fix_html_entities.py @@ -0,0 +1,211 @@ +"""Czysci encje i znaczniki HTML w tytulach i opisach scen i filmow sprzed poprawki normalizera. + +Tuby (glownie xvideos, JSON-LD) oddawaly tytuly typu `we're [AI]`, a opisy +z `"` i `<br>`. Zapisywalismy je tak jak przyszly, wiec `normalize` robil z tytulu +slowa `apos`, `lpar`, a w apce i na stronie SEO encje widac doslownie. Od poprawki +`normalize_scene` przepuszcza tytul przez `unescape_html`, a opis przez `clean_description`, +ale istniejace wiersze trzeba poprawic osobno: scraper pola juz ustawionego nie nadpisuje. + +Przebieg `titles` zmienia na wierszu: + - `title` na wersje po `unescape_html`, + - `title_normalized` na `normalize(nowy tytul)`, tak jak liczy go ingest, + - `slug` tylko gdy byl wyliczony ze starego tytulu (`slugify(stary) == slug`). +Przebieg `descriptions` zmienia `description` na wersje po `clean_description`. + +Zapis jest warunkowy (`WHERE pole = stara wartosc`), wiec wiersz zmieniony w miedzyczasie +przez ingest albo merge jest pomijany. Ponowne odpalenie jest bezpieczne: bierze tylko +wiersze, w ktorych nadal jest encja albo znacznik. + +Uzycie: + python3 -u /srv/scripts/fix_html_entities.py --dry-run + python3 -u /srv/scripts/fix_html_entities.py --only descriptions +""" +from __future__ import annotations + +import argparse +import logging +import random +import sys + +import sqlalchemy as sa + +from app.db import session_scope +from app.normalize.text import clean_description, normalize, slugify, unescape_html + +log = logging.getLogger("fix_html_entities") + +# Ten sam zestaw co `_HTML_ENTITY_RE` w app/normalize/text.py, w skladni Postgresa. +_PG_ENTITY_RE = r"&(#[0-9]{1,7}|#[xX][0-9a-fA-F]{1,6}|[A-Za-z][A-Za-z0-9]{1,31});" +# Wstepny filtr na znaczniki z `_BREAK_TAG_RE`/`_INLINE_TAG_RE`; decyduje `clean_description`. +_PG_TAG_RE = ( + r"</?(a|b|i|u|em|strong|span|div|p|font|small|big|sup|sub|h[1-6]|ul|ol|li|center" + r"|blockquote|br)([[:space:]/>])" +) +_TABLES = ("scenes", "movies") + + +def _plan_titles(table: str, *, limit: int | None) -> list[dict]: + with session_scope() as session: + rows = session.execute( + sa.text( + f"SELECT id, title, title_normalized, slug FROM {table} " + "WHERE title ~ :re" + (" LIMIT :limit" if limit else "") + ), + {"re": _PG_ENTITY_RE, "limit": limit}, + ).all() + + plan: list[dict] = [] + for row_id, title, title_norm, slug in rows: + new_title = unescape_html(title) + if not new_title or new_title == title: + continue + new_slug = slug + if slug is None or slug == slugify(title): + new_slug = slugify(new_title) or slug + plan.append( + { + "id": row_id, + "old_title": title, + "title": new_title, + "old_title_normalized": title_norm, + "title_normalized": normalize(new_title), + "old_slug": slug, + "slug": new_slug, + } + ) + log.info("%s.title: %d wierszy z encja, %d do zmiany", table, len(rows), len(plan)) + return plan + + +def _plan_descriptions(table: str, *, limit: int | None) -> list[dict]: + with session_scope() as session: + rows = session.execute( + sa.text( + f"SELECT id, description FROM {table} " + "WHERE description ~ :ent OR description ~* :tag" + (" LIMIT :limit" if limit else "") + ), + {"ent": _PG_ENTITY_RE, "tag": _PG_TAG_RE, "limit": limit}, + ).all() + + plan = [ + {"id": row_id, "old_description": desc, "description": new} + for row_id, desc in rows + if (new := clean_description(desc)) != desc + ] + log.info("%s.description: %d wierszy z encja lub znacznikiem, %d do zmiany", table, len(rows), len(plan)) + return plan + + +def _apply_titles(table: str, plan: list[dict], *, batch_size: int) -> int: + # Jedno UPDATE na partie przez unnest: rowcount jest wtedy wiarygodny. + stmt = sa.text( + f"UPDATE {table} t SET title = v.title, title_normalized = v.title_normalized, slug = v.slug " + "FROM unnest(CAST(:ids AS uuid[]), CAST(:old_titles AS text[]), CAST(:titles AS text[]), " + "CAST(:norms AS text[]), CAST(:slugs AS text[])) " + "AS v(id, old_title, title, title_normalized, slug) " + "WHERE t.id = v.id AND t.title = v.old_title" + ) + return _run_batches( + f"{table}.title", + stmt, + plan, + batch_size=batch_size, + columns={"ids": "id", "old_titles": "old_title", "titles": "title", "norms": "title_normalized", "slugs": "slug"}, + ) + + +def _apply_descriptions(table: str, plan: list[dict], *, batch_size: int) -> int: + stmt = sa.text( + f"UPDATE {table} t SET description = v.description " + "FROM unnest(CAST(:ids AS uuid[]), CAST(:old AS text[]), CAST(:new AS text[])) " + "AS v(id, old_description, description) " + "WHERE t.id = v.id AND t.description = v.old_description" + ) + return _run_batches( + f"{table}.description", + stmt, + plan, + batch_size=batch_size, + columns={"ids": "id", "old": "old_description", "new": "description"}, + ) + + +def _run_batches(label: str, stmt, plan: list[dict], *, batch_size: int, columns: dict[str, str]) -> int: + updated = 0 + for start in range(0, len(plan), batch_size): + batch = plan[start : start + batch_size] + params = { + param: [str(item[key]) if key == "id" else item[key] for item in batch] + for param, key in columns.items() + } + with session_scope() as session: + updated += session.execute(stmt, params).rowcount + log.info("%s: %d/%d przejrzane, %d zapisane", label, start + len(batch), len(plan), updated) + return updated + + +def _diff_window(old: str, new: str | None, width: int = 90) -> tuple[str, str]: + """Wycinek wokol pierwszej roznicy, bo opisy sa dlugie, a zmiana bywa w srodku.""" + new = new or "" + i = next((k for k, (a, b) in enumerate(zip(old, new)) if a != b), min(len(old), len(new))) + start = max(0, i - width // 3) + return old[start : start + width], new[start : start + width] + + +def _print_title_sample(table: str, plan: list[dict], n: int) -> None: + for item in random.sample(plan, min(n, len(plan))): + print(f"[{table}.title] {item['id']}") + print(f" title: {item['old_title']!r}") + print(f" -> {item['title']!r}") + print(f" norm: {item['old_title_normalized']!r}") + print(f" -> {item['title_normalized']!r}") + if item["slug"] != item["old_slug"]: + print(f" slug: {item['old_slug']!r}") + print(f" -> {item['slug']!r}") + + +def _print_description_sample(table: str, plan: list[dict], n: int) -> None: + for item in random.sample(plan, min(n, len(plan))): + old, new = _diff_window(item["old_description"], item["description"]) + print(f"[{table}.description] {item['id']}") + print(f" ...{old!r}") + print(f" ...{new!r}") + + +def main() -> int: + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--dry-run", action="store_true", help="tylko policz i pokaz probke") + p.add_argument("--only", choices=("titles", "descriptions"), default=None) + p.add_argument("--batch-size", type=int, default=500) + p.add_argument("--sample", type=int, default=20, help="ile losowych zmian pokazac na przebieg") + p.add_argument("--limit", type=int, default=None, help="max wierszy na tabele (test)") + args = p.parse_args() + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") + + passes = [] + if args.only in (None, "titles"): + passes.append((_plan_titles, _print_title_sample, _apply_titles)) + if args.only in (None, "descriptions"): + passes.append((_plan_descriptions, _print_description_sample, _apply_descriptions)) + + total_planned = total_updated = 0 + for plan_fn, sample_fn, apply_fn in passes: + for table in _TABLES: + plan = plan_fn(table, limit=args.limit) + if not plan: + continue + total_planned += len(plan) + sample_fn(table, plan, args.sample) + sys.stdout.flush() + if not args.dry_run: + total_updated += apply_fn(table, plan, batch_size=args.batch_size) + + if args.dry_run: + log.info("DRY RUN: %d zmian w planie, nic nie zapisane", total_planned) + else: + log.info("zapisane %d z %d (reszta zmieniona w miedzyczasie)", total_updated, total_planned) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/fix_html_entity_titles.py b/scripts/fix_html_entity_titles.py deleted file mode 100644 index a45c4ac..0000000 --- a/scripts/fix_html_entity_titles.py +++ /dev/null @@ -1,138 +0,0 @@ -"""Dekoduje encje HTML w tytulach scen i filmow zapisanych przed poprawka normalizera. - -Tuby (glownie xvideos, JSON-LD) oddawaly tytuly typu `we're [AI]`, -a my zapisywalismy je tak jak przyszly. `normalize` robil z tego slowa `apos`, `lpar`, -wiec tytul nie trafial ani w wyszukiwarke, ani w kanon. Od poprawki `normalize_scene` -przepuszcza tytul przez `unescape_html`, ale istniejace wiersze trzeba poprawic osobno: -scraper tytulu juz ustawionego nie nadpisuje. - -Co zmienia na wierszu: - - `title` na wersje po `unescape_html`, - - `title_normalized` na `normalize(nowy tytul)`, tak jak liczy go ingest, - - `slug` tylko gdy byl wyliczony ze starego tytulu (`slugify(stary) == slug`); slug - ustawiony z innego zrodla zostaje. - -Zapis jest warunkowy (`WHERE title = stary`), wiec wiersz zmieniony w miedzyczasie przez -ingest albo merge jest pomijany. Ponowne odpalenie jest bezpieczne: bierze tylko tytuly, -w ktorych nadal jest encja. - -Uzycie: - python3 -u /srv/scripts/fix_html_entity_titles.py --dry-run - python3 -u /srv/scripts/fix_html_entity_titles.py -""" -from __future__ import annotations - -import argparse -import logging -import random -import sys - -import sqlalchemy as sa - -from app.db import session_scope -from app.normalize.text import normalize, slugify, unescape_html - -log = logging.getLogger("fix_html_entity_titles") - -# Ten sam zestaw co `_HTML_ENTITY_RE` w app/normalize/text.py, w skladni Postgresa. -_PG_ENTITY_RE = r"&(#[0-9]{1,7}|#[xX][0-9a-fA-F]{1,6}|[A-Za-z][A-Za-z0-9]{1,31});" -_TABLES = ("scenes", "movies") - - -def _plan(table: str, *, limit: int | None) -> list[dict]: - with session_scope() as session: - rows = session.execute( - sa.text( - f"SELECT id, title, title_normalized, slug FROM {table} " - "WHERE title ~ :re" + (" LIMIT :limit" if limit else "") - ), - {"re": _PG_ENTITY_RE, "limit": limit}, - ).all() - - plan: list[dict] = [] - for row_id, title, title_norm, slug in rows: - new_title = unescape_html(title) - if not new_title or new_title == title: - continue - new_slug = slug - if slug is None or slug == slugify(title): - new_slug = slugify(new_title) or slug - plan.append( - { - "id": row_id, - "old_title": title, - "title": new_title, - "old_title_normalized": title_norm, - "title_normalized": normalize(new_title), - "old_slug": slug, - "slug": new_slug, - } - ) - log.info("%s: %d wierszy z encja, %d do zmiany", table, len(rows), len(plan)) - return plan - - -def _apply(table: str, plan: list[dict], *, batch_size: int) -> int: - # Jedno UPDATE na partie przez unnest: rowcount jest wtedy wiarygodny. - stmt = sa.text( - f"UPDATE {table} t SET title = v.title, title_normalized = v.title_normalized, slug = v.slug " - "FROM unnest(CAST(:ids AS uuid[]), CAST(:old_titles AS text[]), CAST(:titles AS text[]), " - "CAST(:norms AS text[]), CAST(:slugs AS text[])) " - "AS v(id, old_title, title, title_normalized, slug) " - "WHERE t.id = v.id AND t.title = v.old_title" - ) - updated = 0 - for start in range(0, len(plan), batch_size): - batch = plan[start : start + batch_size] - params = { - "ids": [str(item["id"]) for item in batch], - "old_titles": [item["old_title"] for item in batch], - "titles": [item["title"] for item in batch], - "norms": [item["title_normalized"] for item in batch], - "slugs": [item["slug"] for item in batch], - } - with session_scope() as session: - updated += session.execute(stmt, params).rowcount - log.info("%s: %d/%d przejrzane, %d zapisane", table, start + len(batch), len(plan), updated) - return updated - - -def main() -> int: - p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - p.add_argument("--dry-run", action="store_true", help="tylko policz i pokaz probke") - p.add_argument("--batch-size", type=int, default=500) - p.add_argument("--sample", type=int, default=20, help="ile losowych zmian pokazac") - p.add_argument("--limit", type=int, default=None, help="max wierszy na tabele (test)") - args = p.parse_args() - logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") - - total_planned = total_updated = 0 - for table in _TABLES: - plan = _plan(table, limit=args.limit) - if not plan: - continue - total_planned += len(plan) - slug_changes = sum(1 for item in plan if item["slug"] != item["old_slug"]) - log.info("%s: slug zmieniony w %d z %d", table, slug_changes, len(plan)) - for item in random.sample(plan, min(args.sample, len(plan))): - print(f"[{table}] {item['id']}") - print(f" title: {item['old_title']!r}") - print(f" -> {item['title']!r}") - print(f" norm: {item['old_title_normalized']!r}") - print(f" -> {item['title_normalized']!r}") - if item["slug"] != item["old_slug"]: - print(f" slug: {item['old_slug']!r}") - print(f" -> {item['slug']!r}") - sys.stdout.flush() - if not args.dry_run: - total_updated += _apply(table, plan, batch_size=args.batch_size) - - if args.dry_run: - log.info("DRY RUN: %d wierszy do zmiany, nic nie zapisane", total_planned) - else: - log.info("zapisane %d z %d (reszta zmieniona w miedzyczasie)", total_updated, total_planned) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/fix_title_normalized.py b/scripts/fix_title_normalized.py new file mode 100644 index 0000000..cb801b8 --- /dev/null +++ b/scripts/fix_title_normalized.py @@ -0,0 +1,120 @@ +"""Przelicza `title_normalized` scen i filmow tam, gdzie rozjechal sie z `normalize(title)`. + +Ingest liczy `title_normalized = normalize(title)`, ale stare skrypty naprawcze +(`bulk_rescrape_hqporner.py`, `repair_truncated_titles.py`) zapisywaly samo `lower(title)`. +Takie sceny maja w kluczu interpunkcje (`cindy hope - horny boss`, `matty's virgin`), wiec +nie trafiaja w dokladne dopasowanie tytulu w resolverze ani w dedup. Pelny skan z 2026-09-15: +6005 scen, wszystkie w formie `lower(title)`, innych rozjazdow nie bylo. + +Skrypt czyta cala tabele strumieniowo (4,3 mln scen to ok. 1,5 minuty), zbiera wiersze, +w ktorych klucz sie nie zgadza, i zapisuje partiami. Zapis jest warunkowy +(`WHERE title = stary AND title_normalized = stary`), wiec wiersz zmieniony w miedzyczasie +jest pomijany. `title` i `slug` zostaja bez zmian. + +Uzycie: + python3 -u /srv/scripts/fix_title_normalized.py --dry-run + python3 -u /srv/scripts/fix_title_normalized.py +""" +from __future__ import annotations + +import argparse +import collections +import logging +import random +import sys +import time + +import sqlalchemy as sa + +from app.db import engine, session_scope +from app.normalize.text import normalize + +log = logging.getLogger("fix_title_normalized") + +_TABLES = ("scenes", "movies") + + +def _plan(table: str) -> tuple[list[dict], collections.Counter]: + t0 = time.monotonic() + plan: list[dict] = [] + kinds: collections.Counter = collections.Counter() + scanned = 0 + with engine.connect().execution_options(stream_results=True, yield_per=20000) as conn: + for row_id, title, title_norm in conn.execute(sa.text(f"SELECT id, title, title_normalized FROM {table}")): + scanned += 1 + new = normalize(title) + if new == title_norm: + continue + kinds["lower" if title_norm == (title or "").lower() else "inne"] += 1 + plan.append({"id": row_id, "title": title, "old": title_norm, "new": new}) + log.info( + "%s: przejrzane %d w %ds, do zmiany %d %s", + table, scanned, time.monotonic() - t0, len(plan), dict(kinds), + ) + return plan, kinds + + +def _apply(table: str, plan: list[dict], *, batch_size: int) -> int: + stmt = sa.text( + f"UPDATE {table} t SET title_normalized = v.new " + "FROM unnest(CAST(:ids AS uuid[]), CAST(:titles AS text[]), CAST(:olds AS text[]), " + "CAST(:news AS text[])) AS v(id, title, old, new) " + "WHERE t.id = v.id AND t.title = v.title AND t.title_normalized = v.old" + ) + updated = 0 + for start in range(0, len(plan), batch_size): + batch = plan[start : start + batch_size] + params = { + "ids": [str(item["id"]) for item in batch], + "titles": [item["title"] for item in batch], + "olds": [item["old"] for item in batch], + "news": [item["new"] for item in batch], + } + with session_scope() as session: + updated += session.execute(stmt, params).rowcount + log.info("%s: %d/%d przejrzane, %d zapisane", table, start + len(batch), len(plan), updated) + return updated + + +def main() -> int: + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--dry-run", action="store_true", help="tylko policz i pokaz probke") + p.add_argument("--batch-size", type=int, default=500) + p.add_argument("--sample", type=int, default=20, help="ile losowych zmian pokazac na tabele") + p.add_argument( + "--only-lower", + action="store_true", + help="zapisz tylko rozjazdy w formie lower(title), inne tylko pokaz", + ) + args = p.parse_args() + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") + + total_planned = total_updated = 0 + for table in _TABLES: + plan, kinds = _plan(table) + if kinds.get("inne"): + others = [item for item in plan if item["old"] != (item["title"] or "").lower()] + for item in random.sample(others, min(args.sample, len(others))): + print(f"[{table} inne] {item['title']!r}: {item['old']!r} -> {item['new']!r}") + if args.only_lower: + plan = [item for item in plan if item["old"] == (item["title"] or "").lower()] + if not plan: + continue + for item in random.sample(plan, min(args.sample, len(plan))): + print(f"[{table}] {item['title']!r}") + print(f" {item['old']!r}") + print(f" -> {item['new']!r}") + sys.stdout.flush() + total_planned += len(plan) + if not args.dry_run: + total_updated += _apply(table, plan, batch_size=args.batch_size) + + if args.dry_run: + log.info("DRY RUN: %d wierszy do zmiany, nic nie zapisane", total_planned) + else: + log.info("zapisane %d z %d (reszta zmieniona w miedzyczasie)", total_updated, total_planned) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/repair_truncated_titles.py b/scripts/repair_truncated_titles.py index 85bcfbb..7d9a815 100644 --- a/scripts/repair_truncated_titles.py +++ b/scripts/repair_truncated_titles.py @@ -18,6 +18,7 @@ import sqlalchemy as sa from app.connectors.direct_scrapers._browse_base import meta_content from app.db import session_scope +from app.normalize.text import normalize, unescape_html from app.extractors._fetch import browser_get log = logging.getLogger(__name__) @@ -59,7 +60,7 @@ def main() -> int: if r.status_code >= 400: continue - new_title = meta_content(r.text, property="og:title") + new_title = unescape_html(meta_content(r.text, property="og:title")) if not new_title: continue # Only update if longer (i.e. parser found full title after fix) @@ -70,9 +71,9 @@ def main() -> int: with session_scope() as session: session.execute( sa.text( - "UPDATE scenes SET title = :t, title_normalized = LOWER(:t) WHERE id = :sid" + "UPDATE scenes SET title = :t, title_normalized = :tn WHERE id = :sid" ), - {"t": new_title.strip(), "sid": scene_id}, + {"t": new_title, "tn": normalize(new_title), "sid": scene_id}, ) counters["updated"] += 1 if i % 25 == 0: diff --git a/tests/test_normalize.py b/tests/test_normalize.py index fea0939..2d20000 100644 --- a/tests/test_normalize.py +++ b/tests/test_normalize.py @@ -1,6 +1,7 @@ import pytest from app.normalize.text import ( + clean_description, normalize, normalize_person, person_named_in, @@ -107,3 +108,28 @@ def test_person_named_in(title: str, person: str, expected: bool) -> None: ) def test_unescape_html(raw: str | None, expected: str) -> None: assert unescape_html(raw) == expected + + +@pytest.mark.parametrize( + ("raw", "expected"), + [ + ('Manny in "MANUEL CREAMPIES 2". She loved it.', 'Manny in "MANUEL CREAMPIES 2". She loved it.'), + # Znaczniki zakodowane jako encje. + ("Is it love?<br>Eveline only ordered", "Is it love?\nEveline only ordered"), + ("door.</br></br>Jessica", "door.\n\nJessica"), + ("wash.<br><br><br><br>She", "wash.\n\nShe"), + # Surowe znaczniki. + ("Welcome, Carlo!<br /> ", "Welcome, Carlo!"), + ('<p>See <a href="/x">Bunny</a> & <b>more</b>!</p>', "See Bunny & more!"), + # To nie sa znaczniki. + ("I miss you so much <3.", "I miss you so much <3."), + ("It was <a lot of fun> and <b careful>", "It was <a lot of fun> and <b careful>"), + ('<span style="color: red" class=x>Hot</span>', "Hot"), + ("<Yokomiya-san. SSSP! No Cuts]", "<Yokomiya-san. SSSP! No Cuts]"), + ("lips&tongues; Alix &Amp; Cherie", "lips&tongues; Alix & Cherie"), + (" ", None), + (None, None), + ], +) +def test_clean_description(raw: str | None, expected: str | None) -> None: + assert clean_description(raw) == expected