fix(ingest): czysc opisy z encji i znacznikow HTML, title_normalized zawsze przez normalize
Opisy scen szly do bazy z `"`, `<br>` i surowymi `<br />`, a apka i strona SEO pokazuja opis jako tekst, wiec bylo to widac doslownie. clean_description dekoduje encje, `<br>` i koniec akapitu zamienia na nowa linie, a znane znaczniki wycina (`<3` zostaje tekstem). Wpiete w normalize_scene i normalize_movie. bulk_rescrape_hqporner.py i repair_truncated_titles.py liczyly title_normalized jako lower(title), przez co 6005 scen mialo interpunkcje w kluczu dopasowania. Oba licza teraz normalize() i dekoduja encje w tytule. Skrypty: fix_html_entity_titles.py przemianowany na fix_html_entities.py z przebiegiem po opisach; nowy fix_title_normalized.py przelicza klucz tam, gdzie rozjechal sie z normalize(title). Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
goon-foss
parent
d471bb3293
commit
d2b3378d93
9 changed files with 410 additions and 151 deletions
|
|
@ -13,7 +13,7 @@ from app.normalize.scenes import (
|
|||
normalize_studio,
|
||||
normalize_tag,
|
||||
)
|
||||
from app.normalize.text import normalize, slugify, unescape_html
|
||||
from app.normalize.text import clean_description, normalize, slugify, unescape_html
|
||||
|
||||
|
||||
@dataclass
|
||||
|
|
@ -50,7 +50,7 @@ def normalize_movie(raw: RawMovie) -> NormalizedMovie:
|
|||
slug=slugify(title),
|
||||
release_year=raw.release_year,
|
||||
release_date=raw.release_date,
|
||||
description=raw.description,
|
||||
description=clean_description(raw.description),
|
||||
duration_sec=raw.duration_sec,
|
||||
director=raw.director,
|
||||
country=raw.country,
|
||||
|
|
|
|||
|
|
@ -16,7 +16,13 @@ from app.connectors.base import (
|
|||
RawTag,
|
||||
)
|
||||
from app.normalize.tag_inference import infer_tag_slugs
|
||||
from app.normalize.text import normalize, normalize_person, slugify, unescape_html
|
||||
from app.normalize.text import (
|
||||
clean_description,
|
||||
normalize,
|
||||
normalize_person,
|
||||
slugify,
|
||||
unescape_html,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
|
|
@ -194,7 +200,7 @@ def normalize_scene(raw: RawScene) -> NormalizedScene:
|
|||
title_normalized=normalize(title),
|
||||
slug=slugify(title),
|
||||
release_date=_sane_release_date(raw.release_date),
|
||||
description=raw.description,
|
||||
description=clean_description(raw.description),
|
||||
duration_sec=raw.duration_sec,
|
||||
code=raw.code,
|
||||
director=raw.director,
|
||||
|
|
|
|||
|
|
@ -62,14 +62,47 @@ def unescape_html(value: str | None) -> str:
|
|||
"""
|
||||
if not value:
|
||||
return ""
|
||||
text = value
|
||||
return _decode_entities(value).strip()
|
||||
|
||||
|
||||
def _decode_entities(text: str) -> str:
|
||||
for _ in range(_UNESCAPE_MAX_PASSES):
|
||||
new = _HTML_ENTITY_RE.sub(_decode_entity, text)
|
||||
if new == text:
|
||||
break
|
||||
text = new
|
||||
# ` ` i `` (BOM) zostawiaja niewidoczne znaki, w tytule sa zbedne.
|
||||
return text.replace("\xa0", " ").replace("", "").strip()
|
||||
# ` ` i `` (BOM) zostawiaja niewidoczne znaki, w tekscie sa zbedne.
|
||||
return text.replace("\N{NO-BREAK SPACE}", " ").replace("\N{ZERO WIDTH NO-BREAK SPACE}", "")
|
||||
|
||||
|
||||
# Znaczniki, ktore tuby zostawiaja w opisach, surowe albo jako `<br>`. Lista nazw
|
||||
# jest zamknieta, a atrybuty musza miec postac `nazwa=wartosc`, bo `<3`, `<Yokomiya-san`
|
||||
# czy `<a lot of fun>` w opisach to tekst, nie znacznik.
|
||||
_BREAK_TAG_RE = re.compile(r"<br\s*/?>|</(?:br|p|div|li|h[1-6])\s*>", re.IGNORECASE)
|
||||
_INLINE_TAG_RE = re.compile(
|
||||
r"</?(?:a|b|i|u|em|strong|span|div|p|font|small|big|sup|sub|h[1-6]|ul|ol|li|center|blockquote)"
|
||||
r"""(?:\s+[\w:-]+\s*=\s*(?:"[^"]*"|'[^']*'|[^\s"'<>]+))*\s*/?>""",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_TRAILING_WS_RE = re.compile(r"[ \t]+\n")
|
||||
_MANY_NEWLINES_RE = re.compile(r"\n{3,}")
|
||||
|
||||
|
||||
def clean_description(value: str | None) -> str | None:
|
||||
"""Opis ze scrapera jako zwykly tekst: encje odkodowane, znaczniki HTML usuniete.
|
||||
|
||||
Apka pokazuje opis w `Text`, strona SEO przez autoescape, wiec `"` i `<br>`
|
||||
widac doslownie. Encje ida pierwsze, bo czesc tubow koduje tez znaczniki
|
||||
(`<br>`). `<br>` i koniec akapitu zamieniamy na nowa linie, reszte wycinamy.
|
||||
"""
|
||||
if not value:
|
||||
return None
|
||||
text = _decode_entities(value)
|
||||
text = _BREAK_TAG_RE.sub("\n", text)
|
||||
text = _INLINE_TAG_RE.sub("", text)
|
||||
text = _TRAILING_WS_RE.sub("\n", text)
|
||||
text = _MANY_NEWLINES_RE.sub("\n\n", text)
|
||||
return text.strip() or None
|
||||
|
||||
|
||||
def strip_accents(value: str) -> str:
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ from app.extractors.thumb_extract import extract_thumbnail_url
|
|||
from app.models.playback_source import PlaybackSource
|
||||
from app.models.scene import Scene, SceneTag
|
||||
from app.normalize.scenes import NormalizedTag
|
||||
from app.normalize.text import slugify
|
||||
from app.normalize.text import normalize, slugify, unescape_html
|
||||
from app.resolve.tag_resolver import resolve_tag
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
|
@ -71,7 +71,7 @@ def rescrape_scene(session, scene_id: uuid.UUID, pb_id: uuid.UUID, page_url: str
|
|||
|
||||
# Title, jeśli current ma triple space (slug-derived), zastąp z <title>.
|
||||
if (tm := _TITLE_RE.search(html)):
|
||||
new_title = _cleanup_title(tm.group(1))
|
||||
new_title = _cleanup_title(unescape_html(tm.group(1)))
|
||||
if new_title:
|
||||
scene = session.get(Scene, scene_id)
|
||||
if scene and (" " in scene.title or scene.title.strip() != new_title):
|
||||
|
|
@ -79,7 +79,7 @@ def rescrape_scene(session, scene_id: uuid.UUID, pb_id: uuid.UUID, page_url: str
|
|||
# Bezpieczna heurystyka: nie nadpisuj jeśli current jest dłuższy.
|
||||
if " " in scene.title or len(new_title) > len(scene.title):
|
||||
scene.title = new_title
|
||||
scene.title_normalized = new_title.lower()
|
||||
scene.title_normalized = normalize(new_title)
|
||||
out["updated_title"] = 1
|
||||
|
||||
# Duration
|
||||
|
|
|
|||
211
scripts/fix_html_entities.py
Normal file
211
scripts/fix_html_entities.py
Normal file
|
|
@ -0,0 +1,211 @@
|
|||
"""Czysci encje i znaczniki HTML w tytulach i opisach scen i filmow sprzed poprawki normalizera.
|
||||
|
||||
Tuby (glownie xvideos, JSON-LD) oddawaly tytuly typu `we're [AI]`, a opisy
|
||||
z `"` i `<br>`. Zapisywalismy je tak jak przyszly, wiec `normalize` robil z tytulu
|
||||
slowa `apos`, `lpar`, a w apce i na stronie SEO encje widac doslownie. Od poprawki
|
||||
`normalize_scene` przepuszcza tytul przez `unescape_html`, a opis przez `clean_description`,
|
||||
ale istniejace wiersze trzeba poprawic osobno: scraper pola juz ustawionego nie nadpisuje.
|
||||
|
||||
Przebieg `titles` zmienia na wierszu:
|
||||
- `title` na wersje po `unescape_html`,
|
||||
- `title_normalized` na `normalize(nowy tytul)`, tak jak liczy go ingest,
|
||||
- `slug` tylko gdy byl wyliczony ze starego tytulu (`slugify(stary) == slug`).
|
||||
Przebieg `descriptions` zmienia `description` na wersje po `clean_description`.
|
||||
|
||||
Zapis jest warunkowy (`WHERE pole = stara wartosc`), wiec wiersz zmieniony w miedzyczasie
|
||||
przez ingest albo merge jest pomijany. Ponowne odpalenie jest bezpieczne: bierze tylko
|
||||
wiersze, w ktorych nadal jest encja albo znacznik.
|
||||
|
||||
Uzycie:
|
||||
python3 -u /srv/scripts/fix_html_entities.py --dry-run
|
||||
python3 -u /srv/scripts/fix_html_entities.py --only descriptions
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import random
|
||||
import sys
|
||||
|
||||
import sqlalchemy as sa
|
||||
|
||||
from app.db import session_scope
|
||||
from app.normalize.text import clean_description, normalize, slugify, unescape_html
|
||||
|
||||
log = logging.getLogger("fix_html_entities")
|
||||
|
||||
# Ten sam zestaw co `_HTML_ENTITY_RE` w app/normalize/text.py, w skladni Postgresa.
|
||||
_PG_ENTITY_RE = r"&(#[0-9]{1,7}|#[xX][0-9a-fA-F]{1,6}|[A-Za-z][A-Za-z0-9]{1,31});"
|
||||
# Wstepny filtr na znaczniki z `_BREAK_TAG_RE`/`_INLINE_TAG_RE`; decyduje `clean_description`.
|
||||
_PG_TAG_RE = (
|
||||
r"</?(a|b|i|u|em|strong|span|div|p|font|small|big|sup|sub|h[1-6]|ul|ol|li|center"
|
||||
r"|blockquote|br)([[:space:]/>])"
|
||||
)
|
||||
_TABLES = ("scenes", "movies")
|
||||
|
||||
|
||||
def _plan_titles(table: str, *, limit: int | None) -> list[dict]:
|
||||
with session_scope() as session:
|
||||
rows = session.execute(
|
||||
sa.text(
|
||||
f"SELECT id, title, title_normalized, slug FROM {table} "
|
||||
"WHERE title ~ :re" + (" LIMIT :limit" if limit else "")
|
||||
),
|
||||
{"re": _PG_ENTITY_RE, "limit": limit},
|
||||
).all()
|
||||
|
||||
plan: list[dict] = []
|
||||
for row_id, title, title_norm, slug in rows:
|
||||
new_title = unescape_html(title)
|
||||
if not new_title or new_title == title:
|
||||
continue
|
||||
new_slug = slug
|
||||
if slug is None or slug == slugify(title):
|
||||
new_slug = slugify(new_title) or slug
|
||||
plan.append(
|
||||
{
|
||||
"id": row_id,
|
||||
"old_title": title,
|
||||
"title": new_title,
|
||||
"old_title_normalized": title_norm,
|
||||
"title_normalized": normalize(new_title),
|
||||
"old_slug": slug,
|
||||
"slug": new_slug,
|
||||
}
|
||||
)
|
||||
log.info("%s.title: %d wierszy z encja, %d do zmiany", table, len(rows), len(plan))
|
||||
return plan
|
||||
|
||||
|
||||
def _plan_descriptions(table: str, *, limit: int | None) -> list[dict]:
|
||||
with session_scope() as session:
|
||||
rows = session.execute(
|
||||
sa.text(
|
||||
f"SELECT id, description FROM {table} "
|
||||
"WHERE description ~ :ent OR description ~* :tag" + (" LIMIT :limit" if limit else "")
|
||||
),
|
||||
{"ent": _PG_ENTITY_RE, "tag": _PG_TAG_RE, "limit": limit},
|
||||
).all()
|
||||
|
||||
plan = [
|
||||
{"id": row_id, "old_description": desc, "description": new}
|
||||
for row_id, desc in rows
|
||||
if (new := clean_description(desc)) != desc
|
||||
]
|
||||
log.info("%s.description: %d wierszy z encja lub znacznikiem, %d do zmiany", table, len(rows), len(plan))
|
||||
return plan
|
||||
|
||||
|
||||
def _apply_titles(table: str, plan: list[dict], *, batch_size: int) -> int:
|
||||
# Jedno UPDATE na partie przez unnest: rowcount jest wtedy wiarygodny.
|
||||
stmt = sa.text(
|
||||
f"UPDATE {table} t SET title = v.title, title_normalized = v.title_normalized, slug = v.slug "
|
||||
"FROM unnest(CAST(:ids AS uuid[]), CAST(:old_titles AS text[]), CAST(:titles AS text[]), "
|
||||
"CAST(:norms AS text[]), CAST(:slugs AS text[])) "
|
||||
"AS v(id, old_title, title, title_normalized, slug) "
|
||||
"WHERE t.id = v.id AND t.title = v.old_title"
|
||||
)
|
||||
return _run_batches(
|
||||
f"{table}.title",
|
||||
stmt,
|
||||
plan,
|
||||
batch_size=batch_size,
|
||||
columns={"ids": "id", "old_titles": "old_title", "titles": "title", "norms": "title_normalized", "slugs": "slug"},
|
||||
)
|
||||
|
||||
|
||||
def _apply_descriptions(table: str, plan: list[dict], *, batch_size: int) -> int:
|
||||
stmt = sa.text(
|
||||
f"UPDATE {table} t SET description = v.description "
|
||||
"FROM unnest(CAST(:ids AS uuid[]), CAST(:old AS text[]), CAST(:new AS text[])) "
|
||||
"AS v(id, old_description, description) "
|
||||
"WHERE t.id = v.id AND t.description = v.old_description"
|
||||
)
|
||||
return _run_batches(
|
||||
f"{table}.description",
|
||||
stmt,
|
||||
plan,
|
||||
batch_size=batch_size,
|
||||
columns={"ids": "id", "old": "old_description", "new": "description"},
|
||||
)
|
||||
|
||||
|
||||
def _run_batches(label: str, stmt, plan: list[dict], *, batch_size: int, columns: dict[str, str]) -> int:
|
||||
updated = 0
|
||||
for start in range(0, len(plan), batch_size):
|
||||
batch = plan[start : start + batch_size]
|
||||
params = {
|
||||
param: [str(item[key]) if key == "id" else item[key] for item in batch]
|
||||
for param, key in columns.items()
|
||||
}
|
||||
with session_scope() as session:
|
||||
updated += session.execute(stmt, params).rowcount
|
||||
log.info("%s: %d/%d przejrzane, %d zapisane", label, start + len(batch), len(plan), updated)
|
||||
return updated
|
||||
|
||||
|
||||
def _diff_window(old: str, new: str | None, width: int = 90) -> tuple[str, str]:
|
||||
"""Wycinek wokol pierwszej roznicy, bo opisy sa dlugie, a zmiana bywa w srodku."""
|
||||
new = new or ""
|
||||
i = next((k for k, (a, b) in enumerate(zip(old, new)) if a != b), min(len(old), len(new)))
|
||||
start = max(0, i - width // 3)
|
||||
return old[start : start + width], new[start : start + width]
|
||||
|
||||
|
||||
def _print_title_sample(table: str, plan: list[dict], n: int) -> None:
|
||||
for item in random.sample(plan, min(n, len(plan))):
|
||||
print(f"[{table}.title] {item['id']}")
|
||||
print(f" title: {item['old_title']!r}")
|
||||
print(f" -> {item['title']!r}")
|
||||
print(f" norm: {item['old_title_normalized']!r}")
|
||||
print(f" -> {item['title_normalized']!r}")
|
||||
if item["slug"] != item["old_slug"]:
|
||||
print(f" slug: {item['old_slug']!r}")
|
||||
print(f" -> {item['slug']!r}")
|
||||
|
||||
|
||||
def _print_description_sample(table: str, plan: list[dict], n: int) -> None:
|
||||
for item in random.sample(plan, min(n, len(plan))):
|
||||
old, new = _diff_window(item["old_description"], item["description"])
|
||||
print(f"[{table}.description] {item['id']}")
|
||||
print(f" ...{old!r}")
|
||||
print(f" ...{new!r}")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
p.add_argument("--dry-run", action="store_true", help="tylko policz i pokaz probke")
|
||||
p.add_argument("--only", choices=("titles", "descriptions"), default=None)
|
||||
p.add_argument("--batch-size", type=int, default=500)
|
||||
p.add_argument("--sample", type=int, default=20, help="ile losowych zmian pokazac na przebieg")
|
||||
p.add_argument("--limit", type=int, default=None, help="max wierszy na tabele (test)")
|
||||
args = p.parse_args()
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||||
|
||||
passes = []
|
||||
if args.only in (None, "titles"):
|
||||
passes.append((_plan_titles, _print_title_sample, _apply_titles))
|
||||
if args.only in (None, "descriptions"):
|
||||
passes.append((_plan_descriptions, _print_description_sample, _apply_descriptions))
|
||||
|
||||
total_planned = total_updated = 0
|
||||
for plan_fn, sample_fn, apply_fn in passes:
|
||||
for table in _TABLES:
|
||||
plan = plan_fn(table, limit=args.limit)
|
||||
if not plan:
|
||||
continue
|
||||
total_planned += len(plan)
|
||||
sample_fn(table, plan, args.sample)
|
||||
sys.stdout.flush()
|
||||
if not args.dry_run:
|
||||
total_updated += apply_fn(table, plan, batch_size=args.batch_size)
|
||||
|
||||
if args.dry_run:
|
||||
log.info("DRY RUN: %d zmian w planie, nic nie zapisane", total_planned)
|
||||
else:
|
||||
log.info("zapisane %d z %d (reszta zmieniona w miedzyczasie)", total_updated, total_planned)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
|
|
@ -1,138 +0,0 @@
|
|||
"""Dekoduje encje HTML w tytulach scen i filmow zapisanych przed poprawka normalizera.
|
||||
|
||||
Tuby (glownie xvideos, JSON-LD) oddawaly tytuly typu `we're [AI]`,
|
||||
a my zapisywalismy je tak jak przyszly. `normalize` robil z tego slowa `apos`, `lpar`,
|
||||
wiec tytul nie trafial ani w wyszukiwarke, ani w kanon. Od poprawki `normalize_scene`
|
||||
przepuszcza tytul przez `unescape_html`, ale istniejace wiersze trzeba poprawic osobno:
|
||||
scraper tytulu juz ustawionego nie nadpisuje.
|
||||
|
||||
Co zmienia na wierszu:
|
||||
- `title` na wersje po `unescape_html`,
|
||||
- `title_normalized` na `normalize(nowy tytul)`, tak jak liczy go ingest,
|
||||
- `slug` tylko gdy byl wyliczony ze starego tytulu (`slugify(stary) == slug`); slug
|
||||
ustawiony z innego zrodla zostaje.
|
||||
|
||||
Zapis jest warunkowy (`WHERE title = stary`), wiec wiersz zmieniony w miedzyczasie przez
|
||||
ingest albo merge jest pomijany. Ponowne odpalenie jest bezpieczne: bierze tylko tytuly,
|
||||
w ktorych nadal jest encja.
|
||||
|
||||
Uzycie:
|
||||
python3 -u /srv/scripts/fix_html_entity_titles.py --dry-run
|
||||
python3 -u /srv/scripts/fix_html_entity_titles.py
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import random
|
||||
import sys
|
||||
|
||||
import sqlalchemy as sa
|
||||
|
||||
from app.db import session_scope
|
||||
from app.normalize.text import normalize, slugify, unescape_html
|
||||
|
||||
log = logging.getLogger("fix_html_entity_titles")
|
||||
|
||||
# Ten sam zestaw co `_HTML_ENTITY_RE` w app/normalize/text.py, w skladni Postgresa.
|
||||
_PG_ENTITY_RE = r"&(#[0-9]{1,7}|#[xX][0-9a-fA-F]{1,6}|[A-Za-z][A-Za-z0-9]{1,31});"
|
||||
_TABLES = ("scenes", "movies")
|
||||
|
||||
|
||||
def _plan(table: str, *, limit: int | None) -> list[dict]:
|
||||
with session_scope() as session:
|
||||
rows = session.execute(
|
||||
sa.text(
|
||||
f"SELECT id, title, title_normalized, slug FROM {table} "
|
||||
"WHERE title ~ :re" + (" LIMIT :limit" if limit else "")
|
||||
),
|
||||
{"re": _PG_ENTITY_RE, "limit": limit},
|
||||
).all()
|
||||
|
||||
plan: list[dict] = []
|
||||
for row_id, title, title_norm, slug in rows:
|
||||
new_title = unescape_html(title)
|
||||
if not new_title or new_title == title:
|
||||
continue
|
||||
new_slug = slug
|
||||
if slug is None or slug == slugify(title):
|
||||
new_slug = slugify(new_title) or slug
|
||||
plan.append(
|
||||
{
|
||||
"id": row_id,
|
||||
"old_title": title,
|
||||
"title": new_title,
|
||||
"old_title_normalized": title_norm,
|
||||
"title_normalized": normalize(new_title),
|
||||
"old_slug": slug,
|
||||
"slug": new_slug,
|
||||
}
|
||||
)
|
||||
log.info("%s: %d wierszy z encja, %d do zmiany", table, len(rows), len(plan))
|
||||
return plan
|
||||
|
||||
|
||||
def _apply(table: str, plan: list[dict], *, batch_size: int) -> int:
|
||||
# Jedno UPDATE na partie przez unnest: rowcount jest wtedy wiarygodny.
|
||||
stmt = sa.text(
|
||||
f"UPDATE {table} t SET title = v.title, title_normalized = v.title_normalized, slug = v.slug "
|
||||
"FROM unnest(CAST(:ids AS uuid[]), CAST(:old_titles AS text[]), CAST(:titles AS text[]), "
|
||||
"CAST(:norms AS text[]), CAST(:slugs AS text[])) "
|
||||
"AS v(id, old_title, title, title_normalized, slug) "
|
||||
"WHERE t.id = v.id AND t.title = v.old_title"
|
||||
)
|
||||
updated = 0
|
||||
for start in range(0, len(plan), batch_size):
|
||||
batch = plan[start : start + batch_size]
|
||||
params = {
|
||||
"ids": [str(item["id"]) for item in batch],
|
||||
"old_titles": [item["old_title"] for item in batch],
|
||||
"titles": [item["title"] for item in batch],
|
||||
"norms": [item["title_normalized"] for item in batch],
|
||||
"slugs": [item["slug"] for item in batch],
|
||||
}
|
||||
with session_scope() as session:
|
||||
updated += session.execute(stmt, params).rowcount
|
||||
log.info("%s: %d/%d przejrzane, %d zapisane", table, start + len(batch), len(plan), updated)
|
||||
return updated
|
||||
|
||||
|
||||
def main() -> int:
|
||||
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
p.add_argument("--dry-run", action="store_true", help="tylko policz i pokaz probke")
|
||||
p.add_argument("--batch-size", type=int, default=500)
|
||||
p.add_argument("--sample", type=int, default=20, help="ile losowych zmian pokazac")
|
||||
p.add_argument("--limit", type=int, default=None, help="max wierszy na tabele (test)")
|
||||
args = p.parse_args()
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||||
|
||||
total_planned = total_updated = 0
|
||||
for table in _TABLES:
|
||||
plan = _plan(table, limit=args.limit)
|
||||
if not plan:
|
||||
continue
|
||||
total_planned += len(plan)
|
||||
slug_changes = sum(1 for item in plan if item["slug"] != item["old_slug"])
|
||||
log.info("%s: slug zmieniony w %d z %d", table, slug_changes, len(plan))
|
||||
for item in random.sample(plan, min(args.sample, len(plan))):
|
||||
print(f"[{table}] {item['id']}")
|
||||
print(f" title: {item['old_title']!r}")
|
||||
print(f" -> {item['title']!r}")
|
||||
print(f" norm: {item['old_title_normalized']!r}")
|
||||
print(f" -> {item['title_normalized']!r}")
|
||||
if item["slug"] != item["old_slug"]:
|
||||
print(f" slug: {item['old_slug']!r}")
|
||||
print(f" -> {item['slug']!r}")
|
||||
sys.stdout.flush()
|
||||
if not args.dry_run:
|
||||
total_updated += _apply(table, plan, batch_size=args.batch_size)
|
||||
|
||||
if args.dry_run:
|
||||
log.info("DRY RUN: %d wierszy do zmiany, nic nie zapisane", total_planned)
|
||||
else:
|
||||
log.info("zapisane %d z %d (reszta zmieniona w miedzyczasie)", total_updated, total_planned)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
120
scripts/fix_title_normalized.py
Normal file
120
scripts/fix_title_normalized.py
Normal file
|
|
@ -0,0 +1,120 @@
|
|||
"""Przelicza `title_normalized` scen i filmow tam, gdzie rozjechal sie z `normalize(title)`.
|
||||
|
||||
Ingest liczy `title_normalized = normalize(title)`, ale stare skrypty naprawcze
|
||||
(`bulk_rescrape_hqporner.py`, `repair_truncated_titles.py`) zapisywaly samo `lower(title)`.
|
||||
Takie sceny maja w kluczu interpunkcje (`cindy hope - horny boss`, `matty's virgin`), wiec
|
||||
nie trafiaja w dokladne dopasowanie tytulu w resolverze ani w dedup. Pelny skan z 2026-09-15:
|
||||
6005 scen, wszystkie w formie `lower(title)`, innych rozjazdow nie bylo.
|
||||
|
||||
Skrypt czyta cala tabele strumieniowo (4,3 mln scen to ok. 1,5 minuty), zbiera wiersze,
|
||||
w ktorych klucz sie nie zgadza, i zapisuje partiami. Zapis jest warunkowy
|
||||
(`WHERE title = stary AND title_normalized = stary`), wiec wiersz zmieniony w miedzyczasie
|
||||
jest pomijany. `title` i `slug` zostaja bez zmian.
|
||||
|
||||
Uzycie:
|
||||
python3 -u /srv/scripts/fix_title_normalized.py --dry-run
|
||||
python3 -u /srv/scripts/fix_title_normalized.py
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import collections
|
||||
import logging
|
||||
import random
|
||||
import sys
|
||||
import time
|
||||
|
||||
import sqlalchemy as sa
|
||||
|
||||
from app.db import engine, session_scope
|
||||
from app.normalize.text import normalize
|
||||
|
||||
log = logging.getLogger("fix_title_normalized")
|
||||
|
||||
_TABLES = ("scenes", "movies")
|
||||
|
||||
|
||||
def _plan(table: str) -> tuple[list[dict], collections.Counter]:
|
||||
t0 = time.monotonic()
|
||||
plan: list[dict] = []
|
||||
kinds: collections.Counter = collections.Counter()
|
||||
scanned = 0
|
||||
with engine.connect().execution_options(stream_results=True, yield_per=20000) as conn:
|
||||
for row_id, title, title_norm in conn.execute(sa.text(f"SELECT id, title, title_normalized FROM {table}")):
|
||||
scanned += 1
|
||||
new = normalize(title)
|
||||
if new == title_norm:
|
||||
continue
|
||||
kinds["lower" if title_norm == (title or "").lower() else "inne"] += 1
|
||||
plan.append({"id": row_id, "title": title, "old": title_norm, "new": new})
|
||||
log.info(
|
||||
"%s: przejrzane %d w %ds, do zmiany %d %s",
|
||||
table, scanned, time.monotonic() - t0, len(plan), dict(kinds),
|
||||
)
|
||||
return plan, kinds
|
||||
|
||||
|
||||
def _apply(table: str, plan: list[dict], *, batch_size: int) -> int:
|
||||
stmt = sa.text(
|
||||
f"UPDATE {table} t SET title_normalized = v.new "
|
||||
"FROM unnest(CAST(:ids AS uuid[]), CAST(:titles AS text[]), CAST(:olds AS text[]), "
|
||||
"CAST(:news AS text[])) AS v(id, title, old, new) "
|
||||
"WHERE t.id = v.id AND t.title = v.title AND t.title_normalized = v.old"
|
||||
)
|
||||
updated = 0
|
||||
for start in range(0, len(plan), batch_size):
|
||||
batch = plan[start : start + batch_size]
|
||||
params = {
|
||||
"ids": [str(item["id"]) for item in batch],
|
||||
"titles": [item["title"] for item in batch],
|
||||
"olds": [item["old"] for item in batch],
|
||||
"news": [item["new"] for item in batch],
|
||||
}
|
||||
with session_scope() as session:
|
||||
updated += session.execute(stmt, params).rowcount
|
||||
log.info("%s: %d/%d przejrzane, %d zapisane", table, start + len(batch), len(plan), updated)
|
||||
return updated
|
||||
|
||||
|
||||
def main() -> int:
|
||||
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
p.add_argument("--dry-run", action="store_true", help="tylko policz i pokaz probke")
|
||||
p.add_argument("--batch-size", type=int, default=500)
|
||||
p.add_argument("--sample", type=int, default=20, help="ile losowych zmian pokazac na tabele")
|
||||
p.add_argument(
|
||||
"--only-lower",
|
||||
action="store_true",
|
||||
help="zapisz tylko rozjazdy w formie lower(title), inne tylko pokaz",
|
||||
)
|
||||
args = p.parse_args()
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||||
|
||||
total_planned = total_updated = 0
|
||||
for table in _TABLES:
|
||||
plan, kinds = _plan(table)
|
||||
if kinds.get("inne"):
|
||||
others = [item for item in plan if item["old"] != (item["title"] or "").lower()]
|
||||
for item in random.sample(others, min(args.sample, len(others))):
|
||||
print(f"[{table} inne] {item['title']!r}: {item['old']!r} -> {item['new']!r}")
|
||||
if args.only_lower:
|
||||
plan = [item for item in plan if item["old"] == (item["title"] or "").lower()]
|
||||
if not plan:
|
||||
continue
|
||||
for item in random.sample(plan, min(args.sample, len(plan))):
|
||||
print(f"[{table}] {item['title']!r}")
|
||||
print(f" {item['old']!r}")
|
||||
print(f" -> {item['new']!r}")
|
||||
sys.stdout.flush()
|
||||
total_planned += len(plan)
|
||||
if not args.dry_run:
|
||||
total_updated += _apply(table, plan, batch_size=args.batch_size)
|
||||
|
||||
if args.dry_run:
|
||||
log.info("DRY RUN: %d wierszy do zmiany, nic nie zapisane", total_planned)
|
||||
else:
|
||||
log.info("zapisane %d z %d (reszta zmieniona w miedzyczasie)", total_updated, total_planned)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
|
|
@ -18,6 +18,7 @@ import sqlalchemy as sa
|
|||
|
||||
from app.connectors.direct_scrapers._browse_base import meta_content
|
||||
from app.db import session_scope
|
||||
from app.normalize.text import normalize, unescape_html
|
||||
from app.extractors._fetch import browser_get
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
|
@ -59,7 +60,7 @@ def main() -> int:
|
|||
if r.status_code >= 400:
|
||||
continue
|
||||
|
||||
new_title = meta_content(r.text, property="og:title")
|
||||
new_title = unescape_html(meta_content(r.text, property="og:title"))
|
||||
if not new_title:
|
||||
continue
|
||||
# Only update if longer (i.e. parser found full title after fix)
|
||||
|
|
@ -70,9 +71,9 @@ def main() -> int:
|
|||
with session_scope() as session:
|
||||
session.execute(
|
||||
sa.text(
|
||||
"UPDATE scenes SET title = :t, title_normalized = LOWER(:t) WHERE id = :sid"
|
||||
"UPDATE scenes SET title = :t, title_normalized = :tn WHERE id = :sid"
|
||||
),
|
||||
{"t": new_title.strip(), "sid": scene_id},
|
||||
{"t": new_title, "tn": normalize(new_title), "sid": scene_id},
|
||||
)
|
||||
counters["updated"] += 1
|
||||
if i % 25 == 0:
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import pytest
|
||||
|
||||
from app.normalize.text import (
|
||||
clean_description,
|
||||
normalize,
|
||||
normalize_person,
|
||||
person_named_in,
|
||||
|
|
@ -107,3 +108,28 @@ def test_person_named_in(title: str, person: str, expected: bool) -> None:
|
|||
)
|
||||
def test_unescape_html(raw: str | None, expected: str) -> None:
|
||||
assert unescape_html(raw) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("raw", "expected"),
|
||||
[
|
||||
('Manny in "MANUEL CREAMPIES 2". She loved it.', 'Manny in "MANUEL CREAMPIES 2". She loved it.'),
|
||||
# Znaczniki zakodowane jako encje.
|
||||
("Is it love?<br>Eveline only ordered", "Is it love?\nEveline only ordered"),
|
||||
("door.</br></br>Jessica", "door.\n\nJessica"),
|
||||
("wash.<br><br><br><br>She", "wash.\n\nShe"),
|
||||
# Surowe znaczniki.
|
||||
("Welcome, Carlo!<br /> ", "Welcome, Carlo!"),
|
||||
('<p>See <a href="/x">Bunny</a> & <b>more</b>!</p>', "See Bunny & more!"),
|
||||
# To nie sa znaczniki.
|
||||
("I miss you so much <3.", "I miss you so much <3."),
|
||||
("It was <a lot of fun> and <b careful>", "It was <a lot of fun> and <b careful>"),
|
||||
('<span style="color: red" class=x>Hot</span>', "Hot"),
|
||||
("<Yokomiya-san. SSSP! No Cuts]", "<Yokomiya-san. SSSP! No Cuts]"),
|
||||
("lips&tongues; Alix &Amp; Cherie", "lips&tongues; Alix & Cherie"),
|
||||
(" ", None),
|
||||
(None, None),
|
||||
],
|
||||
)
|
||||
def test_clean_description(raw: str | None, expected: str | None) -> None:
|
||||
assert clean_description(raw) == expected
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue