perf(tags): warianty pisowni tagow jednym zapytaniem
Some checks are pending
Backend tests / test (push) Waiting to run
Some checks are pending
Backend tests / test (push) Waiting to run
enrich-tags wolal resolve_tag dla kazdej nazwy spoza slugow, a ten szukal po alnum-kluczu i liczbie mnogiej osobnymi zapytaniami (N+1, GOON-5F). Nowe resolve_tags_by_altkey robi to jednym IN po indeksie ix_tags_name_altkey; _resolve_by_altkey dla jednej nazwy korzysta z niego (1 zapytanie zamiast 2). Na 603 nazwach z produkcji wyniki identyczne, 0,77 s -> 0,03 s. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
cb9e4ab035
commit
0da36d5c1e
2 changed files with 40 additions and 25 deletions
|
|
@ -1851,7 +1851,7 @@ def enrich_tags_from_tube(
|
|||
from app.models.tag import Tag
|
||||
from app.normalize.scenes import NormalizedTag
|
||||
from app.normalize.text import slugify
|
||||
from app.resolve.tag_resolver import resolve_tag
|
||||
from app.resolve.tag_resolver import resolve_tag, resolve_tags_by_altkey
|
||||
|
||||
scene = session.get(Scene, scene_id)
|
||||
if scene is None:
|
||||
|
|
@ -1923,10 +1923,17 @@ def enrich_tags_from_tube(
|
|||
{t.slug: t for t in session.execute(select(Tag).where(Tag.slug.in_(direct))).scalars()}
|
||||
if direct else {}
|
||||
)
|
||||
# Nazwy spoza slugow: wariant pisowni istniejacego tagu (alnum-klucz, liczba mnoga)
|
||||
# tez jednym zapytaniem. resolve_tag zostaje tylko dla naprawde nowych nazw.
|
||||
direct_set = set(direct)
|
||||
by_alt = resolve_tags_by_altkey(
|
||||
session,
|
||||
[n for n in tag_names if slugify(n) in direct_set and slugify(n) not in by_slug],
|
||||
)
|
||||
tag_ids: list = []
|
||||
for name in tag_names:
|
||||
slug = slugify(name)
|
||||
tag = by_slug.get(slug)
|
||||
tag = by_slug.get(slug) or by_alt.get(name)
|
||||
if tag is None:
|
||||
tag = resolve_tag(session, norm=NormalizedTag(name=name, slug=slug, external_id=None))
|
||||
if tag is not None and tag.id not in tag_ids:
|
||||
|
|
|
|||
|
|
@ -29,35 +29,43 @@ def _resolve_by_altkey(session: Session, name: str) -> Tag | None:
|
|||
scene_count wygrywa). Zapobiega regeneracji duplikatów: nowy wariant "Deep Throat"
|
||||
trafia do istniejącego "deepthroat" zamiast tworzyć osobny tag. Używa indeksu
|
||||
funkcyjnego ix_tags_name_altkey."""
|
||||
key = _altkey(name)
|
||||
if not key:
|
||||
return None
|
||||
expr = func.regexp_replace(func.lower(func.btrim(Tag.name)), "[^a-z0-9]", "", "g")
|
||||
hit = (
|
||||
session.execute(select(Tag).where(expr == key).order_by(Tag.scene_count.desc()))
|
||||
.scalars()
|
||||
.first()
|
||||
)
|
||||
if hit is not None:
|
||||
return hit
|
||||
return resolve_tags_by_altkey(session, [name]).get(name)
|
||||
|
||||
|
||||
def resolve_tags_by_altkey(session: Session, names) -> dict[str, Tag]:
|
||||
"""`_resolve_by_altkey` dla wielu nazw jednym zapytaniem: {nazwa: tag} dla trafień.
|
||||
|
||||
Po zapytaniu na nazwę enrich-tags robił N+1 przy kilku nowych nazwach ze strony
|
||||
tuba (Sentry GOON-5F).
|
||||
"""
|
||||
# Liczba mnoga. Alnum-klucz z natury jej NIE łapie ("blowjob" != "blowjobs"), więc
|
||||
# duplikaty odrastały mimo tamtej prewencji (audyt 2026-08-02: 1944 pary, m.in.
|
||||
# Blowjob/Blowjobs, Teen/Teens, Cumshot/Cumshots). Próbujemy obu kierunków: nowe
|
||||
# "Blowjobs" trafia w istniejące "Blowjob", a nowe "Blowjob" w istniejące "Blowjobs".
|
||||
# Tylko proste `+s`, form nieregularnych (panty/panties) świadomie nie ruszamy, bo
|
||||
# zbyt łatwo o fałszywy zlew (glass/glasses, bra/bras).
|
||||
alts = [key + "s"] if not key.endswith("s") else [key[:-1]]
|
||||
for alt in alts:
|
||||
if len(alt) < 3:
|
||||
# zbyt łatwo o fałszywy zlew (glass/glasses, bra/bras). Dokładny klucz ma pierwszeństwo.
|
||||
keys: dict[str, tuple[str, str | None]] = {}
|
||||
for name in names:
|
||||
key = _altkey(name)
|
||||
if not key:
|
||||
continue
|
||||
hit = (
|
||||
session.execute(select(Tag).where(expr == alt).order_by(Tag.scene_count.desc()))
|
||||
.scalars()
|
||||
.first()
|
||||
)
|
||||
if hit is not None:
|
||||
return hit
|
||||
return None
|
||||
alt = key + "s" if not key.endswith("s") else key[:-1]
|
||||
keys[name] = (key, alt if len(alt) >= 3 else None)
|
||||
wanted = {k for k, _ in keys.values()} | {a for _, a in keys.values() if a}
|
||||
if not wanted:
|
||||
return {}
|
||||
expr = func.regexp_replace(func.lower(func.btrim(Tag.name)), "[^a-z0-9]", "", "g")
|
||||
best: dict[str, Tag] = {}
|
||||
for key, tag in session.execute(
|
||||
select(expr, Tag).where(expr.in_(wanted)).order_by(Tag.scene_count.desc())
|
||||
).all():
|
||||
best.setdefault(key, tag)
|
||||
out: dict[str, Tag] = {}
|
||||
for name, (key, alt) in keys.items():
|
||||
tag = best.get(key) or (best.get(alt) if alt else None)
|
||||
if tag is not None:
|
||||
out[name] = tag
|
||||
return out
|
||||
|
||||
|
||||
def _canonical_dup2_slug(session: Session, slug: str) -> str:
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue