perf(tags): liczniki tagow JAV i animacji liczone w tle
/tags?vertical=jav liczyl agregat po zrodlach i scene_tags na kazde wejscie, dwa razy: 1 s na rozgrzanym cache, 17-24 s na zimnym. Teraz czyta tabele tag_vertical_counts (migracja 0047), ktora odswieza refresh_taxonomy_counts co 3 h. Zakladka JAV: 0,35 s. Gdy tabela jest pusta dla wertykalu, zostaje stary agregat. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
494b15aedd
commit
c97af57979
5 changed files with 123 additions and 19 deletions
41
alembic/versions/20260927_0047_tag_vertical_counts.py
Normal file
41
alembic/versions/20260927_0047_tag_vertical_counts.py
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
"""Liczniki tagow per wertykal (JAV, animacja), liczone w tle zamiast na kazde wejscie.
|
||||
|
||||
/tags?vertical=jav liczyl na zywo: sceny z zywym zrodlem JAV, ich scene_tags, GROUP BY,
|
||||
i to dwa razy (licznik total i strona). Na rozgrzanym cache 1 s, na zimnym 17-24 s
|
||||
(log API 2026-09-27, zakladka JAV w apce). Tabela (vertical, tag_id) zamiast kolumny
|
||||
na `tags` na kazdy wertykal, zeby nowy wertykal nie wymagal migracji. Wypelnia ja
|
||||
refresh_taxonomy_counts (co 3 h); dopoki dla wertykalu nie ma wierszy, API liczy po staremu.
|
||||
|
||||
Revision ID: 0047_tag_vertical_counts
|
||||
Revises: 0046_scenes_title_stats
|
||||
Create Date: 2026-09-27
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy.dialects.postgresql import UUID
|
||||
|
||||
revision = "0047_tag_vertical_counts"
|
||||
down_revision: str | None = "0046_scenes_title_stats"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"tag_vertical_counts",
|
||||
sa.Column("vertical", sa.String(32), nullable=False),
|
||||
sa.Column("tag_id", UUID(as_uuid=True), nullable=False),
|
||||
sa.Column("scene_count", sa.Integer, nullable=False),
|
||||
sa.PrimaryKeyConstraint("vertical", "tag_id", name="pk_tag_vertical_counts"),
|
||||
sa.ForeignKeyConstraint(
|
||||
["tag_id"], ["tags.id"],
|
||||
name="fk_tag_vertical_counts_tag_id_tags",
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table("tag_vertical_counts")
|
||||
|
|
@ -27,6 +27,7 @@ from app.models.playback_source import PlaybackSource
|
|||
from app.models.scene import ScenePerformer, SceneTag
|
||||
from app.models.studio import Studio
|
||||
from app.models.tag import Tag
|
||||
from app.models.tag_vertical_count import TagVerticalCount
|
||||
from app.verticals import VERTICALS
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
|
@ -158,25 +159,36 @@ def list_tags(
|
|||
return TagListOut(items=items, total=total, page=page, per_page=per_page)
|
||||
|
||||
if vertical in VERTICALS:
|
||||
# Wertykał liczymy NA ŻYWO, bez denormalizacji: to małe zbiory (animacja ~9,5 tys.
|
||||
# scen wobec 3,8 mln całości), więc agregat jest tani, a dokładanie kolumny
|
||||
# `scene_count_<wertykał>` do `tags` kosztowałoby migrację przy każdym nowym
|
||||
# wertykale. Zdenormalizowany `Tag.scene_count` opisuje WYŁĄCZNIE główny feed.
|
||||
v_scenes = (
|
||||
select(PlaybackSource.scene_id)
|
||||
.where(
|
||||
PlaybackSource.dead_at.is_(None),
|
||||
PlaybackSource.origin.in_(VERTICALS[vertical]),
|
||||
# Liczniki wertykału czytamy z tag_vertical_counts (refresh_taxonomy_counts co 3 h,
|
||||
# migracja 0047). Agregat na żywo trwał 1 s na rozgrzanym cache i 17-24 s na
|
||||
# zimnym (zakładka JAV, 2026-09-27). Zostaje jako zapas, gdy tabela jest jeszcze
|
||||
# pusta dla tego wertykału, np. zaraz po wdrożeniu.
|
||||
# Zdenormalizowany `Tag.scene_count` opisuje WYŁĄCZNIE główny feed.
|
||||
has_counts = session.execute(
|
||||
select(TagVerticalCount.tag_id).where(TagVerticalCount.vertical == vertical).limit(1)
|
||||
).first()
|
||||
if has_counts:
|
||||
v_count = (
|
||||
select(TagVerticalCount.tag_id, TagVerticalCount.scene_count.label("c"))
|
||||
.where(TagVerticalCount.vertical == vertical)
|
||||
.subquery()
|
||||
)
|
||||
else:
|
||||
v_scenes = (
|
||||
select(PlaybackSource.scene_id)
|
||||
.where(
|
||||
PlaybackSource.dead_at.is_(None),
|
||||
PlaybackSource.origin.in_(VERTICALS[vertical]),
|
||||
)
|
||||
.distinct()
|
||||
.subquery()
|
||||
)
|
||||
v_count = (
|
||||
select(SceneTag.tag_id, func.count().label("c"))
|
||||
.join(v_scenes, v_scenes.c.scene_id == SceneTag.scene_id)
|
||||
.group_by(SceneTag.tag_id)
|
||||
.subquery()
|
||||
)
|
||||
.distinct()
|
||||
.subquery()
|
||||
)
|
||||
v_count = (
|
||||
select(SceneTag.tag_id, func.count().label("c"))
|
||||
.join(v_scenes, v_scenes.c.scene_id == SceneTag.scene_id)
|
||||
.group_by(SceneTag.tag_id)
|
||||
.subquery()
|
||||
)
|
||||
base = (
|
||||
select(Tag, func.coalesce(v_count.c.c, 0).label("scene_count"))
|
||||
.outerjoin(v_count, v_count.c.tag_id == Tag.id)
|
||||
|
|
|
|||
|
|
@ -38,6 +38,7 @@ from app.models.scene import (
|
|||
from app.models.source import Source
|
||||
from app.models.studio import Studio, StudioAlias, StudioExternalRef
|
||||
from app.models.tag import Tag
|
||||
from app.models.tag_vertical_count import TagVerticalCount
|
||||
|
||||
__all__ = [
|
||||
"Base",
|
||||
|
|
@ -84,4 +85,5 @@ __all__ = [
|
|||
"StudioAlias",
|
||||
"StudioExternalRef",
|
||||
"Tag",
|
||||
"TagVerticalCount",
|
||||
]
|
||||
|
|
|
|||
25
app/models/tag_vertical_count.py
Normal file
25
app/models/tag_vertical_count.py
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
"""Liczba scen tagu w wertykale (JAV, animacja), odswiezana w tle.
|
||||
|
||||
Wypelnia `refresh_taxonomy_counts`, czyta /tags?vertical=. Ta sama definicja co dawny
|
||||
agregat na zywo: sceny z >= 1 zywym zrodlem z originow wertykalu, bez progu dlugosci.
|
||||
Patrz migracja 0047.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
|
||||
from sqlalchemy import ForeignKey, Integer, String
|
||||
from sqlalchemy.dialects.postgresql import UUID
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from app.models.base import Base
|
||||
|
||||
|
||||
class TagVerticalCount(Base):
|
||||
__tablename__ = "tag_vertical_counts"
|
||||
|
||||
vertical: Mapped[str] = mapped_column(String(32), primary_key=True)
|
||||
tag_id: Mapped[uuid.UUID] = mapped_column(
|
||||
UUID(as_uuid=True), ForeignKey("tags.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
scene_count: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
|
|
@ -23,7 +23,7 @@ import logging
|
|||
from sqlalchemy import text
|
||||
|
||||
from app.db import session_scope
|
||||
from app.verticals import ALL_VERTICAL_ORIGINS
|
||||
from app.verticals import ALL_VERTICAL_ORIGINS, VERTICALS
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -129,11 +129,35 @@ _STUDIOS_SQL = text(
|
|||
)
|
||||
|
||||
|
||||
# Tagi wertykalu (lista w zakladce JAV/animacja): sceny z zywym zrodlem z jego originow,
|
||||
# bez progu dlugosci, ta sama definicja co dawny agregat w /tags?vertical=. Podmiana
|
||||
# calego kompletu w jednej transakcji, to kilka tysiecy wierszy (migracja 0047).
|
||||
_VERTICAL_TAGS_DELETE_SQL = text("DELETE FROM tag_vertical_counts WHERE vertical = :vertical")
|
||||
_VERTICAL_TAGS_INSERT_SQL = text(
|
||||
"""
|
||||
INSERT INTO tag_vertical_counts (vertical, tag_id, scene_count)
|
||||
SELECT :vertical, st.tag_id, count(*)
|
||||
FROM scene_tags st
|
||||
JOIN (
|
||||
SELECT DISTINCT ps.scene_id FROM playback_sources ps
|
||||
WHERE ps.dead_at IS NULL AND ps.origin = ANY(:origins)
|
||||
) v ON v.scene_id = st.scene_id
|
||||
GROUP BY st.tag_id
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def refresh_taxonomy_counts() -> dict[str, int]:
|
||||
"""Przelicza scene_count dla tags/performers/studios. Zwraca rowcount per tabela
|
||||
(ile wierszy faktycznie się zmieniło)."""
|
||||
out: dict[str, int] = {}
|
||||
with session_scope() as session:
|
||||
for vertical, origins in VERTICALS.items():
|
||||
session.execute(_VERTICAL_TAGS_DELETE_SQL, {"vertical": vertical})
|
||||
out[f"tags_{vertical}"] = session.execute(
|
||||
_VERTICAL_TAGS_INSERT_SQL, {"vertical": vertical, "origins": list(origins)}
|
||||
).rowcount or 0
|
||||
session.commit()
|
||||
for name, stmt in (
|
||||
("tags", _TAGS_SQL),
|
||||
("performers", _PERFORMERS_SQL),
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue