perf(tags): liczniki tagow JAV i animacji liczone w tle
/tags?vertical=jav liczyl agregat po zrodlach i scene_tags na kazde wejscie, dwa razy: 1 s na rozgrzanym cache, 17-24 s na zimnym. Teraz czyta tabele tag_vertical_counts (migracja 0047), ktora odswieza refresh_taxonomy_counts co 3 h. Zakladka JAV: 0,35 s. Gdy tabela jest pusta dla wertykalu, zostaje stary agregat. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
494b15aedd
commit
c97af57979
5 changed files with 123 additions and 19 deletions
41
alembic/versions/20260927_0047_tag_vertical_counts.py
Normal file
41
alembic/versions/20260927_0047_tag_vertical_counts.py
Normal file
|
|
@ -0,0 +1,41 @@
|
||||||
|
"""Liczniki tagow per wertykal (JAV, animacja), liczone w tle zamiast na kazde wejscie.
|
||||||
|
|
||||||
|
/tags?vertical=jav liczyl na zywo: sceny z zywym zrodlem JAV, ich scene_tags, GROUP BY,
|
||||||
|
i to dwa razy (licznik total i strona). Na rozgrzanym cache 1 s, na zimnym 17-24 s
|
||||||
|
(log API 2026-09-27, zakladka JAV w apce). Tabela (vertical, tag_id) zamiast kolumny
|
||||||
|
na `tags` na kazdy wertykal, zeby nowy wertykal nie wymagal migracji. Wypelnia ja
|
||||||
|
refresh_taxonomy_counts (co 3 h); dopoki dla wertykalu nie ma wierszy, API liczy po staremu.
|
||||||
|
|
||||||
|
Revision ID: 0047_tag_vertical_counts
|
||||||
|
Revises: 0046_scenes_title_stats
|
||||||
|
Create Date: 2026-09-27
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
from sqlalchemy.dialects.postgresql import UUID
|
||||||
|
|
||||||
|
revision = "0047_tag_vertical_counts"
|
||||||
|
down_revision: str | None = "0046_scenes_title_stats"
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.create_table(
|
||||||
|
"tag_vertical_counts",
|
||||||
|
sa.Column("vertical", sa.String(32), nullable=False),
|
||||||
|
sa.Column("tag_id", UUID(as_uuid=True), nullable=False),
|
||||||
|
sa.Column("scene_count", sa.Integer, nullable=False),
|
||||||
|
sa.PrimaryKeyConstraint("vertical", "tag_id", name="pk_tag_vertical_counts"),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["tag_id"], ["tags.id"],
|
||||||
|
name="fk_tag_vertical_counts_tag_id_tags",
|
||||||
|
ondelete="CASCADE",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_table("tag_vertical_counts")
|
||||||
|
|
@ -27,6 +27,7 @@ from app.models.playback_source import PlaybackSource
|
||||||
from app.models.scene import ScenePerformer, SceneTag
|
from app.models.scene import ScenePerformer, SceneTag
|
||||||
from app.models.studio import Studio
|
from app.models.studio import Studio
|
||||||
from app.models.tag import Tag
|
from app.models.tag import Tag
|
||||||
|
from app.models.tag_vertical_count import TagVerticalCount
|
||||||
from app.verticals import VERTICALS
|
from app.verticals import VERTICALS
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
@ -158,10 +159,21 @@ def list_tags(
|
||||||
return TagListOut(items=items, total=total, page=page, per_page=per_page)
|
return TagListOut(items=items, total=total, page=page, per_page=per_page)
|
||||||
|
|
||||||
if vertical in VERTICALS:
|
if vertical in VERTICALS:
|
||||||
# Wertykał liczymy NA ŻYWO, bez denormalizacji: to małe zbiory (animacja ~9,5 tys.
|
# Liczniki wertykału czytamy z tag_vertical_counts (refresh_taxonomy_counts co 3 h,
|
||||||
# scen wobec 3,8 mln całości), więc agregat jest tani, a dokładanie kolumny
|
# migracja 0047). Agregat na żywo trwał 1 s na rozgrzanym cache i 17-24 s na
|
||||||
# `scene_count_<wertykał>` do `tags` kosztowałoby migrację przy każdym nowym
|
# zimnym (zakładka JAV, 2026-09-27). Zostaje jako zapas, gdy tabela jest jeszcze
|
||||||
# wertykale. Zdenormalizowany `Tag.scene_count` opisuje WYŁĄCZNIE główny feed.
|
# pusta dla tego wertykału, np. zaraz po wdrożeniu.
|
||||||
|
# Zdenormalizowany `Tag.scene_count` opisuje WYŁĄCZNIE główny feed.
|
||||||
|
has_counts = session.execute(
|
||||||
|
select(TagVerticalCount.tag_id).where(TagVerticalCount.vertical == vertical).limit(1)
|
||||||
|
).first()
|
||||||
|
if has_counts:
|
||||||
|
v_count = (
|
||||||
|
select(TagVerticalCount.tag_id, TagVerticalCount.scene_count.label("c"))
|
||||||
|
.where(TagVerticalCount.vertical == vertical)
|
||||||
|
.subquery()
|
||||||
|
)
|
||||||
|
else:
|
||||||
v_scenes = (
|
v_scenes = (
|
||||||
select(PlaybackSource.scene_id)
|
select(PlaybackSource.scene_id)
|
||||||
.where(
|
.where(
|
||||||
|
|
|
||||||
|
|
@ -38,6 +38,7 @@ from app.models.scene import (
|
||||||
from app.models.source import Source
|
from app.models.source import Source
|
||||||
from app.models.studio import Studio, StudioAlias, StudioExternalRef
|
from app.models.studio import Studio, StudioAlias, StudioExternalRef
|
||||||
from app.models.tag import Tag
|
from app.models.tag import Tag
|
||||||
|
from app.models.tag_vertical_count import TagVerticalCount
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"Base",
|
"Base",
|
||||||
|
|
@ -84,4 +85,5 @@ __all__ = [
|
||||||
"StudioAlias",
|
"StudioAlias",
|
||||||
"StudioExternalRef",
|
"StudioExternalRef",
|
||||||
"Tag",
|
"Tag",
|
||||||
|
"TagVerticalCount",
|
||||||
]
|
]
|
||||||
|
|
|
||||||
25
app/models/tag_vertical_count.py
Normal file
25
app/models/tag_vertical_count.py
Normal file
|
|
@ -0,0 +1,25 @@
|
||||||
|
"""Liczba scen tagu w wertykale (JAV, animacja), odswiezana w tle.
|
||||||
|
|
||||||
|
Wypelnia `refresh_taxonomy_counts`, czyta /tags?vertical=. Ta sama definicja co dawny
|
||||||
|
agregat na zywo: sceny z >= 1 zywym zrodlem z originow wertykalu, bez progu dlugosci.
|
||||||
|
Patrz migracja 0047.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import uuid
|
||||||
|
|
||||||
|
from sqlalchemy import ForeignKey, Integer, String
|
||||||
|
from sqlalchemy.dialects.postgresql import UUID
|
||||||
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
|
||||||
|
from app.models.base import Base
|
||||||
|
|
||||||
|
|
||||||
|
class TagVerticalCount(Base):
|
||||||
|
__tablename__ = "tag_vertical_counts"
|
||||||
|
|
||||||
|
vertical: Mapped[str] = mapped_column(String(32), primary_key=True)
|
||||||
|
tag_id: Mapped[uuid.UUID] = mapped_column(
|
||||||
|
UUID(as_uuid=True), ForeignKey("tags.id", ondelete="CASCADE"), primary_key=True
|
||||||
|
)
|
||||||
|
scene_count: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||||
|
|
@ -23,7 +23,7 @@ import logging
|
||||||
from sqlalchemy import text
|
from sqlalchemy import text
|
||||||
|
|
||||||
from app.db import session_scope
|
from app.db import session_scope
|
||||||
from app.verticals import ALL_VERTICAL_ORIGINS
|
from app.verticals import ALL_VERTICAL_ORIGINS, VERTICALS
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
@ -129,11 +129,35 @@ _STUDIOS_SQL = text(
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# Tagi wertykalu (lista w zakladce JAV/animacja): sceny z zywym zrodlem z jego originow,
|
||||||
|
# bez progu dlugosci, ta sama definicja co dawny agregat w /tags?vertical=. Podmiana
|
||||||
|
# calego kompletu w jednej transakcji, to kilka tysiecy wierszy (migracja 0047).
|
||||||
|
_VERTICAL_TAGS_DELETE_SQL = text("DELETE FROM tag_vertical_counts WHERE vertical = :vertical")
|
||||||
|
_VERTICAL_TAGS_INSERT_SQL = text(
|
||||||
|
"""
|
||||||
|
INSERT INTO tag_vertical_counts (vertical, tag_id, scene_count)
|
||||||
|
SELECT :vertical, st.tag_id, count(*)
|
||||||
|
FROM scene_tags st
|
||||||
|
JOIN (
|
||||||
|
SELECT DISTINCT ps.scene_id FROM playback_sources ps
|
||||||
|
WHERE ps.dead_at IS NULL AND ps.origin = ANY(:origins)
|
||||||
|
) v ON v.scene_id = st.scene_id
|
||||||
|
GROUP BY st.tag_id
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def refresh_taxonomy_counts() -> dict[str, int]:
|
def refresh_taxonomy_counts() -> dict[str, int]:
|
||||||
"""Przelicza scene_count dla tags/performers/studios. Zwraca rowcount per tabela
|
"""Przelicza scene_count dla tags/performers/studios. Zwraca rowcount per tabela
|
||||||
(ile wierszy faktycznie się zmieniło)."""
|
(ile wierszy faktycznie się zmieniło)."""
|
||||||
out: dict[str, int] = {}
|
out: dict[str, int] = {}
|
||||||
with session_scope() as session:
|
with session_scope() as session:
|
||||||
|
for vertical, origins in VERTICALS.items():
|
||||||
|
session.execute(_VERTICAL_TAGS_DELETE_SQL, {"vertical": vertical})
|
||||||
|
out[f"tags_{vertical}"] = session.execute(
|
||||||
|
_VERTICAL_TAGS_INSERT_SQL, {"vertical": vertical, "origins": list(origins)}
|
||||||
|
).rowcount or 0
|
||||||
|
session.commit()
|
||||||
for name, stmt in (
|
for name, stmt in (
|
||||||
("tags", _TAGS_SQL),
|
("tags", _TAGS_SQL),
|
||||||
("performers", _PERFORMERS_SQL),
|
("performers", _PERFORMERS_SQL),
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue