diff --git a/alembic/versions/20260927_0047_tag_vertical_counts.py b/alembic/versions/20260927_0047_tag_vertical_counts.py new file mode 100644 index 0000000..0c32f6b --- /dev/null +++ b/alembic/versions/20260927_0047_tag_vertical_counts.py @@ -0,0 +1,41 @@ +"""Liczniki tagow per wertykal (JAV, animacja), liczone w tle zamiast na kazde wejscie. + +/tags?vertical=jav liczyl na zywo: sceny z zywym zrodlem JAV, ich scene_tags, GROUP BY, +i to dwa razy (licznik total i strona). Na rozgrzanym cache 1 s, na zimnym 17-24 s +(log API 2026-09-27, zakladka JAV w apce). Tabela (vertical, tag_id) zamiast kolumny +na `tags` na kazdy wertykal, zeby nowy wertykal nie wymagal migracji. Wypelnia ja +refresh_taxonomy_counts (co 3 h); dopoki dla wertykalu nie ma wierszy, API liczy po staremu. + +Revision ID: 0047_tag_vertical_counts +Revises: 0046_scenes_title_stats +Create Date: 2026-09-27 +""" +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects.postgresql import UUID + +revision = "0047_tag_vertical_counts" +down_revision: str | None = "0046_scenes_title_stats" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "tag_vertical_counts", + sa.Column("vertical", sa.String(32), nullable=False), + sa.Column("tag_id", UUID(as_uuid=True), nullable=False), + sa.Column("scene_count", sa.Integer, nullable=False), + sa.PrimaryKeyConstraint("vertical", "tag_id", name="pk_tag_vertical_counts"), + sa.ForeignKeyConstraint( + ["tag_id"], ["tags.id"], + name="fk_tag_vertical_counts_tag_id_tags", + ondelete="CASCADE", + ), + ) + + +def downgrade() -> None: + op.drop_table("tag_vertical_counts") diff --git a/app/api/taxonomies.py b/app/api/taxonomies.py index c444d74..78ff2ab 100644 --- a/app/api/taxonomies.py +++ b/app/api/taxonomies.py @@ -27,6 +27,7 @@ from app.models.playback_source import PlaybackSource from app.models.scene import ScenePerformer, SceneTag from app.models.studio import Studio from app.models.tag import Tag +from app.models.tag_vertical_count import TagVerticalCount from app.verticals import VERTICALS log = logging.getLogger(__name__) @@ -158,25 +159,36 @@ def list_tags( return TagListOut(items=items, total=total, page=page, per_page=per_page) if vertical in VERTICALS: - # Wertykał liczymy NA ŻYWO, bez denormalizacji: to małe zbiory (animacja ~9,5 tys. - # scen wobec 3,8 mln całości), więc agregat jest tani, a dokładanie kolumny - # `scene_count_` do `tags` kosztowałoby migrację przy każdym nowym - # wertykale. Zdenormalizowany `Tag.scene_count` opisuje WYŁĄCZNIE główny feed. - v_scenes = ( - select(PlaybackSource.scene_id) - .where( - PlaybackSource.dead_at.is_(None), - PlaybackSource.origin.in_(VERTICALS[vertical]), + # Liczniki wertykału czytamy z tag_vertical_counts (refresh_taxonomy_counts co 3 h, + # migracja 0047). Agregat na żywo trwał 1 s na rozgrzanym cache i 17-24 s na + # zimnym (zakładka JAV, 2026-09-27). Zostaje jako zapas, gdy tabela jest jeszcze + # pusta dla tego wertykału, np. zaraz po wdrożeniu. + # Zdenormalizowany `Tag.scene_count` opisuje WYŁĄCZNIE główny feed. + has_counts = session.execute( + select(TagVerticalCount.tag_id).where(TagVerticalCount.vertical == vertical).limit(1) + ).first() + if has_counts: + v_count = ( + select(TagVerticalCount.tag_id, TagVerticalCount.scene_count.label("c")) + .where(TagVerticalCount.vertical == vertical) + .subquery() + ) + else: + v_scenes = ( + select(PlaybackSource.scene_id) + .where( + PlaybackSource.dead_at.is_(None), + PlaybackSource.origin.in_(VERTICALS[vertical]), + ) + .distinct() + .subquery() + ) + v_count = ( + select(SceneTag.tag_id, func.count().label("c")) + .join(v_scenes, v_scenes.c.scene_id == SceneTag.scene_id) + .group_by(SceneTag.tag_id) + .subquery() ) - .distinct() - .subquery() - ) - v_count = ( - select(SceneTag.tag_id, func.count().label("c")) - .join(v_scenes, v_scenes.c.scene_id == SceneTag.scene_id) - .group_by(SceneTag.tag_id) - .subquery() - ) base = ( select(Tag, func.coalesce(v_count.c.c, 0).label("scene_count")) .outerjoin(v_count, v_count.c.tag_id == Tag.id) diff --git a/app/models/__init__.py b/app/models/__init__.py index 89a7c8d..9ba6619 100644 --- a/app/models/__init__.py +++ b/app/models/__init__.py @@ -38,6 +38,7 @@ from app.models.scene import ( from app.models.source import Source from app.models.studio import Studio, StudioAlias, StudioExternalRef from app.models.tag import Tag +from app.models.tag_vertical_count import TagVerticalCount __all__ = [ "Base", @@ -84,4 +85,5 @@ __all__ = [ "StudioAlias", "StudioExternalRef", "Tag", + "TagVerticalCount", ] diff --git a/app/models/tag_vertical_count.py b/app/models/tag_vertical_count.py new file mode 100644 index 0000000..a3e19f5 --- /dev/null +++ b/app/models/tag_vertical_count.py @@ -0,0 +1,25 @@ +"""Liczba scen tagu w wertykale (JAV, animacja), odswiezana w tle. + +Wypelnia `refresh_taxonomy_counts`, czyta /tags?vertical=. Ta sama definicja co dawny +agregat na zywo: sceny z >= 1 zywym zrodlem z originow wertykalu, bez progu dlugosci. +Patrz migracja 0047. +""" +from __future__ import annotations + +import uuid + +from sqlalchemy import ForeignKey, Integer, String +from sqlalchemy.dialects.postgresql import UUID +from sqlalchemy.orm import Mapped, mapped_column + +from app.models.base import Base + + +class TagVerticalCount(Base): + __tablename__ = "tag_vertical_counts" + + vertical: Mapped[str] = mapped_column(String(32), primary_key=True) + tag_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), ForeignKey("tags.id", ondelete="CASCADE"), primary_key=True + ) + scene_count: Mapped[int] = mapped_column(Integer, nullable=False) diff --git a/app/scheduler/taxonomy_counts.py b/app/scheduler/taxonomy_counts.py index 071e9b2..dcc7069 100644 --- a/app/scheduler/taxonomy_counts.py +++ b/app/scheduler/taxonomy_counts.py @@ -23,7 +23,7 @@ import logging from sqlalchemy import text from app.db import session_scope -from app.verticals import ALL_VERTICAL_ORIGINS +from app.verticals import ALL_VERTICAL_ORIGINS, VERTICALS log = logging.getLogger(__name__) @@ -129,11 +129,35 @@ _STUDIOS_SQL = text( ) +# Tagi wertykalu (lista w zakladce JAV/animacja): sceny z zywym zrodlem z jego originow, +# bez progu dlugosci, ta sama definicja co dawny agregat w /tags?vertical=. Podmiana +# calego kompletu w jednej transakcji, to kilka tysiecy wierszy (migracja 0047). +_VERTICAL_TAGS_DELETE_SQL = text("DELETE FROM tag_vertical_counts WHERE vertical = :vertical") +_VERTICAL_TAGS_INSERT_SQL = text( + """ + INSERT INTO tag_vertical_counts (vertical, tag_id, scene_count) + SELECT :vertical, st.tag_id, count(*) + FROM scene_tags st + JOIN ( + SELECT DISTINCT ps.scene_id FROM playback_sources ps + WHERE ps.dead_at IS NULL AND ps.origin = ANY(:origins) + ) v ON v.scene_id = st.scene_id + GROUP BY st.tag_id + """ +) + + def refresh_taxonomy_counts() -> dict[str, int]: """Przelicza scene_count dla tags/performers/studios. Zwraca rowcount per tabela (ile wierszy faktycznie się zmieniło).""" out: dict[str, int] = {} with session_scope() as session: + for vertical, origins in VERTICALS.items(): + session.execute(_VERTICAL_TAGS_DELETE_SQL, {"vertical": vertical}) + out[f"tags_{vertical}"] = session.execute( + _VERTICAL_TAGS_INSERT_SQL, {"vertical": vertical, "origins": list(origins)} + ).rowcount or 0 + session.commit() for name, stmt in ( ("tags", _TAGS_SQL), ("performers", _PERFORMERS_SQL),