perf(api): lista stron odswiezana w tle, nikt nie czeka na przeliczenie
Agregat po 4,1 mln zywych zrodel trwa 0,6-2 s, a cache wygasal co 5 minut, wiec co wygasniecie ktos placil pelnym czasem (Sentry Slow DB Query list_sources, 186 zdarzen, ostatnie dzisiaj). Teraz po wygasnieciu oddajemy stary wynik i przeliczamy w watku w tle, a TTL idzie z 5 do 15 minut. Liczby zmieniaja sie o promile na godzine. Pomiar na prodzie: pierwsze wejscie po restarcie 2,1 s (nieuniknione), potem 11-15 ms, a wejscie tuz po wygasnieciu 38 ms zamiast pelnego przeliczenia. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
be0f2d7d37
commit
3c1d2c5907
1 changed files with 54 additions and 19 deletions
|
|
@ -16,6 +16,7 @@ from __future__ import annotations
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
|
import threading
|
||||||
import time
|
import time
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
@ -27,7 +28,7 @@ from sqlalchemy import func, select
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from app.auth import require_api_key
|
from app.auth import require_api_key
|
||||||
from app.db import get_session
|
from app.db import get_session, session_scope
|
||||||
from app.models.playback_source import PlaybackSource
|
from app.models.playback_source import PlaybackSource
|
||||||
from app.models.source_stats import SourceStats
|
from app.models.source_stats import SourceStats
|
||||||
|
|
||||||
|
|
@ -86,8 +87,10 @@ class SourceOut(BaseModel):
|
||||||
|
|
||||||
|
|
||||||
_ICON_DIR = Path(__file__).resolve().parent.parent / "static" / "site-icons"
|
_ICON_DIR = Path(__file__).resolve().parent.parent / "static" / "site-icons"
|
||||||
_SOURCES_TTL = 300.0
|
#: Po tylu sekundach wynik uznajemy za stary i przeliczamy W TLE (oddajac stary).
|
||||||
|
_SOURCES_TTL = 900.0
|
||||||
_SOURCES_CACHE: dict = {"at": 0.0, "rows": None}
|
_SOURCES_CACHE: dict = {"at": 0.0, "rows": None}
|
||||||
|
_SOURCES_LOCK = threading.Lock()
|
||||||
|
|
||||||
|
|
||||||
def _icon_url(sitetag: str) -> str | None:
|
def _icon_url(sitetag: str) -> str | None:
|
||||||
|
|
@ -134,20 +137,8 @@ def _sitetag_to_display(sitetag: str) -> str:
|
||||||
return sitetag
|
return sitetag
|
||||||
|
|
||||||
|
|
||||||
@router.get("", response_model=SourceListOut)
|
def _source_rows(session: Session):
|
||||||
def list_sources(
|
return session.execute(
|
||||||
session: Annotated[Session, Depends(get_session)],
|
|
||||||
) -> SourceListOut:
|
|
||||||
"""Zwraca listę tube źródeł z ŻYWYMI playback_sources.
|
|
||||||
|
|
||||||
Filter: `origin LIKE 'tube:%'` (drop canonical:* TPDB trailery to inna semantyka).
|
|
||||||
"""
|
|
||||||
# Agregat idzie po 4,1 mln zywych zrodel (600 ms na kazde wejscie w Sites, Sentry:
|
|
||||||
# Slow DB Query list_sources). Liczby zmieniaja sie o promile na godzine, wiec
|
|
||||||
# wynik zyje w pamieci procesu przez _SOURCES_TTL sekund.
|
|
||||||
now = time.monotonic()
|
|
||||||
if _SOURCES_CACHE["rows"] is None or now - _SOURCES_CACHE["at"] > _SOURCES_TTL:
|
|
||||||
_SOURCES_CACHE["rows"] = session.execute(
|
|
||||||
select(
|
select(
|
||||||
PlaybackSource.origin,
|
PlaybackSource.origin,
|
||||||
func.count(PlaybackSource.id).label("scene_count"),
|
func.count(PlaybackSource.id).label("scene_count"),
|
||||||
|
|
@ -157,8 +148,52 @@ def list_sources(
|
||||||
.where(PlaybackSource.origin.like("tube:%"))
|
.where(PlaybackSource.origin.like("tube:%"))
|
||||||
.group_by(PlaybackSource.origin)
|
.group_by(PlaybackSource.origin)
|
||||||
).all()
|
).all()
|
||||||
_SOURCES_CACHE["at"] = now
|
|
||||||
|
|
||||||
|
def _refresh_source_rows_bg() -> None:
|
||||||
|
if not _SOURCES_LOCK.acquire(blocking=False):
|
||||||
|
return # ktos juz liczy
|
||||||
|
|
||||||
|
def run() -> None:
|
||||||
|
try:
|
||||||
|
with session_scope() as s:
|
||||||
|
rows = _source_rows(s)
|
||||||
|
_SOURCES_CACHE["rows"] = rows
|
||||||
|
_SOURCES_CACHE["at"] = time.monotonic()
|
||||||
|
except Exception: # noqa: BLE001
|
||||||
|
log.exception("sources: odswiezenie listy stron w tle nie wyszlo")
|
||||||
|
finally:
|
||||||
|
_SOURCES_LOCK.release()
|
||||||
|
|
||||||
|
threading.Thread(target=run, name="sources-refresh", daemon=True).start()
|
||||||
|
|
||||||
|
|
||||||
|
def _cached_source_rows(session: Session):
|
||||||
rows = _SOURCES_CACHE["rows"]
|
rows = _SOURCES_CACHE["rows"]
|
||||||
|
if rows is None:
|
||||||
|
with _SOURCES_LOCK:
|
||||||
|
if _SOURCES_CACHE["rows"] is None:
|
||||||
|
_SOURCES_CACHE["rows"] = _source_rows(session)
|
||||||
|
_SOURCES_CACHE["at"] = time.monotonic()
|
||||||
|
return _SOURCES_CACHE["rows"]
|
||||||
|
if time.monotonic() - _SOURCES_CACHE["at"] > _SOURCES_TTL:
|
||||||
|
_refresh_source_rows_bg()
|
||||||
|
return rows
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("", response_model=SourceListOut)
|
||||||
|
def list_sources(
|
||||||
|
session: Annotated[Session, Depends(get_session)],
|
||||||
|
) -> SourceListOut:
|
||||||
|
"""Zwraca listę tube źródeł z ŻYWYMI playback_sources.
|
||||||
|
|
||||||
|
Filter: `origin LIKE 'tube:%'` (drop canonical:* TPDB trailery to inna semantyka).
|
||||||
|
"""
|
||||||
|
# Agregat idzie po 4,1 mln zywych zrodel (0,6-1,6 s na przeliczenie, Sentry: Slow DB
|
||||||
|
# Query list_sources, 186 zdarzen). Liczby zmieniaja sie o promile na godzine, wiec
|
||||||
|
# wynik zyje w pamieci procesu, a po wygasnieciu oddajemy STARY i przeliczamy w tle.
|
||||||
|
# Wczesniej co wygasniecie cache ktos placil pelnym czasem zapytania (2026-09-17).
|
||||||
|
rows = _cached_source_rows(session)
|
||||||
|
|
||||||
# Oceny z source_stats (policzone offline przez run_source_stats). Origin → row.
|
# Oceny z source_stats (policzone offline przez run_source_stats). Origin → row.
|
||||||
stats = {s.origin: s for s in session.execute(select(SourceStats)).scalars().all()}
|
stats = {s.origin: s for s in session.execute(select(SourceStats)).scalars().all()}
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue