fix: source-stats z probki scen, nizsze progi autovacuum
source-stats liczyl flagi tag/obsada dla 2,5 mln par serwis-scena, czyli ok. 5 mln losowych odczytow indeksow; trwal 22-32 min i przekroczyl limit (GOON-V). Teraz flagi z probki 3000 scen na serwis (stala, po md5), liczba scen, swiezosc i miniatury dalej dokladnie. 1 min 51 s, gwiazdki bez zmian, procenty +-2 pkt. Autovacuum na domyslnych 20% nie ruszal duzych tabel od tygodni (scene_performers 11% stron widocznych, statystyki scenes z 25.08). Migracja 0045 daje 2% dla szesciu najwiekszych tabel. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
goon-foss
parent
55adf8b1cb
commit
d3713d64b9
2 changed files with 109 additions and 28 deletions
52
alembic/versions/20260921_0045_autovacuum_big_tables.py
Normal file
52
alembic/versions/20260921_0045_autovacuum_big_tables.py
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
"""Nizsze progi autovacuum i autoanalyze dla duzych tabel.
|
||||
|
||||
Domyslne progi (20% zmienionych wierszy do vacuum, 10% do analyze) przy 4-23 mln wierszy
|
||||
znacza, ze autovacuum prawie nie rusza tych tabel: 2026-09-21 scene_performers mialo
|
||||
ostatni przebieg 27.07 i 11% stron oznaczonych jako widoczne, playback_sources 24%,
|
||||
a statystyki scenes byly z 25.08. Skutek: index-only scan i tak czyta sterte (w
|
||||
source-stats 94% odczytow scene_performers szlo na dysk, job przekroczyl 30 min,
|
||||
Sentry GOON-V), a planista myli sie przy szukaniu po tytule (GOON-73).
|
||||
|
||||
2% daje przebieg co ~100 tys. zmian. Autovacuum jest dlawiony (cost_delay 2 ms), wiec
|
||||
czesciej znaczy krocej, nie ciezej. SET na parametrach tabeli bierze tylko
|
||||
SHARE UPDATE EXCLUSIVE, nie blokuje odczytow ani zapisow.
|
||||
|
||||
Revision ID: 0045_autovacuum_big_tables
|
||||
Revises: 0044_playback_eporner_thumb_idx
|
||||
Create Date: 2026-09-21
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision = "0045_autovacuum_big_tables"
|
||||
down_revision: str | None = "0044_playback_eporner_thumb_idx"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
_TABLES = (
|
||||
"scenes",
|
||||
"playback_sources",
|
||||
"scene_performers",
|
||||
"scene_tags",
|
||||
"scene_external_refs",
|
||||
"external_records",
|
||||
)
|
||||
_PARAMS = (
|
||||
"autovacuum_vacuum_scale_factor",
|
||||
"autovacuum_vacuum_insert_scale_factor",
|
||||
"autovacuum_analyze_scale_factor",
|
||||
)
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.execute("SET LOCAL lock_timeout = '5s'")
|
||||
for table in _TABLES:
|
||||
params = ", ".join(f"{p} = 0.02" for p in _PARAMS)
|
||||
op.execute(f"ALTER TABLE {table} SET ({params})")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.execute("SET LOCAL lock_timeout = '5s'")
|
||||
for table in _TABLES:
|
||||
op.execute(f"ALTER TABLE {table} RESET ({', '.join(_PARAMS)})")
|
||||
|
|
@ -33,6 +33,12 @@ _TELEMETRY_WINDOW_DAYS = 7
|
|||
# mniejszej próbce lecimy na proxy/heurystykę (nie zerujemy gwiazdek znanemu-dobremu).
|
||||
_TELEMETRY_MIN_ATTEMPTS = 25
|
||||
_EVENT_RETENTION_DAYS = 30
|
||||
# Flagi tag/perf/desc/studio/dur liczone z probki scen na origin, nie ze wszystkich.
|
||||
# Pelny przebieg to ~5 mln losowych odczytow indeksow scene_tags/scene_performers
|
||||
# (2,5 mln par origin-scena) i 22-32 min; 2026-09-21 przekroczyl 30 min limitu
|
||||
# (Sentry GOON-V). Probka po md5(scene_id) jest stala miedzy przebiegami, wiec
|
||||
# gwiazdki nie skacza; blad procentu przy 3000 scen to ok. 1 pkt.
|
||||
_RICHNESS_SAMPLE = 3000
|
||||
# Richness: wagi składowych (suma=1.0). thumb to minimum higieny, canonical-bogactwo
|
||||
# (desc/studio/tag/perf) waży więcej. dur średnio.
|
||||
_RICHNESS_WEIGHTS = {
|
||||
|
|
@ -137,43 +143,65 @@ def run_source_stats() -> dict:
|
|||
win_start = now - timedelta(days=_TELEMETRY_WINDOW_DAYS)
|
||||
|
||||
with session_scope() as session:
|
||||
# 1) Freshness + richness: agregat po DISTINCT (origin, scene) z live sources,
|
||||
# join do flag scen (has_tag/perf/desc/studio/dur policzone raz).
|
||||
# 1a) Freshness + miniatury: dokladnie, po DISTINCT (origin, scene) z live sources.
|
||||
rows = session.execute(
|
||||
text(
|
||||
"""
|
||||
WITH scene_flags AS (
|
||||
SELECT s.id,
|
||||
(s.description IS NOT NULL AND length(btrim(s.description))>0) AS has_desc,
|
||||
(s.studio_id IS NOT NULL) AS has_studio,
|
||||
(s.duration_sec IS NOT NULL) AS has_dur,
|
||||
EXISTS(SELECT 1 FROM scene_tags st WHERE st.scene_id=s.id) AS has_tag,
|
||||
EXISTS(SELECT 1 FROM scene_performers sp WHERE sp.scene_id=s.id) AS has_perf
|
||||
FROM scenes s
|
||||
),
|
||||
live AS (
|
||||
WITH live AS (
|
||||
SELECT DISTINCT ON (origin, scene_id) origin, scene_id,
|
||||
(thumbnail_url IS NOT NULL) AS has_thumb, created_at
|
||||
FROM playback_sources
|
||||
WHERE dead_at IS NULL AND origin LIKE 'tube:%'
|
||||
)
|
||||
SELECT l.origin,
|
||||
SELECT origin,
|
||||
count(*) AS scenes,
|
||||
count(*) FILTER (WHERE l.created_at > :win) AS new_7d,
|
||||
max(l.created_at) AS newest_at,
|
||||
100.0*avg(l.has_thumb::int) AS thumb,
|
||||
100.0*avg(f.has_tag::int) AS tag,
|
||||
100.0*avg(f.has_perf::int) AS perf,
|
||||
100.0*avg(f.has_desc::int) AS descr,
|
||||
100.0*avg(f.has_studio::int) AS studio,
|
||||
100.0*avg(f.has_dur::int) AS dur
|
||||
FROM live l JOIN scene_flags f ON f.id = l.scene_id
|
||||
GROUP BY l.origin
|
||||
count(*) FILTER (WHERE created_at > :win) AS new_7d,
|
||||
max(created_at) AS newest_at,
|
||||
100.0*avg(has_thumb::int) AS thumb
|
||||
FROM live
|
||||
GROUP BY origin
|
||||
"""
|
||||
),
|
||||
{"win": win_start},
|
||||
).all()
|
||||
|
||||
# 1b) Reszta richness z probki scen (patrz _RICHNESS_SAMPLE).
|
||||
flags = {
|
||||
r.origin: r
|
||||
for r in session.execute(
|
||||
text(
|
||||
"""
|
||||
WITH live AS (
|
||||
SELECT DISTINCT origin, scene_id
|
||||
FROM playback_sources
|
||||
WHERE dead_at IS NULL AND origin LIKE 'tube:%'
|
||||
),
|
||||
sample AS (
|
||||
SELECT origin, scene_id FROM (
|
||||
SELECT origin, scene_id, row_number() OVER (
|
||||
PARTITION BY origin ORDER BY md5(scene_id::text)
|
||||
) AS rn
|
||||
FROM live
|
||||
) ranked
|
||||
WHERE rn <= :n
|
||||
)
|
||||
SELECT sm.origin,
|
||||
100.0*avg((EXISTS(SELECT 1 FROM scene_tags st
|
||||
WHERE st.scene_id=s.id))::int) AS tag,
|
||||
100.0*avg((EXISTS(SELECT 1 FROM scene_performers sp
|
||||
WHERE sp.scene_id=s.id))::int) AS perf,
|
||||
100.0*avg((s.description IS NOT NULL
|
||||
AND length(btrim(s.description))>0)::int) AS descr,
|
||||
100.0*avg((s.studio_id IS NOT NULL)::int) AS studio,
|
||||
100.0*avg((s.duration_sec IS NOT NULL)::int) AS dur
|
||||
FROM sample sm JOIN scenes s ON s.id = sm.scene_id
|
||||
GROUP BY sm.origin
|
||||
"""
|
||||
),
|
||||
{"n": _RICHNESS_SAMPLE},
|
||||
).all()
|
||||
}
|
||||
|
||||
# 2) Telemetria health per origin (okno 7d).
|
||||
tele = {
|
||||
r.origin: r
|
||||
|
|
@ -198,13 +226,14 @@ def run_source_stats() -> dict:
|
|||
for r in rows:
|
||||
origin = r.origin
|
||||
sitetag = origin.split(":", 1)[1] if ":" in origin else origin
|
||||
f = flags.get(origin)
|
||||
pcts = {
|
||||
"thumb": float(r.thumb or 0),
|
||||
"tag": float(r.tag or 0),
|
||||
"perf": float(r.perf or 0),
|
||||
"desc": float(r.descr or 0),
|
||||
"studio": float(r.studio or 0),
|
||||
"dur": float(r.dur or 0),
|
||||
"tag": float(f.tag or 0) if f else 0.0,
|
||||
"perf": float(f.perf or 0) if f else 0.0,
|
||||
"desc": float(f.descr or 0) if f else 0.0,
|
||||
"studio": float(f.studio or 0) if f else 0.0,
|
||||
"dur": float(f.dur or 0) if f else 0.0,
|
||||
}
|
||||
freshness = _freshness_score(r.newest_at, int(r.new_7d), int(r.scenes))
|
||||
richness = _richness_score(pcts)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue