From d3713d64b962e8d3e017d54a5b4d4931fd9bdf5e Mon Sep 17 00:00:00 2001 From: goon-foss Date: Mon, 21 Sep 2026 20:28:23 +0200 Subject: [PATCH] fix: source-stats z probki scen, nizsze progi autovacuum source-stats liczyl flagi tag/obsada dla 2,5 mln par serwis-scena, czyli ok. 5 mln losowych odczytow indeksow; trwal 22-32 min i przekroczyl limit (GOON-V). Teraz flagi z probki 3000 scen na serwis (stala, po md5), liczba scen, swiezosc i miniatury dalej dokladnie. 1 min 51 s, gwiazdki bez zmian, procenty +-2 pkt. Autovacuum na domyslnych 20% nie ruszal duzych tabel od tygodni (scene_performers 11% stron widocznych, statystyki scenes z 25.08). Migracja 0045 daje 2% dla szesciu najwiekszych tabel. Co-Authored-By: Claude Opus 5 --- .../20260921_0045_autovacuum_big_tables.py | 52 ++++++++++++ app/scheduler/source_stats.py | 85 +++++++++++++------ 2 files changed, 109 insertions(+), 28 deletions(-) create mode 100644 alembic/versions/20260921_0045_autovacuum_big_tables.py diff --git a/alembic/versions/20260921_0045_autovacuum_big_tables.py b/alembic/versions/20260921_0045_autovacuum_big_tables.py new file mode 100644 index 0000000..88b0f27 --- /dev/null +++ b/alembic/versions/20260921_0045_autovacuum_big_tables.py @@ -0,0 +1,52 @@ +"""Nizsze progi autovacuum i autoanalyze dla duzych tabel. + +Domyslne progi (20% zmienionych wierszy do vacuum, 10% do analyze) przy 4-23 mln wierszy +znacza, ze autovacuum prawie nie rusza tych tabel: 2026-09-21 scene_performers mialo +ostatni przebieg 27.07 i 11% stron oznaczonych jako widoczne, playback_sources 24%, +a statystyki scenes byly z 25.08. Skutek: index-only scan i tak czyta sterte (w +source-stats 94% odczytow scene_performers szlo na dysk, job przekroczyl 30 min, +Sentry GOON-V), a planista myli sie przy szukaniu po tytule (GOON-73). + +2% daje przebieg co ~100 tys. zmian. Autovacuum jest dlawiony (cost_delay 2 ms), wiec +czesciej znaczy krocej, nie ciezej. SET na parametrach tabeli bierze tylko +SHARE UPDATE EXCLUSIVE, nie blokuje odczytow ani zapisow. + +Revision ID: 0045_autovacuum_big_tables +Revises: 0044_playback_eporner_thumb_idx +Create Date: 2026-09-21 +""" +from __future__ import annotations + +from alembic import op + +revision = "0045_autovacuum_big_tables" +down_revision: str | None = "0044_playback_eporner_thumb_idx" +branch_labels = None +depends_on = None + +_TABLES = ( + "scenes", + "playback_sources", + "scene_performers", + "scene_tags", + "scene_external_refs", + "external_records", +) +_PARAMS = ( + "autovacuum_vacuum_scale_factor", + "autovacuum_vacuum_insert_scale_factor", + "autovacuum_analyze_scale_factor", +) + + +def upgrade() -> None: + op.execute("SET LOCAL lock_timeout = '5s'") + for table in _TABLES: + params = ", ".join(f"{p} = 0.02" for p in _PARAMS) + op.execute(f"ALTER TABLE {table} SET ({params})") + + +def downgrade() -> None: + op.execute("SET LOCAL lock_timeout = '5s'") + for table in _TABLES: + op.execute(f"ALTER TABLE {table} RESET ({', '.join(_PARAMS)})") diff --git a/app/scheduler/source_stats.py b/app/scheduler/source_stats.py index f629f43..817b4a6 100644 --- a/app/scheduler/source_stats.py +++ b/app/scheduler/source_stats.py @@ -33,6 +33,12 @@ _TELEMETRY_WINDOW_DAYS = 7 # mniejszej próbce lecimy na proxy/heurystykę (nie zerujemy gwiazdek znanemu-dobremu). _TELEMETRY_MIN_ATTEMPTS = 25 _EVENT_RETENTION_DAYS = 30 +# Flagi tag/perf/desc/studio/dur liczone z probki scen na origin, nie ze wszystkich. +# Pelny przebieg to ~5 mln losowych odczytow indeksow scene_tags/scene_performers +# (2,5 mln par origin-scena) i 22-32 min; 2026-09-21 przekroczyl 30 min limitu +# (Sentry GOON-V). Probka po md5(scene_id) jest stala miedzy przebiegami, wiec +# gwiazdki nie skacza; blad procentu przy 3000 scen to ok. 1 pkt. +_RICHNESS_SAMPLE = 3000 # Richness: wagi składowych (suma=1.0). thumb to minimum higieny, canonical-bogactwo # (desc/studio/tag/perf) waży więcej. dur średnio. _RICHNESS_WEIGHTS = { @@ -137,43 +143,65 @@ def run_source_stats() -> dict: win_start = now - timedelta(days=_TELEMETRY_WINDOW_DAYS) with session_scope() as session: - # 1) Freshness + richness: agregat po DISTINCT (origin, scene) z live sources, - # join do flag scen (has_tag/perf/desc/studio/dur policzone raz). + # 1a) Freshness + miniatury: dokladnie, po DISTINCT (origin, scene) z live sources. rows = session.execute( text( """ - WITH scene_flags AS ( - SELECT s.id, - (s.description IS NOT NULL AND length(btrim(s.description))>0) AS has_desc, - (s.studio_id IS NOT NULL) AS has_studio, - (s.duration_sec IS NOT NULL) AS has_dur, - EXISTS(SELECT 1 FROM scene_tags st WHERE st.scene_id=s.id) AS has_tag, - EXISTS(SELECT 1 FROM scene_performers sp WHERE sp.scene_id=s.id) AS has_perf - FROM scenes s - ), - live AS ( + WITH live AS ( SELECT DISTINCT ON (origin, scene_id) origin, scene_id, (thumbnail_url IS NOT NULL) AS has_thumb, created_at FROM playback_sources WHERE dead_at IS NULL AND origin LIKE 'tube:%' ) - SELECT l.origin, + SELECT origin, count(*) AS scenes, - count(*) FILTER (WHERE l.created_at > :win) AS new_7d, - max(l.created_at) AS newest_at, - 100.0*avg(l.has_thumb::int) AS thumb, - 100.0*avg(f.has_tag::int) AS tag, - 100.0*avg(f.has_perf::int) AS perf, - 100.0*avg(f.has_desc::int) AS descr, - 100.0*avg(f.has_studio::int) AS studio, - 100.0*avg(f.has_dur::int) AS dur - FROM live l JOIN scene_flags f ON f.id = l.scene_id - GROUP BY l.origin + count(*) FILTER (WHERE created_at > :win) AS new_7d, + max(created_at) AS newest_at, + 100.0*avg(has_thumb::int) AS thumb + FROM live + GROUP BY origin """ ), {"win": win_start}, ).all() + # 1b) Reszta richness z probki scen (patrz _RICHNESS_SAMPLE). + flags = { + r.origin: r + for r in session.execute( + text( + """ + WITH live AS ( + SELECT DISTINCT origin, scene_id + FROM playback_sources + WHERE dead_at IS NULL AND origin LIKE 'tube:%' + ), + sample AS ( + SELECT origin, scene_id FROM ( + SELECT origin, scene_id, row_number() OVER ( + PARTITION BY origin ORDER BY md5(scene_id::text) + ) AS rn + FROM live + ) ranked + WHERE rn <= :n + ) + SELECT sm.origin, + 100.0*avg((EXISTS(SELECT 1 FROM scene_tags st + WHERE st.scene_id=s.id))::int) AS tag, + 100.0*avg((EXISTS(SELECT 1 FROM scene_performers sp + WHERE sp.scene_id=s.id))::int) AS perf, + 100.0*avg((s.description IS NOT NULL + AND length(btrim(s.description))>0)::int) AS descr, + 100.0*avg((s.studio_id IS NOT NULL)::int) AS studio, + 100.0*avg((s.duration_sec IS NOT NULL)::int) AS dur + FROM sample sm JOIN scenes s ON s.id = sm.scene_id + GROUP BY sm.origin + """ + ), + {"n": _RICHNESS_SAMPLE}, + ).all() + } + # 2) Telemetria health per origin (okno 7d). tele = { r.origin: r @@ -198,13 +226,14 @@ def run_source_stats() -> dict: for r in rows: origin = r.origin sitetag = origin.split(":", 1)[1] if ":" in origin else origin + f = flags.get(origin) pcts = { "thumb": float(r.thumb or 0), - "tag": float(r.tag or 0), - "perf": float(r.perf or 0), - "desc": float(r.descr or 0), - "studio": float(r.studio or 0), - "dur": float(r.dur or 0), + "tag": float(f.tag or 0) if f else 0.0, + "perf": float(f.perf or 0) if f else 0.0, + "desc": float(f.descr or 0) if f else 0.0, + "studio": float(f.studio or 0) if f else 0.0, + "dur": float(f.dur or 0) if f else 0.0, } freshness = _freshness_score(r.newest_at, int(r.new_7d), int(r.scenes)) richness = _richness_score(pcts)