"""Wypelnia playback_sources.sort_date w istniejacych wierszach (migracja 0048). Kopia scripts/backfill_scene_tags_sort_date.py dla playback_sources: partiami po zakresach stron heapu, commit po kazdej partii, VACUUM co kilka partii, pauza oddaje dysk apce. Nowe wiersze w trakcie wypelnia juz wyzwalacz z 0048. Wznawialny: rusza tylko wiersze z sort_date IS NULL, na koncu dobiera resztki. docker compose run -d --name backfill_ps_sort worker \ python3 -u /srv/scripts/backfill_playback_sort_date.py [stron_na_partie] [pauza_s] Po nim: alembic upgrade (0049, indeks) i dopiero wtedy kod, ktory z indeksu korzysta. """ from __future__ import annotations import sys import time from sqlalchemy import text from app.db import engine PAGES = int(sys.argv[1]) if len(sys.argv) > 1 else 1000 PAUSE = float(sys.argv[2]) if len(sys.argv) > 2 else 10.0 VACUUM_EVERY = 8 SORT_DATE = "COALESCE(s.release_date, (s.created_at AT TIME ZONE 'UTC')::date)" def rel_pages() -> int: with engine.connect() as c: return int(c.execute(text( "SELECT pg_relation_size('playback_sources') / current_setting('block_size')::int" )).scalar()) def vacuum(analyze: bool = False) -> None: with engine.connect().execution_options(isolation_level="AUTOCOMMIT") as c: c.execute(text("VACUUM (ANALYZE) playback_sources" if analyze else "VACUUM playback_sources")) t0 = time.time() total = 0 page = 0 batch = 0 pages = rel_pages() print(f"playback_sources: {pages} stron, partia {PAGES} stron, pauza {PAUSE} s", flush=True) while page < pages: tb = time.time() with engine.begin() as c: c.execute(text("SET LOCAL statement_timeout = '600s'")) n = c.execute(text(f""" UPDATE playback_sources ps SET sort_date = {SORT_DATE} FROM scenes s WHERE s.id = ps.scene_id AND ps.sort_date IS NULL AND ps.ctid >= format('(%s,0)', CAST(:p0 AS int))::tid AND ps.ctid < format('(%s,0)', CAST(:p1 AS int))::tid """), {"p0": page, "p1": page + PAGES}).rowcount total += n page += PAGES batch += 1 if batch % VACUUM_EVERY == 0: vacuum() pages = rel_pages() # tabela moze urosnac, zanim VACUUM zwolni miejsce print(f"partia {batch}: strony do {page}/{pages}, {n} wierszy w {time.time() - tb:.1f} s, " f"razem {total}, {time.time() - t0:.0f} s", flush=True) if n: time.sleep(PAUSE) with engine.begin() as c: rest = c.execute(text(f""" UPDATE playback_sources ps SET sort_date = {SORT_DATE} FROM scenes s WHERE s.id = ps.scene_id AND ps.sort_date IS NULL """)).rowcount print(f"resztki: {rest}", flush=True) vacuum(analyze=True) with engine.connect() as c: left = c.execute(text("SELECT count(*) FROM playback_sources WHERE sort_date IS NULL")).scalar() print(f"KONIEC: {total + rest} wierszy, puste po wszystkim: {left}, {time.time() - t0:.0f} s", flush=True)