77679b26bb
Some checks failed
Backend tests / test (push) Has been cancelled
Ekran serwisu (/scenes?origin=tube:X&sort=release_date) dla serwisow, ktorych scen nie ma wsrod najnowszych w katalogu (perverzija, hdporn.gg, tubepornclassic, yesporn), sortowal wszystkie ich sceny: 9-19 s na zimnym cache, 1-4 s na cieplym, 39 wejsc w 3 h. Wzorem scene_tags.sort_date (GOON-64): data sortu powielona w playback_sources i pilnowana wyzwalaczami (0048), wypelniona skryptem, indeks (origin, sort_date DESC, scene_id DESC) WHERE dead_at IS NULL (0049). API dolacza DISTINCT ON zgodny z indeksem, bo 190 tys. scen ma dwa zywe zrodla z tego samego serwisu. Po zmianie 0,03-0,24 s, takze dalsze strony. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
76 lines
2.9 KiB
Python
76 lines
2.9 KiB
Python
"""Wypelnia playback_sources.sort_date w istniejacych wierszach (migracja 0048).
|
|
|
|
Kopia scripts/backfill_scene_tags_sort_date.py dla playback_sources: partiami po zakresach
|
|
stron heapu, commit po kazdej partii, VACUUM co kilka partii, pauza oddaje dysk apce.
|
|
Nowe wiersze w trakcie wypelnia juz wyzwalacz z 0048. Wznawialny: rusza tylko wiersze z
|
|
sort_date IS NULL, na koncu dobiera resztki.
|
|
|
|
docker compose run -d --name backfill_ps_sort worker \
|
|
python3 -u /srv/scripts/backfill_playback_sort_date.py [stron_na_partie] [pauza_s]
|
|
Po nim: alembic upgrade (0049, indeks) i dopiero wtedy kod, ktory z indeksu korzysta.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import sys
|
|
import time
|
|
|
|
from sqlalchemy import text
|
|
|
|
from app.db import engine
|
|
|
|
PAGES = int(sys.argv[1]) if len(sys.argv) > 1 else 1000
|
|
PAUSE = float(sys.argv[2]) if len(sys.argv) > 2 else 10.0
|
|
VACUUM_EVERY = 8
|
|
SORT_DATE = "COALESCE(s.release_date, (s.created_at AT TIME ZONE 'UTC')::date)"
|
|
|
|
|
|
def rel_pages() -> int:
|
|
with engine.connect() as c:
|
|
return int(c.execute(text(
|
|
"SELECT pg_relation_size('playback_sources') / current_setting('block_size')::int"
|
|
)).scalar())
|
|
|
|
|
|
def vacuum(analyze: bool = False) -> None:
|
|
with engine.connect().execution_options(isolation_level="AUTOCOMMIT") as c:
|
|
c.execute(text("VACUUM (ANALYZE) playback_sources" if analyze else "VACUUM playback_sources"))
|
|
|
|
|
|
t0 = time.time()
|
|
total = 0
|
|
page = 0
|
|
batch = 0
|
|
pages = rel_pages()
|
|
print(f"playback_sources: {pages} stron, partia {PAGES} stron, pauza {PAUSE} s", flush=True)
|
|
while page < pages:
|
|
tb = time.time()
|
|
with engine.begin() as c:
|
|
c.execute(text("SET LOCAL statement_timeout = '600s'"))
|
|
n = c.execute(text(f"""
|
|
UPDATE playback_sources ps SET sort_date = {SORT_DATE}
|
|
FROM scenes s
|
|
WHERE s.id = ps.scene_id AND ps.sort_date IS NULL
|
|
AND ps.ctid >= format('(%s,0)', CAST(:p0 AS int))::tid
|
|
AND ps.ctid < format('(%s,0)', CAST(:p1 AS int))::tid
|
|
"""), {"p0": page, "p1": page + PAGES}).rowcount
|
|
total += n
|
|
page += PAGES
|
|
batch += 1
|
|
if batch % VACUUM_EVERY == 0:
|
|
vacuum()
|
|
pages = rel_pages() # tabela moze urosnac, zanim VACUUM zwolni miejsce
|
|
print(f"partia {batch}: strony do {page}/{pages}, {n} wierszy w {time.time() - tb:.1f} s, "
|
|
f"razem {total}, {time.time() - t0:.0f} s", flush=True)
|
|
if n:
|
|
time.sleep(PAUSE)
|
|
|
|
with engine.begin() as c:
|
|
rest = c.execute(text(f"""
|
|
UPDATE playback_sources ps SET sort_date = {SORT_DATE}
|
|
FROM scenes s WHERE s.id = ps.scene_id AND ps.sort_date IS NULL
|
|
""")).rowcount
|
|
print(f"resztki: {rest}", flush=True)
|
|
vacuum(analyze=True)
|
|
with engine.connect() as c:
|
|
left = c.execute(text("SELECT count(*) FROM playback_sources WHERE sort_date IS NULL")).scalar()
|
|
print(f"KONIEC: {total + rest} wierszy, puste po wszystkim: {left}, {time.time() - t0:.0f} s", flush=True)
|