diff --git a/scripts/edge_ab_report.py b/scripts/edge_ab_report.py new file mode 100644 index 0000000..275c5ad --- /dev/null +++ b/scripts/edge_ab_report.py @@ -0,0 +1,62 @@ +"""Porownanie punktacji feedu Edge: grupa nowa vs kontrolna (stara punktacja). + +Przydzial do grupy jest staly po hashu id urzadzenia (`app.clips_feed.uses_v1_scoring`), +wiec wystarczy pogrupowac zdarzenia z okresu po wdrozeniu. Liczymy tylko slot 'affinity', +bo tylko tam punktacja dziala (explore losuje tak samo w obu grupach). Srednie sa po +urzadzeniach (kazde urzadzenie z >= MIN_VIEWS obejrzeniami waży tyle samo), zeby kilku +najaktywniejszych nie przesadzalo o wyniku. + + docker compose exec -T worker python3 /srv/scripts/edge_ab_report.py [od] [do] +Domyslnie od wdrozenia (2026-10-02 10:00 UTC) do teraz. Okno sprzed wdrozenia to test A/A: +obie grupy mialy wtedy te sama punktacje, wiec roznice pokazuja sam szum. +""" +from __future__ import annotations + +import statistics +import sys +from collections import defaultdict + +from sqlalchemy import text + +from app.clips_feed import uses_v1_scoring +from app.db import session_scope + +SINCE = sys.argv[1] if len(sys.argv) > 1 else "2026-10-02T10:00:00+00:00" +UNTIL = sys.argv[2] if len(sys.argv) > 2 else "2100-01-01T00:00:00+00:00" +MIN_VIEWS = 20 +VIEW_KINDS = ("complete", "skip", "impression", "autoswipe") + +with session_scope() as s: + rows = s.execute(text(""" + SELECT device_id, kind, dwell_ms FROM clip_feed_events + WHERE created_at >= CAST(:since AS timestamptz) AND created_at < CAST(:until AS timestamptz) + AND slot = 'affinity' + """), {"since": SINCE, "until": UNTIL}).all() + +per = defaultdict(lambda: {"views": 0, "complete": 0, "skip": 0, "like": 0, "not_interested": 0, "dwell": []}) +for dev, kind, dwell in rows: + d = per[dev] + if kind in VIEW_KINDS: + d["views"] += 1 + if dwell: + d["dwell"].append(dwell) + if kind in d and kind != "views": + d[kind] += 1 + +groups: dict[str, list[dict]] = {"nowa": [], "kontrolna": []} +for dev, d in per.items(): + if d["views"] >= MIN_VIEWS: + groups["kontrolna" if uses_v1_scoring(dev) else "nowa"].append(d) + +print(f"od {SINCE} do {UNTIL}, zdarzen {len(rows)}, urzadzen z >= {MIN_VIEWS} obejrzeniami: " + + ", ".join(f"{g} {len(v)}" for g, v in groups.items())) +for g, devs in groups.items(): + if not devs: + continue + def avg(f): + return 100 * statistics.mean(f(d) for d in devs) + print(f" {g:9s} dokonczone {avg(lambda d: d['complete'] / d['views']):5.1f}%" + f" lajki {avg(lambda d: d['like'] / d['views']):4.1f}%" + f" ucieczki {avg(lambda d: d['skip'] / d['views']):5.1f}%" + f" nie-interesuje {avg(lambda d: d['not_interested'] / d['views']):4.1f}%" + f" mediana dwell {statistics.median(statistics.median(d['dwell']) for d in devs if d['dwell']) / 1000:.1f} s")