6c2ebeecf5
Grupy po hashu urzadzenia, tylko slot affinity, srednie po urzadzeniach. Test A/A na tygodniu przed wdrozeniem: dokonczenia 13,0% vs 13,1%, lajki 3,4% vs 3,1%, ucieczki 49,1% vs 49,6% (357 i 111 urzadzen), czyli szum ~0,5 pp. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
62 lines
2.6 KiB
Python
62 lines
2.6 KiB
Python
"""Porownanie punktacji feedu Edge: grupa nowa vs kontrolna (stara punktacja).
|
|
|
|
Przydzial do grupy jest staly po hashu id urzadzenia (`app.clips_feed.uses_v1_scoring`),
|
|
wiec wystarczy pogrupowac zdarzenia z okresu po wdrozeniu. Liczymy tylko slot 'affinity',
|
|
bo tylko tam punktacja dziala (explore losuje tak samo w obu grupach). Srednie sa po
|
|
urzadzeniach (kazde urzadzenie z >= MIN_VIEWS obejrzeniami waży tyle samo), zeby kilku
|
|
najaktywniejszych nie przesadzalo o wyniku.
|
|
|
|
docker compose exec -T worker python3 /srv/scripts/edge_ab_report.py [od] [do]
|
|
Domyslnie od wdrozenia (2026-10-02 10:00 UTC) do teraz. Okno sprzed wdrozenia to test A/A:
|
|
obie grupy mialy wtedy te sama punktacje, wiec roznice pokazuja sam szum.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import statistics
|
|
import sys
|
|
from collections import defaultdict
|
|
|
|
from sqlalchemy import text
|
|
|
|
from app.clips_feed import uses_v1_scoring
|
|
from app.db import session_scope
|
|
|
|
SINCE = sys.argv[1] if len(sys.argv) > 1 else "2026-10-02T10:00:00+00:00"
|
|
UNTIL = sys.argv[2] if len(sys.argv) > 2 else "2100-01-01T00:00:00+00:00"
|
|
MIN_VIEWS = 20
|
|
VIEW_KINDS = ("complete", "skip", "impression", "autoswipe")
|
|
|
|
with session_scope() as s:
|
|
rows = s.execute(text("""
|
|
SELECT device_id, kind, dwell_ms FROM clip_feed_events
|
|
WHERE created_at >= CAST(:since AS timestamptz) AND created_at < CAST(:until AS timestamptz)
|
|
AND slot = 'affinity'
|
|
"""), {"since": SINCE, "until": UNTIL}).all()
|
|
|
|
per = defaultdict(lambda: {"views": 0, "complete": 0, "skip": 0, "like": 0, "not_interested": 0, "dwell": []})
|
|
for dev, kind, dwell in rows:
|
|
d = per[dev]
|
|
if kind in VIEW_KINDS:
|
|
d["views"] += 1
|
|
if dwell:
|
|
d["dwell"].append(dwell)
|
|
if kind in d and kind != "views":
|
|
d[kind] += 1
|
|
|
|
groups: dict[str, list[dict]] = {"nowa": [], "kontrolna": []}
|
|
for dev, d in per.items():
|
|
if d["views"] >= MIN_VIEWS:
|
|
groups["kontrolna" if uses_v1_scoring(dev) else "nowa"].append(d)
|
|
|
|
print(f"od {SINCE} do {UNTIL}, zdarzen {len(rows)}, urzadzen z >= {MIN_VIEWS} obejrzeniami: "
|
|
+ ", ".join(f"{g} {len(v)}" for g, v in groups.items()))
|
|
for g, devs in groups.items():
|
|
if not devs:
|
|
continue
|
|
def avg(f):
|
|
return 100 * statistics.mean(f(d) for d in devs)
|
|
print(f" {g:9s} dokonczone {avg(lambda d: d['complete'] / d['views']):5.1f}%"
|
|
f" lajki {avg(lambda d: d['like'] / d['views']):4.1f}%"
|
|
f" ucieczki {avg(lambda d: d['skip'] / d['views']):5.1f}%"
|
|
f" nie-interesuje {avg(lambda d: d['not_interested'] / d['views']):4.1f}%"
|
|
f" mediana dwell {statistics.median(statistics.median(d['dwell']) for d in devs if d['dwell']) / 1000:.1f} s")
|