251032a13b
edge_replay_features.py liczy na prodzie cechy znane w chwili kazdego
wyswietlenia, edge_tuning_analysis.py ocenia formuly (AUC w obrebie
urzadzenia, podzial w czasie). Na nich dobrana punktacja z 4666fcb.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
95 lines
4.6 KiB
Python
95 lines
4.6 KiB
Python
"""Odtworzenie uczenia feedu Edge zdarzenie po zdarzeniu (bez zagladania w przyszlosc).
|
|
|
|
Dla kazdego wyswietlenia (complete/skip/impression/autoswipe) zapisuje cechy znane W TEJ
|
|
CHWILI: skladniki obecnej punktacji z wag urzadzenia sprzed zdarzenia oraz liczniki klipu
|
|
z wczesniejszych zdarzen wszystkich urzadzen. Wynik CSV na stdout.
|
|
|
|
Uzyte do przestrojenia punktacji 2026-10-02 (patrz app/clips_feed.py, score_candidate).
|
|
Wybor tagow na starcie (onboarding) nie jest zdarzeniem, wiec tu go brakuje.
|
|
|
|
docker compose exec -T worker sh -c "cd /srv && python3 scripts/edge_replay_features.py | gzip -c" > edge.csv.gz
|
|
Analiza lokalnie: python scripts/edge_tuning_analysis.py edge.csv (pandas + scikit-learn).
|
|
"""
|
|
import csv
|
|
import math
|
|
import sys
|
|
from collections import defaultdict
|
|
from datetime import timedelta
|
|
|
|
from sqlalchemy import text
|
|
|
|
from app import clips_feed as cf
|
|
from app.db import session_scope
|
|
|
|
with session_scope() as s:
|
|
ev = s.execute(text("""
|
|
select device_id, clip_id, kind, dwell_ms, reason, slot, created_at
|
|
from clip_feed_events where clip_id is not null order by created_at""")).all()
|
|
ids = list({e.clip_id for e in ev})
|
|
clips = {}
|
|
for i in range(0, len(ids), 5000):
|
|
for r in s.execute(text("""select id, creator, likes, published_at, niches, duration_sec, width, height,
|
|
has_audio, creator_verified
|
|
from clips where id = any(:ids)"""), {"ids": ids[i:i + 5000]}).all():
|
|
clips[r.id] = r
|
|
tags = defaultdict(list)
|
|
for i in range(0, len(ids), 5000):
|
|
for cid, t in s.execute(text("select clip_id, tag from clip_tags where clip_id = any(:ids)"), {"ids": ids[i:i + 5000]}).all():
|
|
tags[cid].append(t)
|
|
n_all = s.execute(text("select count(*) from clips")).scalar()
|
|
tag_df = dict(s.execute(text("select tag, count(*) from clip_tags group by tag")).all())
|
|
idf = {t: math.log((n_all + 1) / (n + 1)) for t, n in tag_df.items()}
|
|
print(f"zdarzen {len(ev)}, klipow {len(clips)}, tagow {len(idf)}", file=sys.stderr)
|
|
|
|
aff = defaultdict(cf.Affinity)
|
|
last_day = {}
|
|
dev_n = defaultdict(int)
|
|
# liczniki klipu z wczesniejszych zdarzen (wszystkie urzadzenia)
|
|
c_seen = defaultdict(int)
|
|
c_comp = defaultdict(int)
|
|
c_skip = defaultdict(int)
|
|
c_like = defaultdict(int)
|
|
liked = set((e.device_id, e.clip_id) for e in ev if e.kind == "like")
|
|
|
|
w = csv.writer(sys.stdout)
|
|
w.writerow(["dev", "ts", "slot", "kind", "dwell", "liked", "tag_sum", "tag_pos", "tag_n", "niche_sum", "creator_w",
|
|
"tag_idf_sum", "tag_idf_pos", "pop", "fresh", "dur", "vertical", "audio", "verified", "c_seen", "c_comp", "c_skip", "c_like", "dev_n"])
|
|
OUTCOME = {"complete", "skip", "impression", "autoswipe"}
|
|
for e in ev:
|
|
c = clips.get(e.clip_id)
|
|
if c is None:
|
|
continue
|
|
a = aff[e.device_id]
|
|
day = e.created_at.date()
|
|
prev = last_day.get(e.device_id)
|
|
if prev is not None and day > prev:
|
|
for _ in range((day - prev).days):
|
|
a.tags = cf.decay_weights(a.tags)
|
|
a.niches = cf.decay_weights(a.niches)
|
|
a.creators = cf.decay_weights(a.creators)
|
|
last_day[e.device_id] = day
|
|
t = tags.get(e.clip_id, [])
|
|
niches = list(c.niches or [])
|
|
if e.kind in OUTCOME:
|
|
ts_vals = [a.tags.get(x, 0.0) for x in t]
|
|
fresh = int(bool(c.published_at and c.published_at >= e.created_at - timedelta(days=cf.FRESH_DAYS)))
|
|
vert = int(bool(c.width and c.height and c.height > c.width))
|
|
w.writerow([e.device_id[:12], e.created_at.isoformat(), e.slot or "", e.kind, e.dwell_ms or 0,
|
|
int((e.device_id, e.clip_id) in liked),
|
|
round(sum(ts_vals), 3), sum(1 for v in ts_vals if v > 0), len(t),
|
|
round(sum(a.niches.get(n, 0.0) for n in niches), 3),
|
|
round(a.creators.get(c.creator or "", 0.0), 3),
|
|
round(sum(a.tags.get(x, 0.0) * idf.get(x, 0.0) for x in t), 3),
|
|
round(sum(idf.get(x, 0.0) for x in t if a.tags.get(x, 0.0) > 0), 3),
|
|
round(math.log1p(max(c.likes or 0, 0)), 3), fresh, round(c.duration_sec or 0, 1), vert,
|
|
int(bool(c.has_audio)), int(bool(c.creator_verified)),
|
|
c_seen[e.clip_id], c_comp[e.clip_id], c_skip[e.clip_id], c_like[e.clip_id], dev_n[e.device_id]])
|
|
c_seen[e.clip_id] += 1
|
|
if e.kind == "complete":
|
|
c_comp[e.clip_id] += 1
|
|
elif e.kind == "skip":
|
|
c_skip[e.clip_id] += 1
|
|
dev_n[e.device_id] += 1
|
|
elif e.kind == "like":
|
|
c_like[e.clip_id] += 1
|
|
cf.apply_event(a, kind=e.kind, dwell_ms=e.dwell_ms, reason=e.reason, tags=t, niches=niches, creator=c.creator)
|