From 8753243e1560389990b2439aee2581728c4bb1fe Mon Sep 17 00:00:00 2001 From: goon-foss Date: Fri, 2 Oct 2026 11:30:50 +0200 Subject: [PATCH] fix(edge): skasowany klip nie zostawia czarnego ekranu Klip skasowany u zrodla daje 403 i na wideo, i na plakacie, a apka nic z tym nie robila. Teraz odtwarzacz zglasza zdarzenie 'error' (i 'stall' po 12 s bez obrazu), przy 403/404/410 przewija dalej (max 3 z rzedu, bledy sieci i TLS nie przewijaja), a serwer od razu sprawdza plik i oznacza klip martwy. Reaper co godzine po 1000 klipow zamiast 2000 na dobe: ~4% zywych klipow w feedzie zwracalo juz 403. Zejscie z zepsutego klipu nie psuje wag jako "skip". Co-Authored-By: Claude Opus 5.5 --- app/api/clips.py | 14 +++++++ app/clips_feed.py | 4 +- app/config.py | 5 +++ app/scheduler/clips_ingest.py | 45 ++++++++++++++++++++- app/scheduler/jobs.py | 22 ++++++++++ app/scheduler/worker.py | 2 + mobile/src/components/swipe/ClipPager.tsx | 49 ++++++++++++++++++++++- mobile/src/screens/ClipCreatorScreen.tsx | 14 ++++++- mobile/src/screens/SwipeScreen.tsx | 28 ++++++++++++- 9 files changed, 176 insertions(+), 7 deletions(-) diff --git a/app/api/clips.py b/app/api/clips.py index 43b71a3..d4a4d22 100644 --- a/app/api/clips.py +++ b/app/api/clips.py @@ -17,6 +17,7 @@ bez image proxy: RedGifs nie sprawdza Referera, a proxy tylko dodałoby VPS do t from __future__ import annotations import logging +import threading import time import uuid from datetime import UTC, datetime @@ -479,9 +480,22 @@ def clip_events( if changed: _save_affinity(session, device_id, row, aff, changed) session.commit() + broken = list({e.clip_id for e in events if e.kind == "error"}) + if broken: + # W tle, zeby apka nie czekala na HEAD do RedGifs. + threading.Thread(target=_check_broken, args=(broken,), name="clip-check", daemon=True).start() return Response(status_code=204) +def _check_broken(clip_ids: list) -> None: + from app.scheduler.clips_ingest import check_clips_now + + try: + check_clips_now(clip_ids) + except Exception: # noqa: BLE001 + log.exception("clips: sprawdzenie zgloszonych klipow nie wyszlo") + + # ---- onboarding, afinicznosc ------------------------------------------------------------ _OPTIONS_CACHE: dict[str, tuple[float, OnboardingOptionsOut]] = {} diff --git a/app/clips_feed.py b/app/clips_feed.py index 168a531..fc36780 100644 --- a/app/clips_feed.py +++ b/app/clips_feed.py @@ -49,8 +49,10 @@ EVENT_DELTAS: dict[str, tuple[float, float, float]] = { "not_interested": (-4.0, -4.0, -3.0), } REASON_PENALTY = -8.0 +#: `error`: odtwarzacz w apce sie wywrocil albo stal bez obrazu (reason 'stall'). Nie +#: rusza wag, a API od razu sprawdza plik i oznacza klip martwy (check_clips_now). ALLOWED_KINDS = frozenset( - {"impression", "skip", "complete", "like", "unlike", "not_interested", "autoswipe", "open"} + {"impression", "skip", "complete", "like", "unlike", "not_interested", "autoswipe", "open", "error"} ) #: Kolejność „siły" wyniku w `clip_seen`: nie cofamy do słabszego. OUTCOME_RANK = {"served": 0, "skip": 1, "complete": 2, "like": 3, "not_interested": 4} diff --git a/app/config.py b/app/config.py index 55f671f..944144d 100644 --- a/app/config.py +++ b/app/config.py @@ -245,6 +245,11 @@ class Settings(BaseSettings): sched_clips_maintenance_hours: int = Field( default=24, validation_alias="GOON_SCHED_CLIPS_MAINTENANCE_HOURS" ) + # Osobny reaper martwych klipow co godzine. Sama konserwacja (2000/dobe) nie nadazala: + # 68 tys. nowych klipow tygodniowo, a ~3% zywych w feedzie dawalo juz 403 (czarne + # ekrany w Edge, 2026-10-02). 1000/h = pelny obieg katalogu w okolo tydzien. + sched_clips_reap_hours: int = Field(default=1, validation_alias="GOON_SCHED_CLIPS_REAP_HOURS") + clips_reap_batch: int = Field(default=1000, validation_alias="GOON_CLIPS_REAP_BATCH") # Feed podaje tylko klipy do tej długości (RedGifs p90 = 60 s; dłuższe to nie „swipe"). clips_feed_max_duration_sec: int = Field(default=120, validation_alias="GOON_CLIPS_FEED_MAX_DURATION_SEC") diff --git a/app/scheduler/clips_ingest.py b/app/scheduler/clips_ingest.py index 8c279f8..dafac74 100644 --- a/app/scheduler/clips_ingest.py +++ b/app/scheduler/clips_ingest.py @@ -177,7 +177,48 @@ _DEAD_CODES = {403, 404, 410} _REAP_UA = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/152.0.0.0 Safari/537.36"} -def reap_dead(*, limit: int = 2000, apply: bool = True) -> dict[str, int]: +def _head_dead(http, url: str) -> str | None: + """'http 403' itp., gdy plik klipu zniknal; None, gdy zyje albo nie wiadomo (blad sieci).""" + import httpx + + try: + r = http.head(url) + except httpx.HTTPError: + return None + return f"http {r.status_code}" if r.status_code in _DEAD_CODES else None + + +def check_clips_now(clip_ids: list) -> int: + """Natychmiastowe sprawdzenie klipow zgloszonych przez apke jako zepsute (zdarzenie + `error`). Ten sam test co reaper, tylko bez czekania na kolejke. Zwraca ile oznaczono.""" + import httpx + from sqlalchemy import select + + if not clip_ids: + return 0 + with session_scope() as session: + rows = session.execute( + select(Clip.id, Clip.hd_url, Clip.sd_url).where(Clip.id.in_(clip_ids), Clip.dead_at.is_(None)) + ).all() + marked = 0 + with httpx.Client(headers=_REAP_UA, timeout=15, follow_redirects=True) as http: + for cid, hd, sd in rows: + url = hd or sd + reason = _head_dead(http, url) if url else None + if reason is None: + continue + with session_scope() as session: + session.execute( + text("UPDATE clips SET dead_at = now(), dead_reason = :r WHERE id = :id AND dead_at IS NULL"), + {"r": f"{reason} (apka)", "id": cid}, + ) + marked += 1 + if marked: + log.info("clips: %d zgloszonych z apki oznaczone jako martwe", marked) + return marked + + +def reap_dead(*, limit: int = 2000, apply: bool = True, sleep: float = 0.2) -> dict[str, int]: """Reaper martwych klipow: tworcy kasuja klipy, RedGifs oddaje wtedy 404/410 na pliku. HEAD na `hd_url` (albo `sd_url`) dla najdawniej sprawdzanych zywych klipow @@ -214,7 +255,7 @@ def reap_dead(*, limit: int = 2000, apply: bool = True) -> dict[str, int]: if r.status_code in _DEAD_CODES: dead += 1 dead_ids.append((cid, f"http {r.status_code}")) - _time.sleep(0.2) + _time.sleep(sleep) if apply and rows: dead_set = {d[0] for d in dead_ids} with session_scope() as session: diff --git a/app/scheduler/jobs.py b/app/scheduler/jobs.py index 42bfd63..641c83e 100644 --- a/app/scheduler/jobs.py +++ b/app/scheduler/jobs.py @@ -524,6 +524,16 @@ def _job_clips_latest(pages: int) -> None: log.exception("[scheduler] clips-latest failed") +def _job_clips_reap(batch: int) -> None: + """Feed Swipe: HEAD na plikach najdawniej sprawdzanych klipow, 403/404/410 = martwy.""" + from app.scheduler.clips_ingest import reap_dead + + try: + _run_with_timeout(lambda: reap_dead(limit=batch, apply=True, sleep=0.1), label="clips-reap") + except Exception: + log.exception("[scheduler] clips-reap failed") + + def _job_clips_maintenance() -> None: """Feed Swipe: decay afiniczności, retencja zdarzeń, reaper martwych plików.""" from app.scheduler.clips_ingest import run_clips_maintenance @@ -637,6 +647,18 @@ def build_scheduler(cfg: dict[str, Any]) -> BlockingScheduler: ) log.info("scheduler: clips-maintenance every %dh", cfg["clips_maintenance_hours"]) + if cfg.get("clips_reap_hours"): + reap_batch = cfg.get("clips_reap_batch") or 1000 + sched.add_job( + lambda: _job_clips_reap(reap_batch), + IntervalTrigger(hours=cfg["clips_reap_hours"], start_date=INTERVAL_ANCHOR), + id="clips_reap", + replace_existing=True, + max_instances=1, + coalesce=True, + ) + log.info("scheduler: clips-reap every %dh (batch=%d)", cfg["clips_reap_hours"], reap_batch) + if cfg.get("bulk_dedup_hours"): sched.add_job( _job_bulk_dedup_performers, diff --git a/app/scheduler/worker.py b/app/scheduler/worker.py index 765fcba..b016f5f 100644 --- a/app/scheduler/worker.py +++ b/app/scheduler/worker.py @@ -258,6 +258,8 @@ def run_forever() -> int: "clips_hours": getattr(settings, "sched_clips_hours", 1) or None, "clips_latest_pages": getattr(settings, "clips_latest_pages", 5), "clips_maintenance_hours": getattr(settings, "sched_clips_maintenance_hours", 24) or None, + "clips_reap_hours": getattr(settings, "sched_clips_reap_hours", 1) or None, + "clips_reap_batch": getattr(settings, "clips_reap_batch", 1000), } sched = build_scheduler(cfg) log.info("worker scheduled mode starting (jobs=%d)", len(sched.get_jobs())) diff --git a/mobile/src/components/swipe/ClipPager.tsx b/mobile/src/components/swipe/ClipPager.tsx index b2673fe..913a2b0 100644 --- a/mobile/src/components/swipe/ClipPager.tsx +++ b/mobile/src/components/swipe/ClipPager.tsx @@ -49,8 +49,30 @@ interface Props { onCreatorPress?: (creator: string) => void; /** Ref pagera dla wywolujacego (autoSwipe robi `setPage`). */ pagerRef?: React.MutableRefObject; + /** + * Klip na stronie `index` sie nie odtworzy. `reason` to komunikat odtwarzacza albo + * 'stall' (po STALL_MS dalej bez obrazu). Przy bledzie wywolujacy przewija dalej, przy + * 'stall' tylko zglasza: wolne lacze to nie zepsuty klip. + */ + onError?: (index: number, reason: string) => void; } +/** Tyle czekamy na pierwszy obraz, zanim zglosimy klip jako stojacy. */ +const STALL_MS = 12000; + +/** + * Plik klipu zniknal (twórca skasowal): ExoPlayer konczy sie kodem HTTP 403/404/410. + * Tylko wtedy przewijamy sami. Bledy sieci i TLS (np. SSLHandshakeException na + * emulatorze, blokada RedGifs w danym kraju) dotycza wszystkich klipow naraz, wiec + * samo przewijanie przelecialoby przez caly feed w kilka sekund. + */ +export function clipGone(reason: string): boolean { + return /response code: (403|404|410)\b/i.test(reason); +} + +/** Po tylu zepsutych klipach z rzedu przestajemy przewijac: to raczej siec niz klipy. */ +export const MAX_AUTO_SKIPS = 3; + /** Adres wideo klipu (HD, a gdy go nie ma, SD). Klip bez zadnego z nich sie nie odtworzy. */ export function clipSource(item: ClipFeedItem): string | null { return item.clip.hd_url || item.clip.sd_url || null; @@ -69,6 +91,7 @@ export function ClipPager({ onNotInterested, onCreatorPress, pagerRef, + onError, }: Props) { const localRef = React.useRef(null); const ref = pagerRef ?? localRef; @@ -122,6 +145,8 @@ export function ClipPager({ itemsRef.current = items; const soundRef = React.useRef(soundOn); soundRef.current = soundOn; + const errorRef = React.useRef(onError); + errorRef.current = onError; const activeSrc = items[index] ? clipSource(items[index]) : null; const nextSrc = items[index + 1] ? clipSource(items[index + 1]) : null; React.useEffect(() => { @@ -170,6 +195,24 @@ export function ClipPager({ const sub = active.addListener('playToEnd', () => { loopsRef.current += 1; }); + // Czarny ekran (zgloszenie Jana 2026-10-02): skasowany klip daje 403 i na wideo, i na + // plakacie, a nic tego nie zglaszalo. Klip mogl sie wywrocic jeszcze w buforze + // (player standby), wtedy statusChange juz nie przyjdzie, stad sprawdzenie od razu. + // Kazdy rodzaj raz na wejscie na strone; po 'stall' blad moze jeszcze przyjsc. + const reported = new Set(); + const report = (reason: string) => { + const kind = reason === 'stall' ? 'stall' : 'error'; + if (reported.has(kind) || reported.has('error')) return; + reported.add(kind); + errorRef.current?.(index, reason); + }; + const subStatus = active.addListener('statusChange', (e) => { + if (e.status === 'error') report(e.error?.message || 'error'); + }); + if (active.status === 'error') report('error'); + const stall = setTimeout(() => { + if (active.status !== 'readyToPlay' && active.status !== 'error') report('stall'); + }, STALL_MS); if (nextSrc) { ensureSource(standby, nextSrc); @@ -180,7 +223,11 @@ export function ClipPager({ } } Haptics.selectionAsync().catch(() => {}); - return () => sub.remove(); + return () => { + sub.remove(); + subStatus.remove(); + clearTimeout(stall); + }; }, [index, activeSrc, nextSrc, ensureSource]); // Zmiana dzwieku bez zmiany strony. diff --git a/mobile/src/screens/ClipCreatorScreen.tsx b/mobile/src/screens/ClipCreatorScreen.tsx index 6ce2965..147bfcc 100644 --- a/mobile/src/screens/ClipCreatorScreen.tsx +++ b/mobile/src/screens/ClipCreatorScreen.tsx @@ -22,7 +22,7 @@ import { ActivityIndicator, FlatList, Modal, Pressable, StyleSheet, Text, View } import { useSafeAreaInsets } from 'react-native-safe-area-context'; import { useClient } from '../ClientContext'; -import { ClipPager, clipSource } from '../components/swipe/ClipPager'; +import { ClipPager, clipGone, clipSource } from '../components/swipe/ClipPager'; import { newSessionId } from '../lib/swipeTelemetry'; import type { RootStackParamList } from '../navigation'; import { fonts, theme } from '../theme'; @@ -52,6 +52,7 @@ export function ClipCreatorScreen() { const [soundOn, setSoundOn] = React.useState(false); const [liked, setLiked] = React.useState>(new Set()); const sessionId = React.useRef(newSessionId()); + const pagerRef = React.useRef(null); const q = useInfiniteQuery({ queryKey: ['clips-by-creator', creator], @@ -164,6 +165,7 @@ export function ClipCreatorScreen() { items={items} index={playing} initialPage={playing} + pagerRef={pagerRef} onPageSelected={setPlaying} soundOn={soundOn} onToggleSound={() => setSoundOn((v) => !v)} @@ -173,6 +175,16 @@ export function ClipCreatorScreen() { sendEvent(it.clip.id, 'not_interested', `creator:${creator}`); setPlaying(null); }} + onError={(i, reason) => { + const it = items[i]; + if (!it) return; + client + .postClipEvents(sessionId.current, [ + { clip_id: it.clip.id, kind: 'error', reason: reason.slice(0, 150), slot: 'creator' }, + ]) + .catch(() => {}); + if (clipGone(reason) && i + 1 < items.length) setTimeout(() => pagerRef.current?.setPage(i + 1), 600); + }} /> ) : null} {/* Modal rysuje sie od samej gory ekranu, wiec bez insetu przycisk laduje pod diff --git a/mobile/src/screens/SwipeScreen.tsx b/mobile/src/screens/SwipeScreen.tsx index 5d43860..de81e32 100644 --- a/mobile/src/screens/SwipeScreen.tsx +++ b/mobile/src/screens/SwipeScreen.tsx @@ -25,7 +25,7 @@ import React from 'react'; import { ActivityIndicator, Modal, Pressable, ScrollView, StyleSheet, Text, TextInput, View } from 'react-native'; import { useClient } from '../ClientContext'; -import { ClipPager, clipSource, pagerAvailable, type ClipLeaveInfo } from '../components/swipe/ClipPager'; +import { ClipPager, MAX_AUTO_SKIPS, clipGone, clipSource, pagerAvailable, type ClipLeaveInfo } from '../components/swipe/ClipPager'; import type { RootStackParamList } from '../navigation'; import { NATIVE_RUNTIME, isNewerVersion } from '../lib/appVersion'; import { setBarHidden } from '../lib/bottomBar'; @@ -113,11 +113,34 @@ export function SwipeScreen() { return () => t.stop(); }, []); + // ---- zepsuty klip: zglos i przewin dalej --------------------------------------------- + // Skasowany u zrodla klip dawal czarny ekran, a zejscie z niego liczylo sie jak zwykly + // „skip" i psulo wagi tagow. Teraz idzie jako 'error' (serwer od razu sprawdza plik + // i oznacza martwy), a zejscie z takiej strony nie trafia do telemetrii. + const brokenRef = React.useRef>(new Set()); + const autoSkipsRef = React.useRef(0); + const onClipError = React.useCallback( + (position: number, reason: string) => { + const it = items[position]; + if (!it) return; + telemetry.current?.push({ clip_id: it.clip.id, kind: 'error', reason: reason.slice(0, 150), slot: it.slot, position }); + if (!clipGone(reason)) return; + brokenRef.current.add(it.clip.id); + if (autoSkipsRef.current >= MAX_AUTO_SKIPS) return; + autoSkipsRef.current += 1; + setTimeout(() => { + if (position + 1 < items.length) pagerRef.current?.setPage(position + 1); + }, 600); + }, + [items], + ); + // ---- zejscie ze strony: jedno zdarzenie z dwell ------------------------------------ const onLeave = React.useCallback( (prev: number, info: ClipLeaveInfo) => { const left = items[prev]; - if (!left) return; + if (!left || brokenRef.current.has(left.clip.id)) return; + autoSkipsRef.current = 0; // normalny klip po drodze: licznik przewiniec od nowa telemetry.current?.push({ clip_id: left.clip.id, kind: leaveKind(info.dwellMs, info.loops, info.positionSec, left.clip.duration_sec), @@ -248,6 +271,7 @@ export function SwipeScreen() { onLike={likeItem} onNotInterested={(it) => setSheetItem(it)} onCreatorPress={(creator) => navigation.navigate('ClipCreator', { creator })} + onError={onClipError} /> ) : feed.isLoading ? (