fix(edge): skasowany klip nie zostawia czarnego ekranu
Klip skasowany u zrodla daje 403 i na wideo, i na plakacie, a apka nic z tym nie robila. Teraz odtwarzacz zglasza zdarzenie 'error' (i 'stall' po 12 s bez obrazu), przy 403/404/410 przewija dalej (max 3 z rzedu, bledy sieci i TLS nie przewijaja), a serwer od razu sprawdza plik i oznacza klip martwy. Reaper co godzine po 1000 klipow zamiast 2000 na dobe: ~4% zywych klipow w feedzie zwracalo juz 403. Zejscie z zepsutego klipu nie psuje wag jako "skip". Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
goon-foss
parent
92f83b93fe
commit
8753243e15
9 changed files with 176 additions and 7 deletions
|
|
@ -17,6 +17,7 @@ bez image proxy: RedGifs nie sprawdza Referera, a proxy tylko dodałoby VPS do t
|
|||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
|
|
@ -479,9 +480,22 @@ def clip_events(
|
|||
if changed:
|
||||
_save_affinity(session, device_id, row, aff, changed)
|
||||
session.commit()
|
||||
broken = list({e.clip_id for e in events if e.kind == "error"})
|
||||
if broken:
|
||||
# W tle, zeby apka nie czekala na HEAD do RedGifs.
|
||||
threading.Thread(target=_check_broken, args=(broken,), name="clip-check", daemon=True).start()
|
||||
return Response(status_code=204)
|
||||
|
||||
|
||||
def _check_broken(clip_ids: list) -> None:
|
||||
from app.scheduler.clips_ingest import check_clips_now
|
||||
|
||||
try:
|
||||
check_clips_now(clip_ids)
|
||||
except Exception: # noqa: BLE001
|
||||
log.exception("clips: sprawdzenie zgloszonych klipow nie wyszlo")
|
||||
|
||||
|
||||
# ---- onboarding, afinicznosc ------------------------------------------------------------
|
||||
_OPTIONS_CACHE: dict[str, tuple[float, OnboardingOptionsOut]] = {}
|
||||
|
||||
|
|
|
|||
|
|
@ -49,8 +49,10 @@ EVENT_DELTAS: dict[str, tuple[float, float, float]] = {
|
|||
"not_interested": (-4.0, -4.0, -3.0),
|
||||
}
|
||||
REASON_PENALTY = -8.0
|
||||
#: `error`: odtwarzacz w apce sie wywrocil albo stal bez obrazu (reason 'stall'). Nie
|
||||
#: rusza wag, a API od razu sprawdza plik i oznacza klip martwy (check_clips_now).
|
||||
ALLOWED_KINDS = frozenset(
|
||||
{"impression", "skip", "complete", "like", "unlike", "not_interested", "autoswipe", "open"}
|
||||
{"impression", "skip", "complete", "like", "unlike", "not_interested", "autoswipe", "open", "error"}
|
||||
)
|
||||
#: Kolejność „siły" wyniku w `clip_seen`: nie cofamy do słabszego.
|
||||
OUTCOME_RANK = {"served": 0, "skip": 1, "complete": 2, "like": 3, "not_interested": 4}
|
||||
|
|
|
|||
|
|
@ -245,6 +245,11 @@ class Settings(BaseSettings):
|
|||
sched_clips_maintenance_hours: int = Field(
|
||||
default=24, validation_alias="GOON_SCHED_CLIPS_MAINTENANCE_HOURS"
|
||||
)
|
||||
# Osobny reaper martwych klipow co godzine. Sama konserwacja (2000/dobe) nie nadazala:
|
||||
# 68 tys. nowych klipow tygodniowo, a ~3% zywych w feedzie dawalo juz 403 (czarne
|
||||
# ekrany w Edge, 2026-10-02). 1000/h = pelny obieg katalogu w okolo tydzien.
|
||||
sched_clips_reap_hours: int = Field(default=1, validation_alias="GOON_SCHED_CLIPS_REAP_HOURS")
|
||||
clips_reap_batch: int = Field(default=1000, validation_alias="GOON_CLIPS_REAP_BATCH")
|
||||
# Feed podaje tylko klipy do tej długości (RedGifs p90 = 60 s; dłuższe to nie „swipe").
|
||||
clips_feed_max_duration_sec: int = Field(default=120, validation_alias="GOON_CLIPS_FEED_MAX_DURATION_SEC")
|
||||
|
||||
|
|
|
|||
|
|
@ -177,7 +177,48 @@ _DEAD_CODES = {403, 404, 410}
|
|||
_REAP_UA = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/152.0.0.0 Safari/537.36"}
|
||||
|
||||
|
||||
def reap_dead(*, limit: int = 2000, apply: bool = True) -> dict[str, int]:
|
||||
def _head_dead(http, url: str) -> str | None:
|
||||
"""'http 403' itp., gdy plik klipu zniknal; None, gdy zyje albo nie wiadomo (blad sieci)."""
|
||||
import httpx
|
||||
|
||||
try:
|
||||
r = http.head(url)
|
||||
except httpx.HTTPError:
|
||||
return None
|
||||
return f"http {r.status_code}" if r.status_code in _DEAD_CODES else None
|
||||
|
||||
|
||||
def check_clips_now(clip_ids: list) -> int:
|
||||
"""Natychmiastowe sprawdzenie klipow zgloszonych przez apke jako zepsute (zdarzenie
|
||||
`error`). Ten sam test co reaper, tylko bez czekania na kolejke. Zwraca ile oznaczono."""
|
||||
import httpx
|
||||
from sqlalchemy import select
|
||||
|
||||
if not clip_ids:
|
||||
return 0
|
||||
with session_scope() as session:
|
||||
rows = session.execute(
|
||||
select(Clip.id, Clip.hd_url, Clip.sd_url).where(Clip.id.in_(clip_ids), Clip.dead_at.is_(None))
|
||||
).all()
|
||||
marked = 0
|
||||
with httpx.Client(headers=_REAP_UA, timeout=15, follow_redirects=True) as http:
|
||||
for cid, hd, sd in rows:
|
||||
url = hd or sd
|
||||
reason = _head_dead(http, url) if url else None
|
||||
if reason is None:
|
||||
continue
|
||||
with session_scope() as session:
|
||||
session.execute(
|
||||
text("UPDATE clips SET dead_at = now(), dead_reason = :r WHERE id = :id AND dead_at IS NULL"),
|
||||
{"r": f"{reason} (apka)", "id": cid},
|
||||
)
|
||||
marked += 1
|
||||
if marked:
|
||||
log.info("clips: %d zgloszonych z apki oznaczone jako martwe", marked)
|
||||
return marked
|
||||
|
||||
|
||||
def reap_dead(*, limit: int = 2000, apply: bool = True, sleep: float = 0.2) -> dict[str, int]:
|
||||
"""Reaper martwych klipow: tworcy kasuja klipy, RedGifs oddaje wtedy 404/410 na pliku.
|
||||
|
||||
HEAD na `hd_url` (albo `sd_url`) dla najdawniej sprawdzanych zywych klipow
|
||||
|
|
@ -214,7 +255,7 @@ def reap_dead(*, limit: int = 2000, apply: bool = True) -> dict[str, int]:
|
|||
if r.status_code in _DEAD_CODES:
|
||||
dead += 1
|
||||
dead_ids.append((cid, f"http {r.status_code}"))
|
||||
_time.sleep(0.2)
|
||||
_time.sleep(sleep)
|
||||
if apply and rows:
|
||||
dead_set = {d[0] for d in dead_ids}
|
||||
with session_scope() as session:
|
||||
|
|
|
|||
|
|
@ -524,6 +524,16 @@ def _job_clips_latest(pages: int) -> None:
|
|||
log.exception("[scheduler] clips-latest failed")
|
||||
|
||||
|
||||
def _job_clips_reap(batch: int) -> None:
|
||||
"""Feed Swipe: HEAD na plikach najdawniej sprawdzanych klipow, 403/404/410 = martwy."""
|
||||
from app.scheduler.clips_ingest import reap_dead
|
||||
|
||||
try:
|
||||
_run_with_timeout(lambda: reap_dead(limit=batch, apply=True, sleep=0.1), label="clips-reap")
|
||||
except Exception:
|
||||
log.exception("[scheduler] clips-reap failed")
|
||||
|
||||
|
||||
def _job_clips_maintenance() -> None:
|
||||
"""Feed Swipe: decay afiniczności, retencja zdarzeń, reaper martwych plików."""
|
||||
from app.scheduler.clips_ingest import run_clips_maintenance
|
||||
|
|
@ -637,6 +647,18 @@ def build_scheduler(cfg: dict[str, Any]) -> BlockingScheduler:
|
|||
)
|
||||
log.info("scheduler: clips-maintenance every %dh", cfg["clips_maintenance_hours"])
|
||||
|
||||
if cfg.get("clips_reap_hours"):
|
||||
reap_batch = cfg.get("clips_reap_batch") or 1000
|
||||
sched.add_job(
|
||||
lambda: _job_clips_reap(reap_batch),
|
||||
IntervalTrigger(hours=cfg["clips_reap_hours"], start_date=INTERVAL_ANCHOR),
|
||||
id="clips_reap",
|
||||
replace_existing=True,
|
||||
max_instances=1,
|
||||
coalesce=True,
|
||||
)
|
||||
log.info("scheduler: clips-reap every %dh (batch=%d)", cfg["clips_reap_hours"], reap_batch)
|
||||
|
||||
if cfg.get("bulk_dedup_hours"):
|
||||
sched.add_job(
|
||||
_job_bulk_dedup_performers,
|
||||
|
|
|
|||
|
|
@ -258,6 +258,8 @@ def run_forever() -> int:
|
|||
"clips_hours": getattr(settings, "sched_clips_hours", 1) or None,
|
||||
"clips_latest_pages": getattr(settings, "clips_latest_pages", 5),
|
||||
"clips_maintenance_hours": getattr(settings, "sched_clips_maintenance_hours", 24) or None,
|
||||
"clips_reap_hours": getattr(settings, "sched_clips_reap_hours", 1) or None,
|
||||
"clips_reap_batch": getattr(settings, "clips_reap_batch", 1000),
|
||||
}
|
||||
sched = build_scheduler(cfg)
|
||||
log.info("worker scheduled mode starting (jobs=%d)", len(sched.get_jobs()))
|
||||
|
|
|
|||
|
|
@ -49,8 +49,30 @@ interface Props {
|
|||
onCreatorPress?: (creator: string) => void;
|
||||
/** Ref pagera dla wywolujacego (autoSwipe robi `setPage`). */
|
||||
pagerRef?: React.MutableRefObject<any>;
|
||||
/**
|
||||
* Klip na stronie `index` sie nie odtworzy. `reason` to komunikat odtwarzacza albo
|
||||
* 'stall' (po STALL_MS dalej bez obrazu). Przy bledzie wywolujacy przewija dalej, przy
|
||||
* 'stall' tylko zglasza: wolne lacze to nie zepsuty klip.
|
||||
*/
|
||||
onError?: (index: number, reason: string) => void;
|
||||
}
|
||||
|
||||
/** Tyle czekamy na pierwszy obraz, zanim zglosimy klip jako stojacy. */
|
||||
const STALL_MS = 12000;
|
||||
|
||||
/**
|
||||
* Plik klipu zniknal (twórca skasowal): ExoPlayer konczy sie kodem HTTP 403/404/410.
|
||||
* Tylko wtedy przewijamy sami. Bledy sieci i TLS (np. SSLHandshakeException na
|
||||
* emulatorze, blokada RedGifs w danym kraju) dotycza wszystkich klipow naraz, wiec
|
||||
* samo przewijanie przelecialoby przez caly feed w kilka sekund.
|
||||
*/
|
||||
export function clipGone(reason: string): boolean {
|
||||
return /response code: (403|404|410)\b/i.test(reason);
|
||||
}
|
||||
|
||||
/** Po tylu zepsutych klipach z rzedu przestajemy przewijac: to raczej siec niz klipy. */
|
||||
export const MAX_AUTO_SKIPS = 3;
|
||||
|
||||
/** Adres wideo klipu (HD, a gdy go nie ma, SD). Klip bez zadnego z nich sie nie odtworzy. */
|
||||
export function clipSource(item: ClipFeedItem): string | null {
|
||||
return item.clip.hd_url || item.clip.sd_url || null;
|
||||
|
|
@ -69,6 +91,7 @@ export function ClipPager({
|
|||
onNotInterested,
|
||||
onCreatorPress,
|
||||
pagerRef,
|
||||
onError,
|
||||
}: Props) {
|
||||
const localRef = React.useRef<any>(null);
|
||||
const ref = pagerRef ?? localRef;
|
||||
|
|
@ -122,6 +145,8 @@ export function ClipPager({
|
|||
itemsRef.current = items;
|
||||
const soundRef = React.useRef(soundOn);
|
||||
soundRef.current = soundOn;
|
||||
const errorRef = React.useRef(onError);
|
||||
errorRef.current = onError;
|
||||
const activeSrc = items[index] ? clipSource(items[index]) : null;
|
||||
const nextSrc = items[index + 1] ? clipSource(items[index + 1]) : null;
|
||||
React.useEffect(() => {
|
||||
|
|
@ -170,6 +195,24 @@ export function ClipPager({
|
|||
const sub = active.addListener('playToEnd', () => {
|
||||
loopsRef.current += 1;
|
||||
});
|
||||
// Czarny ekran (zgloszenie Jana 2026-10-02): skasowany klip daje 403 i na wideo, i na
|
||||
// plakacie, a nic tego nie zglaszalo. Klip mogl sie wywrocic jeszcze w buforze
|
||||
// (player standby), wtedy statusChange juz nie przyjdzie, stad sprawdzenie od razu.
|
||||
// Kazdy rodzaj raz na wejscie na strone; po 'stall' blad moze jeszcze przyjsc.
|
||||
const reported = new Set<string>();
|
||||
const report = (reason: string) => {
|
||||
const kind = reason === 'stall' ? 'stall' : 'error';
|
||||
if (reported.has(kind) || reported.has('error')) return;
|
||||
reported.add(kind);
|
||||
errorRef.current?.(index, reason);
|
||||
};
|
||||
const subStatus = active.addListener('statusChange', (e) => {
|
||||
if (e.status === 'error') report(e.error?.message || 'error');
|
||||
});
|
||||
if (active.status === 'error') report('error');
|
||||
const stall = setTimeout(() => {
|
||||
if (active.status !== 'readyToPlay' && active.status !== 'error') report('stall');
|
||||
}, STALL_MS);
|
||||
|
||||
if (nextSrc) {
|
||||
ensureSource(standby, nextSrc);
|
||||
|
|
@ -180,7 +223,11 @@ export function ClipPager({
|
|||
}
|
||||
}
|
||||
Haptics.selectionAsync().catch(() => {});
|
||||
return () => sub.remove();
|
||||
return () => {
|
||||
sub.remove();
|
||||
subStatus.remove();
|
||||
clearTimeout(stall);
|
||||
};
|
||||
}, [index, activeSrc, nextSrc, ensureSource]);
|
||||
|
||||
// Zmiana dzwieku bez zmiany strony.
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ import { ActivityIndicator, FlatList, Modal, Pressable, StyleSheet, Text, View }
|
|||
import { useSafeAreaInsets } from 'react-native-safe-area-context';
|
||||
|
||||
import { useClient } from '../ClientContext';
|
||||
import { ClipPager, clipSource } from '../components/swipe/ClipPager';
|
||||
import { ClipPager, clipGone, clipSource } from '../components/swipe/ClipPager';
|
||||
import { newSessionId } from '../lib/swipeTelemetry';
|
||||
import type { RootStackParamList } from '../navigation';
|
||||
import { fonts, theme } from '../theme';
|
||||
|
|
@ -52,6 +52,7 @@ export function ClipCreatorScreen() {
|
|||
const [soundOn, setSoundOn] = React.useState(false);
|
||||
const [liked, setLiked] = React.useState<Set<string>>(new Set());
|
||||
const sessionId = React.useRef(newSessionId());
|
||||
const pagerRef = React.useRef<any>(null);
|
||||
|
||||
const q = useInfiniteQuery({
|
||||
queryKey: ['clips-by-creator', creator],
|
||||
|
|
@ -164,6 +165,7 @@ export function ClipCreatorScreen() {
|
|||
items={items}
|
||||
index={playing}
|
||||
initialPage={playing}
|
||||
pagerRef={pagerRef}
|
||||
onPageSelected={setPlaying}
|
||||
soundOn={soundOn}
|
||||
onToggleSound={() => setSoundOn((v) => !v)}
|
||||
|
|
@ -173,6 +175,16 @@ export function ClipCreatorScreen() {
|
|||
sendEvent(it.clip.id, 'not_interested', `creator:${creator}`);
|
||||
setPlaying(null);
|
||||
}}
|
||||
onError={(i, reason) => {
|
||||
const it = items[i];
|
||||
if (!it) return;
|
||||
client
|
||||
.postClipEvents(sessionId.current, [
|
||||
{ clip_id: it.clip.id, kind: 'error', reason: reason.slice(0, 150), slot: 'creator' },
|
||||
])
|
||||
.catch(() => {});
|
||||
if (clipGone(reason) && i + 1 < items.length) setTimeout(() => pagerRef.current?.setPage(i + 1), 600);
|
||||
}}
|
||||
/>
|
||||
) : null}
|
||||
{/* Modal rysuje sie od samej gory ekranu, wiec bez insetu przycisk laduje pod
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ import React from 'react';
|
|||
import { ActivityIndicator, Modal, Pressable, ScrollView, StyleSheet, Text, TextInput, View } from 'react-native';
|
||||
|
||||
import { useClient } from '../ClientContext';
|
||||
import { ClipPager, clipSource, pagerAvailable, type ClipLeaveInfo } from '../components/swipe/ClipPager';
|
||||
import { ClipPager, MAX_AUTO_SKIPS, clipGone, clipSource, pagerAvailable, type ClipLeaveInfo } from '../components/swipe/ClipPager';
|
||||
import type { RootStackParamList } from '../navigation';
|
||||
import { NATIVE_RUNTIME, isNewerVersion } from '../lib/appVersion';
|
||||
import { setBarHidden } from '../lib/bottomBar';
|
||||
|
|
@ -113,11 +113,34 @@ export function SwipeScreen() {
|
|||
return () => t.stop();
|
||||
}, []);
|
||||
|
||||
// ---- zepsuty klip: zglos i przewin dalej ---------------------------------------------
|
||||
// Skasowany u zrodla klip dawal czarny ekran, a zejscie z niego liczylo sie jak zwykly
|
||||
// „skip" i psulo wagi tagow. Teraz idzie jako 'error' (serwer od razu sprawdza plik
|
||||
// i oznacza martwy), a zejscie z takiej strony nie trafia do telemetrii.
|
||||
const brokenRef = React.useRef<Set<string>>(new Set());
|
||||
const autoSkipsRef = React.useRef(0);
|
||||
const onClipError = React.useCallback(
|
||||
(position: number, reason: string) => {
|
||||
const it = items[position];
|
||||
if (!it) return;
|
||||
telemetry.current?.push({ clip_id: it.clip.id, kind: 'error', reason: reason.slice(0, 150), slot: it.slot, position });
|
||||
if (!clipGone(reason)) return;
|
||||
brokenRef.current.add(it.clip.id);
|
||||
if (autoSkipsRef.current >= MAX_AUTO_SKIPS) return;
|
||||
autoSkipsRef.current += 1;
|
||||
setTimeout(() => {
|
||||
if (position + 1 < items.length) pagerRef.current?.setPage(position + 1);
|
||||
}, 600);
|
||||
},
|
||||
[items],
|
||||
);
|
||||
|
||||
// ---- zejscie ze strony: jedno zdarzenie z dwell ------------------------------------
|
||||
const onLeave = React.useCallback(
|
||||
(prev: number, info: ClipLeaveInfo) => {
|
||||
const left = items[prev];
|
||||
if (!left) return;
|
||||
if (!left || brokenRef.current.has(left.clip.id)) return;
|
||||
autoSkipsRef.current = 0; // normalny klip po drodze: licznik przewiniec od nowa
|
||||
telemetry.current?.push({
|
||||
clip_id: left.clip.id,
|
||||
kind: leaveKind(info.dwellMs, info.loops, info.positionSec, left.clip.duration_sec),
|
||||
|
|
@ -248,6 +271,7 @@ export function SwipeScreen() {
|
|||
onLike={likeItem}
|
||||
onNotInterested={(it) => setSheetItem(it)}
|
||||
onCreatorPress={(creator) => navigation.navigate('ClipCreator', { creator })}
|
||||
onError={onClipError}
|
||||
/>
|
||||
) : feed.isLoading ? (
|
||||
<View style={styles.center}>
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue