From 4f5de92bfc044ec0160498e29a4c05499882a6e2 Mon Sep 17 00:00:00 2001 From: goon-foss Date: Mon, 14 Sep 2026 16:32:57 +0200 Subject: [PATCH] feat(clips): tabele klipow do feedu Swipe + ingest RedGifs (etap 2) Nowy wertykal krotkich pionowych klipow tworcow, osobno od scen: inne kolumny (tworca, audio, lajki, nisze), inny cykl zycia, inna taksonomia, brak filtra 60 s. Tabele `clips` (unikalne source+source_id) i `clip_tags` z indeksem (tag, clip_id) pod skan po tagu od losowego uuid w przyszlym feedzie. Zrodlo nr 1: RedGifs, oficjalne API v2 z tokenem tymczasowym (zwiazany z IP i UA, na 401 odnowienie). Globalne `order=latest` bez search_text daje ~100 na strone, kazde zapytanie widzi maks. 10 000 wynikow, wiec katalog chodzi sie po niszach (skrypt backfillu). Pliki media.redgifs.com sa publiczne i bez podpisu (206 z VPS-a i z telefonu), do bazy wchodzi gotowy adres. Scheduler: job clips-latest co godzine (5 stron), zatrzymuje sie na stronie bez nowych. Reaper (HEAD, 403/404/410 -> dead_at) jako skrypt, upsert odzywia klip, ktory zrodlo znow podaje. API: GET /clips i /clips/stats do pilota; wlasciwy feed, pamiec widzianych i telemetria to etap 3. Sprawdzone tego dnia i odlozone: nsfwswipe (lustro v.redd.it, bez audio, stoi od 19.08), Pornhub Shorties (zrodlo nr 2, HLS z tokenem 2 h, Referer na segmentach, 0/110 performerow z kanonu). --- alembic/versions/20260914_0036_clips.py | 70 +++++++++ app/api/clips.py | 158 ++++++++++++++++++++ app/config.py | 6 + app/connectors/clips/__init__.py | 2 + app/connectors/clips/redgifs.py | 190 ++++++++++++++++++++++++ app/main.py | 2 + app/models/__init__.py | 3 + app/models/clip.py | 85 +++++++++++ app/scheduler/clips_ingest.py | 165 ++++++++++++++++++++ app/scheduler/jobs.py | 22 +++ app/scheduler/worker.py | 3 + scripts/clips_backfill_redgifs.py | 37 +++++ scripts/clips_reap_dead.py | 83 +++++++++++ tests/test_redgifs_mapping.py | 54 +++++++ 14 files changed, 880 insertions(+) create mode 100644 alembic/versions/20260914_0036_clips.py create mode 100644 app/api/clips.py create mode 100644 app/connectors/clips/__init__.py create mode 100644 app/connectors/clips/redgifs.py create mode 100644 app/models/clip.py create mode 100644 app/scheduler/clips_ingest.py create mode 100644 scripts/clips_backfill_redgifs.py create mode 100644 scripts/clips_reap_dead.py create mode 100644 tests/test_redgifs_mapping.py diff --git a/alembic/versions/20260914_0036_clips.py b/alembic/versions/20260914_0036_clips.py new file mode 100644 index 0000000..1df5088 --- /dev/null +++ b/alembic/versions/20260914_0036_clips.py @@ -0,0 +1,70 @@ +"""Klipy do feedu Swipe: tabele `clips` i `clip_tags`. + +Osobny wertykał (krótkie pionowe wideo twórców z RedGifs / Pornhub Shorties), nie sceny. +Uzasadnienie w docstringu `app/models/clip.py`. + +Revision ID: 0036_clips +Revises: 0035_scenes_duration_idx +Create Date: 2026-09-14 +""" +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +revision = "0036_clips" +down_revision: str | None = "0035_scenes_duration_idx" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "clips", + sa.Column("id", postgresql.UUID(as_uuid=True), server_default=sa.text("gen_random_uuid()"), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.Column("source", sa.String(16), nullable=False), + sa.Column("source_id", sa.String(64), nullable=False), + sa.Column("title", sa.Text(), nullable=True), + sa.Column("creator", sa.String(128), nullable=True), + sa.Column("creator_verified", sa.Boolean(), nullable=False, server_default=sa.false()), + sa.Column("duration_sec", sa.Float(), nullable=True), + sa.Column("width", sa.Integer(), nullable=True), + sa.Column("height", sa.Integer(), nullable=True), + sa.Column("has_audio", sa.Boolean(), nullable=False, server_default=sa.false()), + sa.Column("content_type", sa.String(32), nullable=True), + sa.Column("niches", postgresql.ARRAY(sa.String(64)), nullable=False, server_default="{}"), + sa.Column("hd_url", sa.Text(), nullable=True), + sa.Column("sd_url", sa.Text(), nullable=True), + sa.Column("poster_url", sa.Text(), nullable=True), + sa.Column("thumb_url", sa.Text(), nullable=True), + sa.Column("hls", sa.Boolean(), nullable=False, server_default=sa.false()), + sa.Column("likes", sa.Integer(), nullable=False, server_default="0"), + sa.Column("views", sa.Integer(), nullable=False, server_default="0"), + sa.Column("published_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("dead_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("dead_reason", sa.String(128), nullable=True), + sa.PrimaryKeyConstraint("id", name="pk_clips"), + sa.UniqueConstraint("source", "source_id", name="uq_clips_source_source_id"), + ) + op.create_index("ix_clips_published_at", "clips", ["published_at"]) + op.create_index("ix_clips_creator", "clips", ["creator"]) + + op.create_table( + "clip_tags", + sa.Column("clip_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("tag", sa.String(64), nullable=False), + sa.ForeignKeyConstraint(["clip_id"], ["clips.id"], name="fk_clip_tags_clip_id_clips", ondelete="CASCADE"), + sa.PrimaryKeyConstraint("clip_id", "tag", name="pk_clip_tags"), + ) + op.create_index("ix_clip_tags_tag_clip", "clip_tags", ["tag", "clip_id"]) + + +def downgrade() -> None: + op.drop_index("ix_clip_tags_tag_clip", table_name="clip_tags") + op.drop_table("clip_tags") + op.drop_index("ix_clips_creator", table_name="clips") + op.drop_index("ix_clips_published_at", table_name="clips") + op.drop_table("clips") diff --git a/app/api/clips.py b/app/api/clips.py new file mode 100644 index 0000000..dc682b0 --- /dev/null +++ b/app/api/clips.py @@ -0,0 +1,158 @@ +"""Klipy (feed Swipe), etap 2: podstawowy listing i statystyki do pilota. + +Właściwy feed z afinicznością, pamięcią widzianych i telemetrią to etap 3 +(`/clips/feed`, `/clips/events`). Tu tylko to, czego trzeba, żeby zweryfikować ingest +i odtwarzanie z apki/emulatora: lista najnowszych żywych klipów z filtrem po tagu, +źródle i audio oraz zbiorcze liczniki. +""" +from __future__ import annotations + +import uuid +from datetime import datetime + +from fastapi import APIRouter, Depends, Query +from pydantic import BaseModel +from sqlalchemy import func, select +from sqlalchemy.orm import Session + +from app.auth import require_api_key +from app.db import get_session +from app.models.clip import Clip, ClipTag + +router = APIRouter(prefix="/clips", tags=["clips"], dependencies=[Depends(require_api_key)]) + + +class ClipOut(BaseModel): + id: uuid.UUID + source: str + source_id: str + title: str | None + creator: str | None + creator_verified: bool + duration_sec: float | None + width: int | None + height: int | None + has_audio: bool + content_type: str | None + niches: list[str] + tags: list[str] + hd_url: str | None + sd_url: str | None + poster_url: str | None + thumb_url: str | None + likes: int + views: int + published_at: datetime | None + + +class ClipListOut(BaseModel): + items: list[ClipOut] + limit: int + offset: int + has_more: bool + + +class ClipStatsOut(BaseModel): + total: int + alive: int + dead: int + with_audio: int + verified: int + by_source: dict[str, int] + newest_published_at: datetime | None + tags: int + + +def _tags_for(session: Session, clip_ids: list[uuid.UUID]) -> dict[uuid.UUID, list[str]]: + if not clip_ids: + return {} + out: dict[uuid.UUID, list[str]] = {} + for cid, tag in session.execute( + select(ClipTag.clip_id, ClipTag.tag).where(ClipTag.clip_id.in_(clip_ids)).order_by(ClipTag.tag) + ): + out.setdefault(cid, []).append(tag) + return out + + +@router.get("", response_model=ClipListOut) +def list_clips( + session: Session = Depends(get_session), + tag: str | None = Query(default=None, max_length=64), + source: str | None = Query(default=None, max_length=16), + creator: str | None = Query(default=None, max_length=128), + has_audio: bool | None = None, + max_duration_sec: float | None = Query(default=None, gt=0), + limit: int = Query(default=50, ge=1, le=200), + offset: int = Query(default=0, ge=0), +) -> ClipListOut: + q = select(Clip).where(Clip.dead_at.is_(None)) + if tag: + q = q.where(Clip.id.in_(select(ClipTag.clip_id).where(ClipTag.tag == tag))) + if source: + q = q.where(Clip.source == source) + if creator: + q = q.where(Clip.creator == creator) + if has_audio is not None: + q = q.where(Clip.has_audio.is_(has_audio)) + if max_duration_sec is not None: + q = q.where(Clip.duration_sec <= max_duration_sec) + rows = session.execute( + q.order_by(Clip.published_at.desc().nullslast(), Clip.id).offset(offset).limit(limit + 1) + ).scalars().all() + has_more = len(rows) > limit + rows = rows[:limit] + tags = _tags_for(session, [r.id for r in rows]) + items = [ + ClipOut( + id=r.id, + source=r.source, + source_id=r.source_id, + title=r.title, + creator=r.creator, + creator_verified=r.creator_verified, + duration_sec=r.duration_sec, + width=r.width, + height=r.height, + has_audio=r.has_audio, + content_type=r.content_type, + niches=list(r.niches or []), + tags=tags.get(r.id, []), + hd_url=r.hd_url, + sd_url=r.sd_url, + poster_url=r.poster_url, + thumb_url=r.thumb_url, + likes=r.likes, + views=r.views, + published_at=r.published_at, + ) + for r in rows + ] + return ClipListOut(items=items, limit=limit, offset=offset, has_more=has_more) + + +@router.get("/stats", response_model=ClipStatsOut) +def clip_stats(session: Session = Depends(get_session)) -> ClipStatsOut: + row = session.execute( + select( + func.count(), + func.count().filter(Clip.dead_at.is_(None)), + func.count().filter(Clip.dead_at.is_not(None)), + func.count().filter(Clip.has_audio.is_(True), Clip.dead_at.is_(None)), + func.count().filter(Clip.creator_verified.is_(True), Clip.dead_at.is_(None)), + func.max(Clip.published_at), + ) + ).one() + by_source = { + s: n for s, n in session.execute(select(Clip.source, func.count()).group_by(Clip.source)) + } + tags = session.execute(select(func.count(func.distinct(ClipTag.tag)))).scalar_one() + return ClipStatsOut( + total=row[0], + alive=row[1], + dead=row[2], + with_audio=row[3], + verified=row[4], + by_source=by_source, + newest_published_at=row[5], + tags=tags, + ) diff --git a/app/config.py b/app/config.py index 25c5471..6207abc 100644 --- a/app/config.py +++ b/app/config.py @@ -225,6 +225,12 @@ class Settings(BaseSettings): default=6, validation_alias="GOON_SCHED_SOURCE_STATS_HOURS" ) + # Klipy do feedu Swipe (RedGifs): godzinny run globalnego „najnowsze", N stron po + # 100 (RedGifs publikuje kilkaset klipów na godzinę). 0 = wyłączony. Backfill po + # niszach ręcznie: `python -m scripts.clips_backfill_redgifs`. + sched_clips_hours: int = Field(default=1, validation_alias="GOON_SCHED_CLIPS_HOURS") + clips_latest_pages: int = Field(default=5, validation_alias="GOON_CLIPS_LATEST_PAGES") + # Bright Data ISP proxy (stałe IP od ISP, rozliczane ryczałtem NIE per-GB), # używany do ingestu HTML (scrape) tubów które blokują VPS IP twardym Cloudflare # 403 nawet z browser-TLS (superporn). Streamu i tak nie ruszamy proxy (tokeny CDN diff --git a/app/connectors/clips/__init__.py b/app/connectors/clips/__init__.py new file mode 100644 index 0000000..5588e29 --- /dev/null +++ b/app/connectors/clips/__init__.py @@ -0,0 +1,2 @@ +"""Konektory klipów (feed Swipe). Osobne od `direct_scrapers`: inny model (`Clip`), +inny potok (upsert bez dedupu i bez matchera performerów), własne joby.""" diff --git a/app/connectors/clips/redgifs.py b/app/connectors/clips/redgifs.py new file mode 100644 index 0000000..227c431 --- /dev/null +++ b/app/connectors/clips/redgifs.py @@ -0,0 +1,190 @@ +"""RedGifs, oficjalne publiczne API v2. Główne źródło klipów do feedu Swipe. Dodane 2026-09-14. + +**Dlaczego RedGifs, a nie nsfwswipe.** Sprawdzone tego samego dnia: nsfwswipe to lustro +wideo natywnych Reddita (v.redd.it) bez audio, z ostatnim klipem z 19.08 (stoi od +czterech tygodni) i z losowym, nieoficjalnym API oddającym HTML. RedGifs daje JSON, +klipy z tej minuty, 1080x1920, audio w ok. 1/3 klipów, twórcę z odznaką weryfikacji, +lajki i widoki. Pliki `media.redgifs.com/.mp4` są publiczne, bez podpisu, grają bez +Referera z VPS-a i z telefonu (206), więc do bazy wchodzi gotowy adres, bez ekstraktora. + +**Token.** `GET /v2/auth/temporary` oddaje token na kilka godzin, związany z IP i User-Agentem +wołającego: kolejne zapytania MUSZĄ iść z tymi samymi nagłówkami, inaczej 401 `WrongSender` +(zjadło pół godziny przy pierwszym podejściu, bo curl i node miały inne UA). Na 401 +odnawiamy token raz i powtarzamy. + +**Listing.** `GET /v2/gifs/search?order=latest&count=100&page=N` bez `search_text` to +globalne „najnowsze" (98-100 na stronę). `order` ∈ latest|top|top7|top28|trending|score, +NIE `new` (400 BadOrder). Każde zapytanie widzi maks. 10 000 wyników (`pages` ≤ 100, +strona 101 → 500), więc pełny katalog chodzi się po niszach (`/v2/niches`, 1812 +społeczności, `/v2/niches//gifs`) i po twórcach (`/v2/users//search`). +Brak nagłówków rate-limit; trzymamy ~3 zapytania/s i ustępujemy przy 429. + +**Rekord gifa** (pola, które bierzemy): `id`, `type` (1 = wideo, 2 = obraz, pomijamy), +`urls{hd,sd,poster,thumbnail}`, `duration`, `width/height`, `hasAudio`, `hls`, `tags[]`, +`niches[]`, `contentType`, `sexuality[]`, `likes`, `views`, `createDate` (unix s), +`userName`, `verified`, `cta`/`promoted` (reklama, informacyjnie). +""" +from __future__ import annotations + +import logging +import re +import time +from dataclasses import dataclass, field +from datetime import UTC, datetime + +import httpx + +log = logging.getLogger(__name__) + +_BASE = "https://api.redgifs.com/v2" +#: Stałe UA i Origin: token jest do nich przywiązany (patrz docstring). +_HEADERS = { + "User-Agent": ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/152.0.0.0 Safari/537.36" + ), + "Accept": "application/json", + "Origin": "https://www.redgifs.com", + "Referer": "https://www.redgifs.com/", +} +_PAGE_MAX = 100 +_MIN_INTERVAL_S = 0.35 +_TAG_SLUG_RE = re.compile(r"[^a-z0-9]+") + + +@dataclass +class RawClip: + source: str + source_id: str + title: str | None + creator: str | None + creator_verified: bool + duration_sec: float | None + width: int | None + height: int | None + has_audio: bool + content_type: str | None + niches: list[str] = field(default_factory=list) + tags: list[str] = field(default_factory=list) + hd_url: str | None = None + sd_url: str | None = None + poster_url: str | None = None + thumb_url: str | None = None + hls: bool = False + likes: int = 0 + views: int = 0 + published_at: datetime | None = None + + +def tag_slug(name: str) -> str: + """"Big Tits" → "big-tits". Ta sama reguła dla RedGifs i (później) Pornhuba, żeby + afiniczność liczyła się na wspólnych slugach.""" + return _TAG_SLUG_RE.sub("-", (name or "").strip().lower()).strip("-")[:64] + + +def map_gif(g: dict) -> RawClip | None: + """Rekord API → RawClip. None dla obrazów, rekordów bez pliku i bez id.""" + if not isinstance(g, dict) or g.get("type") not in (1, None): + return None + gid = str(g.get("id") or "").strip() + urls = g.get("urls") or {} + hd, sd = urls.get("hd"), urls.get("sd") + if not gid or not (hd or sd): + return None + created = g.get("createDate") + published = ( + datetime.fromtimestamp(int(created), tz=UTC) if isinstance(created, (int, float)) and created > 0 else None + ) + tags: list[str] = [] + seen: set[str] = set() + for t in g.get("tags") or []: + s = tag_slug(str(t)) + if s and s not in seen: + seen.add(s) + tags.append(s) + niches = [str(n)[:64] for n in (g.get("niches") or []) if n] + dur = g.get("duration") + return RawClip( + source="redgifs", + source_id=gid, + title=(g.get("description") or None), + creator=(g.get("userName") or None), + creator_verified=bool(g.get("verified")), + duration_sec=float(dur) if isinstance(dur, (int, float)) and dur > 0 else None, + width=g.get("width") if isinstance(g.get("width"), int) else None, + height=g.get("height") if isinstance(g.get("height"), int) else None, + has_audio=bool(g.get("hasAudio")), + content_type=(str(g["contentType"])[:32] if g.get("contentType") else None), + niches=niches, + tags=tags, + hd_url=hd, + sd_url=sd, + poster_url=urls.get("poster"), + thumb_url=urls.get("thumbnail"), + hls=bool(g.get("hls")), + likes=int(g.get("likes") or 0), + views=int(g.get("views") or 0), + published_at=published, + ) + + +class RedgifsClient: + """Cienki klient z tokenem tymczasowym i odnowieniem na 401.""" + + def __init__(self, *, timeout: float = 30.0) -> None: + self._http = httpx.Client(headers=_HEADERS, timeout=timeout) + self._token: str | None = None + self._last_call = 0.0 + + def close(self) -> None: + self._http.close() + + def _auth(self) -> str: + r = self._http.get(f"{_BASE}/auth/temporary") + r.raise_for_status() + tok = (r.json() or {}).get("token") + if not tok: + raise RuntimeError("redgifs: brak tokenu w /auth/temporary") + self._token = tok + return tok + + def _get(self, path: str, params: dict | None = None) -> dict: + wait = _MIN_INTERVAL_S - (time.monotonic() - self._last_call) + if wait > 0: + time.sleep(wait) + tok = self._token or self._auth() + for attempt in range(3): + r = self._http.get(f"{_BASE}{path}", params=params, headers={"Authorization": f"Bearer {tok}"}) + self._last_call = time.monotonic() + if r.status_code == 401 and attempt == 0: + tok = self._auth() + continue + if r.status_code == 429: + time.sleep(5 * (attempt + 1)) + continue + r.raise_for_status() + return r.json() + raise RuntimeError(f"redgifs: {path} nie odpowiedzial poprawnie po 3 probach") + + # ---- listingi --------------------------------------------------------------------- + def latest(self, page: int = 1, count: int = _PAGE_MAX) -> list[dict]: + """Globalne „najnowsze" (bez search_text).""" + return self._get("/gifs/search", {"order": "latest", "count": count, "page": page}).get("gifs") or [] + + def search(self, text: str, *, order: str = "latest", page: int = 1, count: int = _PAGE_MAX) -> list[dict]: + return self._get( + "/gifs/search", {"search_text": text, "order": order, "count": count, "page": page} + ).get("gifs") or [] + + def niches(self, page: int = 1, count: int = _PAGE_MAX) -> dict: + """`{"page","pages","total","niches":[{"id","name","gifs","subscribers","tags":[...]}]}`""" + return self._get("/niches", {"count": count, "page": page}) + + def niche_gifs(self, niche_id: str, *, order: str = "latest", page: int = 1, count: int = _PAGE_MAX) -> list[dict]: + return self._get(f"/niches/{niche_id}/gifs", {"order": order, "count": count, "page": page}).get("gifs") or [] + + def user_gifs(self, user: str, *, order: str = "latest", page: int = 1, count: int = _PAGE_MAX) -> list[dict]: + return self._get(f"/users/{user}/search", {"order": order, "count": count, "page": page}).get("gifs") or [] + + def gif(self, gif_id: str) -> dict | None: + return (self._get(f"/gifs/{gif_id}") or {}).get("gif") diff --git a/app/main.py b/app/main.py index d79b075..f75af35 100644 --- a/app/main.py +++ b/app/main.py @@ -12,6 +12,7 @@ from app.api.admin_html import mount_static from app.api.admin_html import router as admin_html_router from app.api.blacklist import router as blacklist_router from app.api.bug_reports import router as bug_reports_router +from app.api.clips import router as clips_router from app.api.expo_updates import router as expo_updates_router from app.api.favorites import router as favorites_router from app.api.me import router as me_router @@ -82,6 +83,7 @@ app.include_router(taxonomies_router) app.include_router(favorites_router) app.include_router(blacklist_router) app.include_router(bug_reports_router) +app.include_router(clips_router) app.include_router(saved_searches_router) app.include_router(expo_updates_router) app.include_router(watch_router) diff --git a/app/models/__init__.py b/app/models/__init__.py index bd6987d..600348d 100644 --- a/app/models/__init__.py +++ b/app/models/__init__.py @@ -1,6 +1,7 @@ from app.models.base import Base from app.models.blacklist import BlacklistedPerformer, BlacklistedStudio, BlacklistedTag from app.models.bug_report import BugReport +from app.models.clip import Clip, ClipTag from app.models.external_record import ExternalRecord from app.models.favorite_movie import FavoriteMovie from app.models.favorite_performer import FavoritePerformer @@ -43,6 +44,8 @@ __all__ = [ "BlacklistedStudio", "BlacklistedTag", "BugReport", + "Clip", + "ClipTag", "ExternalRecord", "FavoriteMovie", "FavoritePerformer", diff --git a/app/models/clip.py b/app/models/clip.py new file mode 100644 index 0000000..ae29e94 --- /dev/null +++ b/app/models/clip.py @@ -0,0 +1,85 @@ +"""Klipy do feedu Swipe (sekcja w stylu TikToka). Osobny wertykał, NIE sceny. + +Klip to 5-120 s pionowego wideo od twórcy-amatora (RedGifs, Pornhub Shorties, nsfwswipe), +bez obsady z kanonu TPDB, bez dedupu ze scenami i bez filtra 60 s, którym grid ukrywa +krótkie rzeczy. Dlatego własne tabele zamiast wciskania tego w `scenes`: +inne kolumny (twórca, audio, lajki, nisze), inny cykl życia (twórcy kasują klipy, więc +reaper co dobę), inna taksonomia (tagi RedGifs to nie tagi Goona, 1495 vs nasze). + +`(source, source_id)` identyfikuje klip u źródła (`redgifs` + id gifa). Adresy plików +RedGifs są publiczne i bez podpisu (`media.redgifs.com/.mp4`), więc trzymamy je +wprost i telefon gra direct, bez ekstraktora i bez VPS-a. Pornhub Shorties (etap +późniejszy) będzie miał `hd_url` puste i resolve na żądanie. + +Tagi w osobnej tabeli z indeksem `(tag, clip_id)`: feed dobiera kandydatów skanem po tagu +od losowego uuid (jak `sort=random` w `list_scenes`), więc para w tej kolejności to jego +główny indeks. Nisze (społeczności RedGifs) jako tablica, bo służą tylko onboardingowi +i blacklistom, nie skanom. +""" +from __future__ import annotations + +import uuid +from datetime import datetime + +from sqlalchemy import ( + Boolean, + DateTime, + Float, + ForeignKey, + Index, + Integer, + String, + Text, + UniqueConstraint, +) +from sqlalchemy.dialects.postgresql import ARRAY, UUID +from sqlalchemy.orm import Mapped, mapped_column + +from app.models.base import Base, TimestampMixin, UUIDPKMixin + + +class Clip(UUIDPKMixin, TimestampMixin, Base): + __tablename__ = "clips" + __table_args__ = ( + UniqueConstraint("source", "source_id", name="uq_clips_source_source_id"), + Index("ix_clips_published_at", "published_at"), + Index("ix_clips_creator", "creator"), + ) + + #: `redgifs` | `pornhub` | `nsfwswipe` + source: Mapped[str] = mapped_column(String(16), nullable=False) + source_id: Mapped[str] = mapped_column(String(64), nullable=False) + title: Mapped[str | None] = mapped_column(Text, nullable=True) + #: Pseudonim twórcy u źródła (RedGifs `userName`). Nie łączymy z `performers`. + creator: Mapped[str | None] = mapped_column(String(128), nullable=True) + creator_verified: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + duration_sec: Mapped[float | None] = mapped_column(Float, nullable=True) + width: Mapped[int | None] = mapped_column(Integer, nullable=True) + height: Mapped[int | None] = mapped_column(Integer, nullable=True) + has_audio: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + #: RedGifs `contentType`, np. "Solo Female". + content_type: Mapped[str | None] = mapped_column(String(32), nullable=True) + niches: Mapped[list[str]] = mapped_column(ARRAY(String(64)), nullable=False, default=list) + hd_url: Mapped[str | None] = mapped_column(Text, nullable=True) + sd_url: Mapped[str | None] = mapped_column(Text, nullable=True) + poster_url: Mapped[str | None] = mapped_column(Text, nullable=True) + thumb_url: Mapped[str | None] = mapped_column(Text, nullable=True) + #: Źródło deklaruje wariant HLS (RedGifs `hls`). Informacyjne, gramy mp4. + hls: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + likes: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + views: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + #: Data publikacji u źródła (RedGifs `createDate`), do odznaki „nowe" i sortu. + published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + dead_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + dead_reason: Mapped[str | None] = mapped_column(String(128), nullable=True) + + +class ClipTag(Base): + __tablename__ = "clip_tags" + __table_args__ = (Index("ix_clip_tags_tag_clip", "tag", "clip_id"),) + + clip_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), ForeignKey("clips.id", ondelete="CASCADE"), primary_key=True + ) + #: Slug tagu u źródła, małe litery, spacje jako myślniki ("big-tits"). + tag: Mapped[str] = mapped_column(String(64), primary_key=True) diff --git a/app/scheduler/clips_ingest.py b/app/scheduler/clips_ingest.py new file mode 100644 index 0000000..b6899e4 --- /dev/null +++ b/app/scheduler/clips_ingest.py @@ -0,0 +1,165 @@ +"""Ingest klipów (feed Swipe): upsert do `clips` + `clip_tags`, joby RedGifs. + +Potok celowo prosty, bez `_ingest_iter_into_run`: klipy nie mają dedupu, matchera +performerów ani fingerprintów, a `(source, source_id)` jednoznacznie identyfikuje rekord. +Upsert `ON CONFLICT ... DO UPDATE` odświeża lajki/widoki/adresy i **zdejmuje `dead_at`** +(klip, który źródło znów podaje, żyje). Tagi tylko dokładamy (`DO NOTHING`), źródło ich +nie odbiera. + +Dwa tryby: +- `run_redgifs_latest(pages)`: globalne „najnowsze", co godzinę z schedulera. 5 stron = + ~500 klipów/run, RedGifs publikuje kilkaset na godzinę, więc to łapie prawie wszystko. +- `run_redgifs_niches(...)`: backfill po niszach (społecznościach), ręcznie ze skryptu + albo z `docker compose run` (patrz notatka o restarcie workera). Każde zapytanie widzi + maks. 10 000 wyników, dlatego katalog chodzi się per nisza, nie jednym listingiem. +""" +from __future__ import annotations + +import logging +from collections.abc import Iterable + +from sqlalchemy import text +from sqlalchemy.dialects.postgresql import insert as pg_insert +from sqlalchemy.orm import Session + +from app.connectors.clips.redgifs import RawClip, RedgifsClient, map_gif +from app.db import session_scope +from app.models.clip import Clip, ClipTag + +log = logging.getLogger(__name__) + + +def upsert_clips(session: Session, clips: Iterable[RawClip]) -> dict[str, int]: + seen = new = 0 + for c in clips: + seen += 1 + stmt = ( + pg_insert(Clip) + .values( + source=c.source, + source_id=c.source_id, + title=c.title, + creator=c.creator, + creator_verified=c.creator_verified, + duration_sec=c.duration_sec, + width=c.width, + height=c.height, + has_audio=c.has_audio, + content_type=c.content_type, + niches=c.niches, + hd_url=c.hd_url, + sd_url=c.sd_url, + poster_url=c.poster_url, + thumb_url=c.thumb_url, + hls=c.hls, + likes=c.likes, + views=c.views, + published_at=c.published_at, + ) + .on_conflict_do_update( + constraint="uq_clips_source_source_id", + set_={ + "title": c.title, + "creator": c.creator, + "creator_verified": c.creator_verified, + "duration_sec": c.duration_sec, + "has_audio": c.has_audio, + "niches": c.niches, + "hd_url": c.hd_url, + "sd_url": c.sd_url, + "poster_url": c.poster_url, + "thumb_url": c.thumb_url, + "likes": c.likes, + "views": c.views, + "dead_at": None, + "dead_reason": None, + "updated_at": text("now()"), + }, + ) + # xmax = 0 tylko dla świeżo wstawionego wiersza (sztuczka Postgresa na + # rozróżnienie INSERT od UPDATE w jednym zapytaniu). + .returning(Clip.id, text("(xmax = 0) AS inserted")) + ) + row = session.execute(stmt).one() + clip_id, inserted = row[0], bool(row[1]) + if inserted: + new += 1 + if c.tags: + session.execute( + pg_insert(ClipTag) + .values([{"clip_id": clip_id, "tag": t} for t in c.tags]) + .on_conflict_do_nothing() + ) + return {"seen": seen, "new": new, "updated": seen - new} + + +def _merge(total: dict[str, int], part: dict[str, int]) -> None: + for k, v in part.items(): + total[k] = total.get(k, 0) + v + + +def run_redgifs_latest(*, pages: int = 5) -> dict[str, int]: + """Globalne „najnowsze" RedGifs, `pages` stron po 100. Zatrzymuje się na pierwszej + stronie w całości już znanej (kolejne są starsze), żeby godzinny run nie mielił + w kółko tych samych 500 klipów.""" + total: dict[str, int] = {"seen": 0, "new": 0, "updated": 0, "errors": 0} + client = RedgifsClient() + try: + for page in range(1, pages + 1): + try: + gifs = client.latest(page=page) + except Exception as e: + log.warning("clips/redgifs latest: strona %d padla: %s", page, e) + total["errors"] += 1 + break + clips = [c for c in (map_gif(g) for g in gifs) if c is not None] + if not clips: + break + with session_scope() as session: + part = upsert_clips(session, clips) + _merge(total, part) + if part["new"] == 0 and page > 1: + break + finally: + client.close() + log.info("clips/redgifs latest: %s", total) + return total + + +def run_redgifs_niches( + *, niche_pages: int = 1, per_niche_pages: int = 1, order: str = "latest", skip: int = 0 +) -> dict[str, int]: + """Backfill po niszach: `niche_pages` stron listy nisz (100 na stronę, wg API + najpopularniejsze pierwsze), dla każdej niszy `per_niche_pages` stron gifów.""" + total: dict[str, int] = {"seen": 0, "new": 0, "updated": 0, "errors": 0, "niches": 0} + client = RedgifsClient() + try: + niche_ids: list[str] = [] + for p in range(1, niche_pages + 1): + try: + data = client.niches(page=p) + except Exception as e: + log.warning("clips/redgifs niches: lista strona %d padla: %s", p, e) + total["errors"] += 1 + break + niche_ids.extend(str(n.get("id")) for n in data.get("niches") or [] if n.get("id")) + for nid in niche_ids[skip:]: + total["niches"] += 1 + for page in range(1, per_niche_pages + 1): + try: + gifs = client.niche_gifs(nid, order=order, page=page) + except Exception as e: + log.warning("clips/redgifs niche %s strona %d padla: %s", nid, page, e) + total["errors"] += 1 + break + clips = [c for c in (map_gif(g) for g in gifs) if c is not None] + if not clips: + break + with session_scope() as session: + _merge(total, upsert_clips(session, clips)) + if total["niches"] % 20 == 0: + log.info("clips/redgifs niches: %d nisz, %s", total["niches"], total) + finally: + client.close() + log.info("clips/redgifs niches: %s", total) + return total diff --git a/app/scheduler/jobs.py b/app/scheduler/jobs.py index 7fb7fc0..914574b 100644 --- a/app/scheduler/jobs.py +++ b/app/scheduler/jobs.py @@ -477,6 +477,16 @@ def _job_tpdb_movie_enrich(batch: int) -> None: log.exception("[scheduler] tpdb-movie-enrich failed") +def _job_clips_latest(pages: int) -> None: + """Klipy do feedu Swipe: globalne „najnowsze" RedGifs (patrz clips_ingest).""" + from app.scheduler.clips_ingest import run_redgifs_latest + + try: + _run_with_timeout(lambda: run_redgifs_latest(pages=pages), label="clips-latest") + except Exception: + log.exception("[scheduler] clips-latest failed") + + def build_scheduler(cfg: dict[str, Any]) -> BlockingScheduler: """Buduje scheduler na podstawie cfg dictu. @@ -552,6 +562,18 @@ def build_scheduler(cfg: dict[str, Any]) -> BlockingScheduler: cfg["browse_latest_hours"], max_pages, ) + if cfg.get("clips_hours"): + clips_pages = cfg.get("clips_latest_pages") or 5 + sched.add_job( + lambda: _job_clips_latest(clips_pages), + IntervalTrigger(hours=cfg["clips_hours"], start_date=INTERVAL_ANCHOR), + id="clips_latest", + replace_existing=True, + max_instances=1, + coalesce=True, + ) + log.info("scheduler: clips-latest every %dh (pages=%d)", cfg["clips_hours"], clips_pages) + if cfg.get("bulk_dedup_hours"): sched.add_job( _job_bulk_dedup_performers, diff --git a/app/scheduler/worker.py b/app/scheduler/worker.py index de272cb..3727e8a 100644 --- a/app/scheduler/worker.py +++ b/app/scheduler/worker.py @@ -249,6 +249,9 @@ def run_forever() -> int: # Source ranking, przelicz source_stats (freshness/richness/health per origin) # dla Sites screen. Ciężki agregat → co 6h. "source_stats_hours": getattr(settings, "sched_source_stats_hours", 6) or None, + # Klipy do feedu Swipe (RedGifs „najnowsze"), co godzinę, N stron po 100. + "clips_hours": getattr(settings, "sched_clips_hours", 1) or None, + "clips_latest_pages": getattr(settings, "clips_latest_pages", 5), } sched = build_scheduler(cfg) log.info("worker scheduled mode starting (jobs=%d)", len(sched.get_jobs())) diff --git a/scripts/clips_backfill_redgifs.py b/scripts/clips_backfill_redgifs.py new file mode 100644 index 0000000..17398ae --- /dev/null +++ b/scripts/clips_backfill_redgifs.py @@ -0,0 +1,37 @@ +"""Backfill klipów RedGifs po niszach (społecznościach). Do uruchamiania ręcznego. + +Scheduler bierze tylko globalne „najnowsze" (co godzinę). Głębszy katalog chodzi się tu: +lista nisz z API jest posortowana od najpopularniejszych, dla każdej bierzemy N stron +po 100 gifów w wybranym porządku (`latest` = świeże, `top28` = najlepsze z miesiąca). +Jedno zapytanie RedGifs widzi maks. 10 000 wyników, więc `--per-niche-pages` ≤ 100. + +Długi przebieg odpalaj przez `docker compose run -d --rm worker python -m scripts...` +(nie `exec`, patrz notatka o restarcie workera), z pliku, nie z heredoca. + + python -m scripts.clips_backfill_redgifs --niche-pages 1 --per-niche-pages 1 # ~10k + python -m scripts.clips_backfill_redgifs --niche-pages 19 --per-niche-pages 5 --order top28 +""" +from __future__ import annotations + +import argparse +import logging + +from app.scheduler.clips_ingest import run_redgifs_niches + + +def main() -> None: + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s") + ap = argparse.ArgumentParser() + ap.add_argument("--niche-pages", type=int, default=1, help="stron listy nisz (100 nisz/strona)") + ap.add_argument("--per-niche-pages", type=int, default=1, help="stron gifów na niszę (100/strona)") + ap.add_argument("--order", default="latest", choices=["latest", "top", "top7", "top28", "trending", "score"]) + ap.add_argument("--skip", type=int, default=0, help="pomiń pierwsze N nisz (wznowienie)") + a = ap.parse_args() + out = run_redgifs_niches( + niche_pages=a.niche_pages, per_niche_pages=a.per_niche_pages, order=a.order, skip=a.skip + ) + print(out) + + +if __name__ == "__main__": + main() diff --git a/scripts/clips_reap_dead.py b/scripts/clips_reap_dead.py new file mode 100644 index 0000000..0ba6e50 --- /dev/null +++ b/scripts/clips_reap_dead.py @@ -0,0 +1,83 @@ +"""Reaper martwych klipów: twórcy kasują klipy, RedGifs oddaje wtedy 404/410 na pliku. + +Sprawdza HEAD `hd_url` (albo `sd_url`) dla próbki żywych klipów: najpierw te najdawniej +sprawdzane (`updated_at`), bo ingest `latest` i tak odświeża świeże. 404/410/403 → +`dead_at` + `dead_reason`; błędy sieci nie oznaczają śmierci (patrz notatka o dwóch +kontrolach przy przemiale źródeł). Klip, który źródło znów poda w listingu, upsert +odżywia (zdejmuje `dead_at`). + +Uruchomienie (kontener worker): + python -m scripts.clips_reap_dead --limit 500 # dry-run + python -m scripts.clips_reap_dead --limit 500 --yes # zapisz +""" +from __future__ import annotations + +import argparse +import logging +import time + +import httpx +from sqlalchemy import select, text + +from app.db import session_scope +from app.models.clip import Clip + +log = logging.getLogger(__name__) +_UA = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/152.0.0.0 Safari/537.36"} +_DEAD_CODES = {403, 404, 410} + + +def run(*, limit: int, apply: bool) -> dict[str, int]: + checked = dead = errors = 0 + dead_ids: list = [] + with session_scope() as session: + rows = session.execute( + select(Clip.id, Clip.hd_url, Clip.sd_url) + .where(Clip.dead_at.is_(None)) + .order_by(Clip.updated_at.asc()) + .limit(limit) + ).all() + with httpx.Client(headers=_UA, timeout=20, follow_redirects=True) as http: + for cid, hd, sd in rows: + url = hd or sd + if not url: + continue + checked += 1 + try: + r = http.head(url) + except httpx.HTTPError: + errors += 1 + continue + if r.status_code in _DEAD_CODES: + dead += 1 + dead_ids.append((cid, f"http {r.status_code}")) + time.sleep(0.2) + if apply: + with session_scope() as session: + for cid, reason in dead_ids: + session.execute( + text("UPDATE clips SET dead_at = now(), dead_reason = :r WHERE id = :id AND dead_at IS NULL"), + {"r": reason, "id": cid}, + ) + # Sprawdzone i zywe: odswiez updated_at, zeby kolejny run bral inne. + alive = [cid for cid, _hd, _sd in rows if cid not in {d[0] for d in dead_ids}] + if alive: + session.execute( + text("UPDATE clips SET updated_at = now() WHERE id = ANY(:ids)"), {"ids": alive} + ) + out = {"checked": checked, "dead": dead, "errors": errors, "applied": int(apply)} + print(out) + return out + + +def main() -> None: + logging.basicConfig(level=logging.INFO, format="%(message)s") + ap = argparse.ArgumentParser() + ap.add_argument("--limit", type=int, default=500) + ap.add_argument("--yes", action="store_true", help="zapisz dead_at (bez tego dry-run)") + a = ap.parse_args() + run(limit=a.limit, apply=a.yes) + + +if __name__ == "__main__": + main() diff --git a/tests/test_redgifs_mapping.py b/tests/test_redgifs_mapping.py new file mode 100644 index 0000000..024a4e0 --- /dev/null +++ b/tests/test_redgifs_mapping.py @@ -0,0 +1,54 @@ +from datetime import UTC, datetime + +from app.connectors.clips.redgifs import map_gif, tag_slug + +SAMPLE = { + "id": "unusualequalaxolotl", + "type": 1, + "createDate": 1789393316, + "contentType": "Solo Female", + "duration": 9.4, + "hasAudio": True, + "height": 1920, + "width": 1080, + "hls": True, + "likes": 12, + "views": 340, + "niches": ["just-boobs"], + "tags": ["Femdom", "Pegging", "Strap On", "Strap On"], + "urls": { + "sd": "https://media.redgifs.com/UnusualEqualAxolotl-mobile.mp4", + "hd": "https://media.redgifs.com/UnusualEqualAxolotl.mp4", + "poster": "https://media.redgifs.com/UnusualEqualAxolotl-poster.jpg", + "thumbnail": "https://media.redgifs.com/UnusualEqualAxolotl-mobile.jpg", + }, + "userName": "kitsuneroth", + "verified": True, +} + + +def test_map_gif_basic(): + c = map_gif(SAMPLE) + assert c is not None + assert c.source == "redgifs" and c.source_id == "unusualequalaxolotl" + assert c.creator == "kitsuneroth" and c.creator_verified is True + assert c.has_audio is True and c.duration_sec == 9.4 + assert c.width == 1080 and c.height == 1920 + assert c.hd_url.endswith("UnusualEqualAxolotl.mp4") + assert c.published_at == datetime.fromtimestamp(1789393316, tz=UTC) + # slugi, bez duplikatow, kolejnosc zachowana + assert c.tags == ["femdom", "pegging", "strap-on"] + assert c.niches == ["just-boobs"] + assert c.likes == 12 and c.views == 340 + + +def test_map_gif_skips_images_and_fileless(): + assert map_gif({**SAMPLE, "type": 2}) is None + assert map_gif({**SAMPLE, "urls": {}}) is None + assert map_gif({**SAMPLE, "id": ""}) is None + + +def test_tag_slug(): + assert tag_slug("Big Tits") == "big-tits" + assert tag_slug(" 18 Years Old!! ") == "18-years-old" + assert tag_slug("") == ""