feat(clips): tabele klipow do feedu Swipe + ingest RedGifs (etap 2)
Nowy wertykal krotkich pionowych klipow tworcow, osobno od scen: inne kolumny (tworca, audio, lajki, nisze), inny cykl zycia, inna taksonomia, brak filtra 60 s. Tabele `clips` (unikalne source+source_id) i `clip_tags` z indeksem (tag, clip_id) pod skan po tagu od losowego uuid w przyszlym feedzie. Zrodlo nr 1: RedGifs, oficjalne API v2 z tokenem tymczasowym (zwiazany z IP i UA, na 401 odnowienie). Globalne `order=latest` bez search_text daje ~100 na strone, kazde zapytanie widzi maks. 10 000 wynikow, wiec katalog chodzi sie po niszach (skrypt backfillu). Pliki media.redgifs.com sa publiczne i bez podpisu (206 z VPS-a i z telefonu), do bazy wchodzi gotowy adres. Scheduler: job clips-latest co godzine (5 stron), zatrzymuje sie na stronie bez nowych. Reaper (HEAD, 403/404/410 -> dead_at) jako skrypt, upsert odzywia klip, ktory zrodlo znow podaje. API: GET /clips i /clips/stats do pilota; wlasciwy feed, pamiec widzianych i telemetria to etap 3. Sprawdzone tego dnia i odlozone: nsfwswipe (lustro v.redd.it, bez audio, stoi od 19.08), Pornhub Shorties (zrodlo nr 2, HLS z tokenem 2 h, Referer na segmentach, 0/110 performerow z kanonu).
This commit is contained in:
goon-foss
parent
f28acf8f0d
commit
4f5de92bfc
14 changed files with 880 additions and 0 deletions
70
alembic/versions/20260914_0036_clips.py
Normal file
70
alembic/versions/20260914_0036_clips.py
Normal file
|
|
@ -0,0 +1,70 @@
|
|||
"""Klipy do feedu Swipe: tabele `clips` i `clip_tags`.
|
||||
|
||||
Osobny wertykał (krótkie pionowe wideo twórców z RedGifs / Pornhub Shorties), nie sceny.
|
||||
Uzasadnienie w docstringu `app/models/clip.py`.
|
||||
|
||||
Revision ID: 0036_clips
|
||||
Revises: 0035_scenes_duration_idx
|
||||
Create Date: 2026-09-14
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
revision = "0036_clips"
|
||||
down_revision: str | None = "0035_scenes_duration_idx"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"clips",
|
||||
sa.Column("id", postgresql.UUID(as_uuid=True), server_default=sa.text("gen_random_uuid()"), nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
|
||||
sa.Column("source", sa.String(16), nullable=False),
|
||||
sa.Column("source_id", sa.String(64), nullable=False),
|
||||
sa.Column("title", sa.Text(), nullable=True),
|
||||
sa.Column("creator", sa.String(128), nullable=True),
|
||||
sa.Column("creator_verified", sa.Boolean(), nullable=False, server_default=sa.false()),
|
||||
sa.Column("duration_sec", sa.Float(), nullable=True),
|
||||
sa.Column("width", sa.Integer(), nullable=True),
|
||||
sa.Column("height", sa.Integer(), nullable=True),
|
||||
sa.Column("has_audio", sa.Boolean(), nullable=False, server_default=sa.false()),
|
||||
sa.Column("content_type", sa.String(32), nullable=True),
|
||||
sa.Column("niches", postgresql.ARRAY(sa.String(64)), nullable=False, server_default="{}"),
|
||||
sa.Column("hd_url", sa.Text(), nullable=True),
|
||||
sa.Column("sd_url", sa.Text(), nullable=True),
|
||||
sa.Column("poster_url", sa.Text(), nullable=True),
|
||||
sa.Column("thumb_url", sa.Text(), nullable=True),
|
||||
sa.Column("hls", sa.Boolean(), nullable=False, server_default=sa.false()),
|
||||
sa.Column("likes", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("views", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("published_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("dead_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("dead_reason", sa.String(128), nullable=True),
|
||||
sa.PrimaryKeyConstraint("id", name="pk_clips"),
|
||||
sa.UniqueConstraint("source", "source_id", name="uq_clips_source_source_id"),
|
||||
)
|
||||
op.create_index("ix_clips_published_at", "clips", ["published_at"])
|
||||
op.create_index("ix_clips_creator", "clips", ["creator"])
|
||||
|
||||
op.create_table(
|
||||
"clip_tags",
|
||||
sa.Column("clip_id", postgresql.UUID(as_uuid=True), nullable=False),
|
||||
sa.Column("tag", sa.String(64), nullable=False),
|
||||
sa.ForeignKeyConstraint(["clip_id"], ["clips.id"], name="fk_clip_tags_clip_id_clips", ondelete="CASCADE"),
|
||||
sa.PrimaryKeyConstraint("clip_id", "tag", name="pk_clip_tags"),
|
||||
)
|
||||
op.create_index("ix_clip_tags_tag_clip", "clip_tags", ["tag", "clip_id"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("ix_clip_tags_tag_clip", table_name="clip_tags")
|
||||
op.drop_table("clip_tags")
|
||||
op.drop_index("ix_clips_creator", table_name="clips")
|
||||
op.drop_index("ix_clips_published_at", table_name="clips")
|
||||
op.drop_table("clips")
|
||||
158
app/api/clips.py
Normal file
158
app/api/clips.py
Normal file
|
|
@ -0,0 +1,158 @@
|
|||
"""Klipy (feed Swipe), etap 2: podstawowy listing i statystyki do pilota.
|
||||
|
||||
Właściwy feed z afinicznością, pamięcią widzianych i telemetrią to etap 3
|
||||
(`/clips/feed`, `/clips/events`). Tu tylko to, czego trzeba, żeby zweryfikować ingest
|
||||
i odtwarzanie z apki/emulatora: lista najnowszych żywych klipów z filtrem po tagu,
|
||||
źródle i audio oraz zbiorcze liczniki.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.auth import require_api_key
|
||||
from app.db import get_session
|
||||
from app.models.clip import Clip, ClipTag
|
||||
|
||||
router = APIRouter(prefix="/clips", tags=["clips"], dependencies=[Depends(require_api_key)])
|
||||
|
||||
|
||||
class ClipOut(BaseModel):
|
||||
id: uuid.UUID
|
||||
source: str
|
||||
source_id: str
|
||||
title: str | None
|
||||
creator: str | None
|
||||
creator_verified: bool
|
||||
duration_sec: float | None
|
||||
width: int | None
|
||||
height: int | None
|
||||
has_audio: bool
|
||||
content_type: str | None
|
||||
niches: list[str]
|
||||
tags: list[str]
|
||||
hd_url: str | None
|
||||
sd_url: str | None
|
||||
poster_url: str | None
|
||||
thumb_url: str | None
|
||||
likes: int
|
||||
views: int
|
||||
published_at: datetime | None
|
||||
|
||||
|
||||
class ClipListOut(BaseModel):
|
||||
items: list[ClipOut]
|
||||
limit: int
|
||||
offset: int
|
||||
has_more: bool
|
||||
|
||||
|
||||
class ClipStatsOut(BaseModel):
|
||||
total: int
|
||||
alive: int
|
||||
dead: int
|
||||
with_audio: int
|
||||
verified: int
|
||||
by_source: dict[str, int]
|
||||
newest_published_at: datetime | None
|
||||
tags: int
|
||||
|
||||
|
||||
def _tags_for(session: Session, clip_ids: list[uuid.UUID]) -> dict[uuid.UUID, list[str]]:
|
||||
if not clip_ids:
|
||||
return {}
|
||||
out: dict[uuid.UUID, list[str]] = {}
|
||||
for cid, tag in session.execute(
|
||||
select(ClipTag.clip_id, ClipTag.tag).where(ClipTag.clip_id.in_(clip_ids)).order_by(ClipTag.tag)
|
||||
):
|
||||
out.setdefault(cid, []).append(tag)
|
||||
return out
|
||||
|
||||
|
||||
@router.get("", response_model=ClipListOut)
|
||||
def list_clips(
|
||||
session: Session = Depends(get_session),
|
||||
tag: str | None = Query(default=None, max_length=64),
|
||||
source: str | None = Query(default=None, max_length=16),
|
||||
creator: str | None = Query(default=None, max_length=128),
|
||||
has_audio: bool | None = None,
|
||||
max_duration_sec: float | None = Query(default=None, gt=0),
|
||||
limit: int = Query(default=50, ge=1, le=200),
|
||||
offset: int = Query(default=0, ge=0),
|
||||
) -> ClipListOut:
|
||||
q = select(Clip).where(Clip.dead_at.is_(None))
|
||||
if tag:
|
||||
q = q.where(Clip.id.in_(select(ClipTag.clip_id).where(ClipTag.tag == tag)))
|
||||
if source:
|
||||
q = q.where(Clip.source == source)
|
||||
if creator:
|
||||
q = q.where(Clip.creator == creator)
|
||||
if has_audio is not None:
|
||||
q = q.where(Clip.has_audio.is_(has_audio))
|
||||
if max_duration_sec is not None:
|
||||
q = q.where(Clip.duration_sec <= max_duration_sec)
|
||||
rows = session.execute(
|
||||
q.order_by(Clip.published_at.desc().nullslast(), Clip.id).offset(offset).limit(limit + 1)
|
||||
).scalars().all()
|
||||
has_more = len(rows) > limit
|
||||
rows = rows[:limit]
|
||||
tags = _tags_for(session, [r.id for r in rows])
|
||||
items = [
|
||||
ClipOut(
|
||||
id=r.id,
|
||||
source=r.source,
|
||||
source_id=r.source_id,
|
||||
title=r.title,
|
||||
creator=r.creator,
|
||||
creator_verified=r.creator_verified,
|
||||
duration_sec=r.duration_sec,
|
||||
width=r.width,
|
||||
height=r.height,
|
||||
has_audio=r.has_audio,
|
||||
content_type=r.content_type,
|
||||
niches=list(r.niches or []),
|
||||
tags=tags.get(r.id, []),
|
||||
hd_url=r.hd_url,
|
||||
sd_url=r.sd_url,
|
||||
poster_url=r.poster_url,
|
||||
thumb_url=r.thumb_url,
|
||||
likes=r.likes,
|
||||
views=r.views,
|
||||
published_at=r.published_at,
|
||||
)
|
||||
for r in rows
|
||||
]
|
||||
return ClipListOut(items=items, limit=limit, offset=offset, has_more=has_more)
|
||||
|
||||
|
||||
@router.get("/stats", response_model=ClipStatsOut)
|
||||
def clip_stats(session: Session = Depends(get_session)) -> ClipStatsOut:
|
||||
row = session.execute(
|
||||
select(
|
||||
func.count(),
|
||||
func.count().filter(Clip.dead_at.is_(None)),
|
||||
func.count().filter(Clip.dead_at.is_not(None)),
|
||||
func.count().filter(Clip.has_audio.is_(True), Clip.dead_at.is_(None)),
|
||||
func.count().filter(Clip.creator_verified.is_(True), Clip.dead_at.is_(None)),
|
||||
func.max(Clip.published_at),
|
||||
)
|
||||
).one()
|
||||
by_source = {
|
||||
s: n for s, n in session.execute(select(Clip.source, func.count()).group_by(Clip.source))
|
||||
}
|
||||
tags = session.execute(select(func.count(func.distinct(ClipTag.tag)))).scalar_one()
|
||||
return ClipStatsOut(
|
||||
total=row[0],
|
||||
alive=row[1],
|
||||
dead=row[2],
|
||||
with_audio=row[3],
|
||||
verified=row[4],
|
||||
by_source=by_source,
|
||||
newest_published_at=row[5],
|
||||
tags=tags,
|
||||
)
|
||||
|
|
@ -225,6 +225,12 @@ class Settings(BaseSettings):
|
|||
default=6, validation_alias="GOON_SCHED_SOURCE_STATS_HOURS"
|
||||
)
|
||||
|
||||
# Klipy do feedu Swipe (RedGifs): godzinny run globalnego „najnowsze", N stron po
|
||||
# 100 (RedGifs publikuje kilkaset klipów na godzinę). 0 = wyłączony. Backfill po
|
||||
# niszach ręcznie: `python -m scripts.clips_backfill_redgifs`.
|
||||
sched_clips_hours: int = Field(default=1, validation_alias="GOON_SCHED_CLIPS_HOURS")
|
||||
clips_latest_pages: int = Field(default=5, validation_alias="GOON_CLIPS_LATEST_PAGES")
|
||||
|
||||
# Bright Data ISP proxy (stałe IP od ISP, rozliczane ryczałtem NIE per-GB),
|
||||
# używany do ingestu HTML (scrape) tubów które blokują VPS IP twardym Cloudflare
|
||||
# 403 nawet z browser-TLS (superporn). Streamu i tak nie ruszamy proxy (tokeny CDN
|
||||
|
|
|
|||
2
app/connectors/clips/__init__.py
Normal file
2
app/connectors/clips/__init__.py
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
"""Konektory klipów (feed Swipe). Osobne od `direct_scrapers`: inny model (`Clip`),
|
||||
inny potok (upsert bez dedupu i bez matchera performerów), własne joby."""
|
||||
190
app/connectors/clips/redgifs.py
Normal file
190
app/connectors/clips/redgifs.py
Normal file
|
|
@ -0,0 +1,190 @@
|
|||
"""RedGifs, oficjalne publiczne API v2. Główne źródło klipów do feedu Swipe. Dodane 2026-09-14.
|
||||
|
||||
**Dlaczego RedGifs, a nie nsfwswipe.** Sprawdzone tego samego dnia: nsfwswipe to lustro
|
||||
wideo natywnych Reddita (v.redd.it) bez audio, z ostatnim klipem z 19.08 (stoi od
|
||||
czterech tygodni) i z losowym, nieoficjalnym API oddającym HTML. RedGifs daje JSON,
|
||||
klipy z tej minuty, 1080x1920, audio w ok. 1/3 klipów, twórcę z odznaką weryfikacji,
|
||||
lajki i widoki. Pliki `media.redgifs.com/<Id>.mp4` są publiczne, bez podpisu, grają bez
|
||||
Referera z VPS-a i z telefonu (206), więc do bazy wchodzi gotowy adres, bez ekstraktora.
|
||||
|
||||
**Token.** `GET /v2/auth/temporary` oddaje token na kilka godzin, związany z IP i User-Agentem
|
||||
wołającego: kolejne zapytania MUSZĄ iść z tymi samymi nagłówkami, inaczej 401 `WrongSender`
|
||||
(zjadło pół godziny przy pierwszym podejściu, bo curl i node miały inne UA). Na 401
|
||||
odnawiamy token raz i powtarzamy.
|
||||
|
||||
**Listing.** `GET /v2/gifs/search?order=latest&count=100&page=N` bez `search_text` to
|
||||
globalne „najnowsze" (98-100 na stronę). `order` ∈ latest|top|top7|top28|trending|score,
|
||||
NIE `new` (400 BadOrder). Każde zapytanie widzi maks. 10 000 wyników (`pages` ≤ 100,
|
||||
strona 101 → 500), więc pełny katalog chodzi się po niszach (`/v2/niches`, 1812
|
||||
społeczności, `/v2/niches/<id>/gifs`) i po twórcach (`/v2/users/<name>/search`).
|
||||
Brak nagłówków rate-limit; trzymamy ~3 zapytania/s i ustępujemy przy 429.
|
||||
|
||||
**Rekord gifa** (pola, które bierzemy): `id`, `type` (1 = wideo, 2 = obraz, pomijamy),
|
||||
`urls{hd,sd,poster,thumbnail}`, `duration`, `width/height`, `hasAudio`, `hls`, `tags[]`,
|
||||
`niches[]`, `contentType`, `sexuality[]`, `likes`, `views`, `createDate` (unix s),
|
||||
`userName`, `verified`, `cta`/`promoted` (reklama, informacyjnie).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime
|
||||
|
||||
import httpx
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
_BASE = "https://api.redgifs.com/v2"
|
||||
#: Stałe UA i Origin: token jest do nich przywiązany (patrz docstring).
|
||||
_HEADERS = {
|
||||
"User-Agent": (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/152.0.0.0 Safari/537.36"
|
||||
),
|
||||
"Accept": "application/json",
|
||||
"Origin": "https://www.redgifs.com",
|
||||
"Referer": "https://www.redgifs.com/",
|
||||
}
|
||||
_PAGE_MAX = 100
|
||||
_MIN_INTERVAL_S = 0.35
|
||||
_TAG_SLUG_RE = re.compile(r"[^a-z0-9]+")
|
||||
|
||||
|
||||
@dataclass
|
||||
class RawClip:
|
||||
source: str
|
||||
source_id: str
|
||||
title: str | None
|
||||
creator: str | None
|
||||
creator_verified: bool
|
||||
duration_sec: float | None
|
||||
width: int | None
|
||||
height: int | None
|
||||
has_audio: bool
|
||||
content_type: str | None
|
||||
niches: list[str] = field(default_factory=list)
|
||||
tags: list[str] = field(default_factory=list)
|
||||
hd_url: str | None = None
|
||||
sd_url: str | None = None
|
||||
poster_url: str | None = None
|
||||
thumb_url: str | None = None
|
||||
hls: bool = False
|
||||
likes: int = 0
|
||||
views: int = 0
|
||||
published_at: datetime | None = None
|
||||
|
||||
|
||||
def tag_slug(name: str) -> str:
|
||||
""""Big Tits" → "big-tits". Ta sama reguła dla RedGifs i (później) Pornhuba, żeby
|
||||
afiniczność liczyła się na wspólnych slugach."""
|
||||
return _TAG_SLUG_RE.sub("-", (name or "").strip().lower()).strip("-")[:64]
|
||||
|
||||
|
||||
def map_gif(g: dict) -> RawClip | None:
|
||||
"""Rekord API → RawClip. None dla obrazów, rekordów bez pliku i bez id."""
|
||||
if not isinstance(g, dict) or g.get("type") not in (1, None):
|
||||
return None
|
||||
gid = str(g.get("id") or "").strip()
|
||||
urls = g.get("urls") or {}
|
||||
hd, sd = urls.get("hd"), urls.get("sd")
|
||||
if not gid or not (hd or sd):
|
||||
return None
|
||||
created = g.get("createDate")
|
||||
published = (
|
||||
datetime.fromtimestamp(int(created), tz=UTC) if isinstance(created, (int, float)) and created > 0 else None
|
||||
)
|
||||
tags: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for t in g.get("tags") or []:
|
||||
s = tag_slug(str(t))
|
||||
if s and s not in seen:
|
||||
seen.add(s)
|
||||
tags.append(s)
|
||||
niches = [str(n)[:64] for n in (g.get("niches") or []) if n]
|
||||
dur = g.get("duration")
|
||||
return RawClip(
|
||||
source="redgifs",
|
||||
source_id=gid,
|
||||
title=(g.get("description") or None),
|
||||
creator=(g.get("userName") or None),
|
||||
creator_verified=bool(g.get("verified")),
|
||||
duration_sec=float(dur) if isinstance(dur, (int, float)) and dur > 0 else None,
|
||||
width=g.get("width") if isinstance(g.get("width"), int) else None,
|
||||
height=g.get("height") if isinstance(g.get("height"), int) else None,
|
||||
has_audio=bool(g.get("hasAudio")),
|
||||
content_type=(str(g["contentType"])[:32] if g.get("contentType") else None),
|
||||
niches=niches,
|
||||
tags=tags,
|
||||
hd_url=hd,
|
||||
sd_url=sd,
|
||||
poster_url=urls.get("poster"),
|
||||
thumb_url=urls.get("thumbnail"),
|
||||
hls=bool(g.get("hls")),
|
||||
likes=int(g.get("likes") or 0),
|
||||
views=int(g.get("views") or 0),
|
||||
published_at=published,
|
||||
)
|
||||
|
||||
|
||||
class RedgifsClient:
|
||||
"""Cienki klient z tokenem tymczasowym i odnowieniem na 401."""
|
||||
|
||||
def __init__(self, *, timeout: float = 30.0) -> None:
|
||||
self._http = httpx.Client(headers=_HEADERS, timeout=timeout)
|
||||
self._token: str | None = None
|
||||
self._last_call = 0.0
|
||||
|
||||
def close(self) -> None:
|
||||
self._http.close()
|
||||
|
||||
def _auth(self) -> str:
|
||||
r = self._http.get(f"{_BASE}/auth/temporary")
|
||||
r.raise_for_status()
|
||||
tok = (r.json() or {}).get("token")
|
||||
if not tok:
|
||||
raise RuntimeError("redgifs: brak tokenu w /auth/temporary")
|
||||
self._token = tok
|
||||
return tok
|
||||
|
||||
def _get(self, path: str, params: dict | None = None) -> dict:
|
||||
wait = _MIN_INTERVAL_S - (time.monotonic() - self._last_call)
|
||||
if wait > 0:
|
||||
time.sleep(wait)
|
||||
tok = self._token or self._auth()
|
||||
for attempt in range(3):
|
||||
r = self._http.get(f"{_BASE}{path}", params=params, headers={"Authorization": f"Bearer {tok}"})
|
||||
self._last_call = time.monotonic()
|
||||
if r.status_code == 401 and attempt == 0:
|
||||
tok = self._auth()
|
||||
continue
|
||||
if r.status_code == 429:
|
||||
time.sleep(5 * (attempt + 1))
|
||||
continue
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
raise RuntimeError(f"redgifs: {path} nie odpowiedzial poprawnie po 3 probach")
|
||||
|
||||
# ---- listingi ---------------------------------------------------------------------
|
||||
def latest(self, page: int = 1, count: int = _PAGE_MAX) -> list[dict]:
|
||||
"""Globalne „najnowsze" (bez search_text)."""
|
||||
return self._get("/gifs/search", {"order": "latest", "count": count, "page": page}).get("gifs") or []
|
||||
|
||||
def search(self, text: str, *, order: str = "latest", page: int = 1, count: int = _PAGE_MAX) -> list[dict]:
|
||||
return self._get(
|
||||
"/gifs/search", {"search_text": text, "order": order, "count": count, "page": page}
|
||||
).get("gifs") or []
|
||||
|
||||
def niches(self, page: int = 1, count: int = _PAGE_MAX) -> dict:
|
||||
"""`{"page","pages","total","niches":[{"id","name","gifs","subscribers","tags":[...]}]}`"""
|
||||
return self._get("/niches", {"count": count, "page": page})
|
||||
|
||||
def niche_gifs(self, niche_id: str, *, order: str = "latest", page: int = 1, count: int = _PAGE_MAX) -> list[dict]:
|
||||
return self._get(f"/niches/{niche_id}/gifs", {"order": order, "count": count, "page": page}).get("gifs") or []
|
||||
|
||||
def user_gifs(self, user: str, *, order: str = "latest", page: int = 1, count: int = _PAGE_MAX) -> list[dict]:
|
||||
return self._get(f"/users/{user}/search", {"order": order, "count": count, "page": page}).get("gifs") or []
|
||||
|
||||
def gif(self, gif_id: str) -> dict | None:
|
||||
return (self._get(f"/gifs/{gif_id}") or {}).get("gif")
|
||||
|
|
@ -12,6 +12,7 @@ from app.api.admin_html import mount_static
|
|||
from app.api.admin_html import router as admin_html_router
|
||||
from app.api.blacklist import router as blacklist_router
|
||||
from app.api.bug_reports import router as bug_reports_router
|
||||
from app.api.clips import router as clips_router
|
||||
from app.api.expo_updates import router as expo_updates_router
|
||||
from app.api.favorites import router as favorites_router
|
||||
from app.api.me import router as me_router
|
||||
|
|
@ -82,6 +83,7 @@ app.include_router(taxonomies_router)
|
|||
app.include_router(favorites_router)
|
||||
app.include_router(blacklist_router)
|
||||
app.include_router(bug_reports_router)
|
||||
app.include_router(clips_router)
|
||||
app.include_router(saved_searches_router)
|
||||
app.include_router(expo_updates_router)
|
||||
app.include_router(watch_router)
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
from app.models.base import Base
|
||||
from app.models.blacklist import BlacklistedPerformer, BlacklistedStudio, BlacklistedTag
|
||||
from app.models.bug_report import BugReport
|
||||
from app.models.clip import Clip, ClipTag
|
||||
from app.models.external_record import ExternalRecord
|
||||
from app.models.favorite_movie import FavoriteMovie
|
||||
from app.models.favorite_performer import FavoritePerformer
|
||||
|
|
@ -43,6 +44,8 @@ __all__ = [
|
|||
"BlacklistedStudio",
|
||||
"BlacklistedTag",
|
||||
"BugReport",
|
||||
"Clip",
|
||||
"ClipTag",
|
||||
"ExternalRecord",
|
||||
"FavoriteMovie",
|
||||
"FavoritePerformer",
|
||||
|
|
|
|||
85
app/models/clip.py
Normal file
85
app/models/clip.py
Normal file
|
|
@ -0,0 +1,85 @@
|
|||
"""Klipy do feedu Swipe (sekcja w stylu TikToka). Osobny wertykał, NIE sceny.
|
||||
|
||||
Klip to 5-120 s pionowego wideo od twórcy-amatora (RedGifs, Pornhub Shorties, nsfwswipe),
|
||||
bez obsady z kanonu TPDB, bez dedupu ze scenami i bez filtra 60 s, którym grid ukrywa
|
||||
krótkie rzeczy. Dlatego własne tabele zamiast wciskania tego w `scenes`:
|
||||
inne kolumny (twórca, audio, lajki, nisze), inny cykl życia (twórcy kasują klipy, więc
|
||||
reaper co dobę), inna taksonomia (tagi RedGifs to nie tagi Goona, 1495 vs nasze).
|
||||
|
||||
`(source, source_id)` identyfikuje klip u źródła (`redgifs` + id gifa). Adresy plików
|
||||
RedGifs są publiczne i bez podpisu (`media.redgifs.com/<Id>.mp4`), więc trzymamy je
|
||||
wprost i telefon gra direct, bez ekstraktora i bez VPS-a. Pornhub Shorties (etap
|
||||
późniejszy) będzie miał `hd_url` puste i resolve na żądanie.
|
||||
|
||||
Tagi w osobnej tabeli z indeksem `(tag, clip_id)`: feed dobiera kandydatów skanem po tagu
|
||||
od losowego uuid (jak `sort=random` w `list_scenes`), więc para w tej kolejności to jego
|
||||
główny indeks. Nisze (społeczności RedGifs) jako tablica, bo służą tylko onboardingowi
|
||||
i blacklistom, nie skanom.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import (
|
||||
Boolean,
|
||||
DateTime,
|
||||
Float,
|
||||
ForeignKey,
|
||||
Index,
|
||||
Integer,
|
||||
String,
|
||||
Text,
|
||||
UniqueConstraint,
|
||||
)
|
||||
from sqlalchemy.dialects.postgresql import ARRAY, UUID
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from app.models.base import Base, TimestampMixin, UUIDPKMixin
|
||||
|
||||
|
||||
class Clip(UUIDPKMixin, TimestampMixin, Base):
|
||||
__tablename__ = "clips"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("source", "source_id", name="uq_clips_source_source_id"),
|
||||
Index("ix_clips_published_at", "published_at"),
|
||||
Index("ix_clips_creator", "creator"),
|
||||
)
|
||||
|
||||
#: `redgifs` | `pornhub` | `nsfwswipe`
|
||||
source: Mapped[str] = mapped_column(String(16), nullable=False)
|
||||
source_id: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
title: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
#: Pseudonim twórcy u źródła (RedGifs `userName`). Nie łączymy z `performers`.
|
||||
creator: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
creator_verified: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
||||
duration_sec: Mapped[float | None] = mapped_column(Float, nullable=True)
|
||||
width: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
height: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
has_audio: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
||||
#: RedGifs `contentType`, np. "Solo Female".
|
||||
content_type: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
niches: Mapped[list[str]] = mapped_column(ARRAY(String(64)), nullable=False, default=list)
|
||||
hd_url: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
sd_url: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
poster_url: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
thumb_url: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
#: Źródło deklaruje wariant HLS (RedGifs `hls`). Informacyjne, gramy mp4.
|
||||
hls: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
||||
likes: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
views: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
#: Data publikacji u źródła (RedGifs `createDate`), do odznaki „nowe" i sortu.
|
||||
published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
dead_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
dead_reason: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
|
||||
|
||||
class ClipTag(Base):
|
||||
__tablename__ = "clip_tags"
|
||||
__table_args__ = (Index("ix_clip_tags_tag_clip", "tag", "clip_id"),)
|
||||
|
||||
clip_id: Mapped[uuid.UUID] = mapped_column(
|
||||
UUID(as_uuid=True), ForeignKey("clips.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
#: Slug tagu u źródła, małe litery, spacje jako myślniki ("big-tits").
|
||||
tag: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
165
app/scheduler/clips_ingest.py
Normal file
165
app/scheduler/clips_ingest.py
Normal file
|
|
@ -0,0 +1,165 @@
|
|||
"""Ingest klipów (feed Swipe): upsert do `clips` + `clip_tags`, joby RedGifs.
|
||||
|
||||
Potok celowo prosty, bez `_ingest_iter_into_run`: klipy nie mają dedupu, matchera
|
||||
performerów ani fingerprintów, a `(source, source_id)` jednoznacznie identyfikuje rekord.
|
||||
Upsert `ON CONFLICT ... DO UPDATE` odświeża lajki/widoki/adresy i **zdejmuje `dead_at`**
|
||||
(klip, który źródło znów podaje, żyje). Tagi tylko dokładamy (`DO NOTHING`), źródło ich
|
||||
nie odbiera.
|
||||
|
||||
Dwa tryby:
|
||||
- `run_redgifs_latest(pages)`: globalne „najnowsze", co godzinę z schedulera. 5 stron =
|
||||
~500 klipów/run, RedGifs publikuje kilkaset na godzinę, więc to łapie prawie wszystko.
|
||||
- `run_redgifs_niches(...)`: backfill po niszach (społecznościach), ręcznie ze skryptu
|
||||
albo z `docker compose run` (patrz notatka o restarcie workera). Każde zapytanie widzi
|
||||
maks. 10 000 wyników, dlatego katalog chodzi się per nisza, nie jednym listingiem.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections.abc import Iterable
|
||||
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.connectors.clips.redgifs import RawClip, RedgifsClient, map_gif
|
||||
from app.db import session_scope
|
||||
from app.models.clip import Clip, ClipTag
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def upsert_clips(session: Session, clips: Iterable[RawClip]) -> dict[str, int]:
|
||||
seen = new = 0
|
||||
for c in clips:
|
||||
seen += 1
|
||||
stmt = (
|
||||
pg_insert(Clip)
|
||||
.values(
|
||||
source=c.source,
|
||||
source_id=c.source_id,
|
||||
title=c.title,
|
||||
creator=c.creator,
|
||||
creator_verified=c.creator_verified,
|
||||
duration_sec=c.duration_sec,
|
||||
width=c.width,
|
||||
height=c.height,
|
||||
has_audio=c.has_audio,
|
||||
content_type=c.content_type,
|
||||
niches=c.niches,
|
||||
hd_url=c.hd_url,
|
||||
sd_url=c.sd_url,
|
||||
poster_url=c.poster_url,
|
||||
thumb_url=c.thumb_url,
|
||||
hls=c.hls,
|
||||
likes=c.likes,
|
||||
views=c.views,
|
||||
published_at=c.published_at,
|
||||
)
|
||||
.on_conflict_do_update(
|
||||
constraint="uq_clips_source_source_id",
|
||||
set_={
|
||||
"title": c.title,
|
||||
"creator": c.creator,
|
||||
"creator_verified": c.creator_verified,
|
||||
"duration_sec": c.duration_sec,
|
||||
"has_audio": c.has_audio,
|
||||
"niches": c.niches,
|
||||
"hd_url": c.hd_url,
|
||||
"sd_url": c.sd_url,
|
||||
"poster_url": c.poster_url,
|
||||
"thumb_url": c.thumb_url,
|
||||
"likes": c.likes,
|
||||
"views": c.views,
|
||||
"dead_at": None,
|
||||
"dead_reason": None,
|
||||
"updated_at": text("now()"),
|
||||
},
|
||||
)
|
||||
# xmax = 0 tylko dla świeżo wstawionego wiersza (sztuczka Postgresa na
|
||||
# rozróżnienie INSERT od UPDATE w jednym zapytaniu).
|
||||
.returning(Clip.id, text("(xmax = 0) AS inserted"))
|
||||
)
|
||||
row = session.execute(stmt).one()
|
||||
clip_id, inserted = row[0], bool(row[1])
|
||||
if inserted:
|
||||
new += 1
|
||||
if c.tags:
|
||||
session.execute(
|
||||
pg_insert(ClipTag)
|
||||
.values([{"clip_id": clip_id, "tag": t} for t in c.tags])
|
||||
.on_conflict_do_nothing()
|
||||
)
|
||||
return {"seen": seen, "new": new, "updated": seen - new}
|
||||
|
||||
|
||||
def _merge(total: dict[str, int], part: dict[str, int]) -> None:
|
||||
for k, v in part.items():
|
||||
total[k] = total.get(k, 0) + v
|
||||
|
||||
|
||||
def run_redgifs_latest(*, pages: int = 5) -> dict[str, int]:
|
||||
"""Globalne „najnowsze" RedGifs, `pages` stron po 100. Zatrzymuje się na pierwszej
|
||||
stronie w całości już znanej (kolejne są starsze), żeby godzinny run nie mielił
|
||||
w kółko tych samych 500 klipów."""
|
||||
total: dict[str, int] = {"seen": 0, "new": 0, "updated": 0, "errors": 0}
|
||||
client = RedgifsClient()
|
||||
try:
|
||||
for page in range(1, pages + 1):
|
||||
try:
|
||||
gifs = client.latest(page=page)
|
||||
except Exception as e:
|
||||
log.warning("clips/redgifs latest: strona %d padla: %s", page, e)
|
||||
total["errors"] += 1
|
||||
break
|
||||
clips = [c for c in (map_gif(g) for g in gifs) if c is not None]
|
||||
if not clips:
|
||||
break
|
||||
with session_scope() as session:
|
||||
part = upsert_clips(session, clips)
|
||||
_merge(total, part)
|
||||
if part["new"] == 0 and page > 1:
|
||||
break
|
||||
finally:
|
||||
client.close()
|
||||
log.info("clips/redgifs latest: %s", total)
|
||||
return total
|
||||
|
||||
|
||||
def run_redgifs_niches(
|
||||
*, niche_pages: int = 1, per_niche_pages: int = 1, order: str = "latest", skip: int = 0
|
||||
) -> dict[str, int]:
|
||||
"""Backfill po niszach: `niche_pages` stron listy nisz (100 na stronę, wg API
|
||||
najpopularniejsze pierwsze), dla każdej niszy `per_niche_pages` stron gifów."""
|
||||
total: dict[str, int] = {"seen": 0, "new": 0, "updated": 0, "errors": 0, "niches": 0}
|
||||
client = RedgifsClient()
|
||||
try:
|
||||
niche_ids: list[str] = []
|
||||
for p in range(1, niche_pages + 1):
|
||||
try:
|
||||
data = client.niches(page=p)
|
||||
except Exception as e:
|
||||
log.warning("clips/redgifs niches: lista strona %d padla: %s", p, e)
|
||||
total["errors"] += 1
|
||||
break
|
||||
niche_ids.extend(str(n.get("id")) for n in data.get("niches") or [] if n.get("id"))
|
||||
for nid in niche_ids[skip:]:
|
||||
total["niches"] += 1
|
||||
for page in range(1, per_niche_pages + 1):
|
||||
try:
|
||||
gifs = client.niche_gifs(nid, order=order, page=page)
|
||||
except Exception as e:
|
||||
log.warning("clips/redgifs niche %s strona %d padla: %s", nid, page, e)
|
||||
total["errors"] += 1
|
||||
break
|
||||
clips = [c for c in (map_gif(g) for g in gifs) if c is not None]
|
||||
if not clips:
|
||||
break
|
||||
with session_scope() as session:
|
||||
_merge(total, upsert_clips(session, clips))
|
||||
if total["niches"] % 20 == 0:
|
||||
log.info("clips/redgifs niches: %d nisz, %s", total["niches"], total)
|
||||
finally:
|
||||
client.close()
|
||||
log.info("clips/redgifs niches: %s", total)
|
||||
return total
|
||||
|
|
@ -477,6 +477,16 @@ def _job_tpdb_movie_enrich(batch: int) -> None:
|
|||
log.exception("[scheduler] tpdb-movie-enrich failed")
|
||||
|
||||
|
||||
def _job_clips_latest(pages: int) -> None:
|
||||
"""Klipy do feedu Swipe: globalne „najnowsze" RedGifs (patrz clips_ingest)."""
|
||||
from app.scheduler.clips_ingest import run_redgifs_latest
|
||||
|
||||
try:
|
||||
_run_with_timeout(lambda: run_redgifs_latest(pages=pages), label="clips-latest")
|
||||
except Exception:
|
||||
log.exception("[scheduler] clips-latest failed")
|
||||
|
||||
|
||||
def build_scheduler(cfg: dict[str, Any]) -> BlockingScheduler:
|
||||
"""Buduje scheduler na podstawie cfg dictu.
|
||||
|
||||
|
|
@ -552,6 +562,18 @@ def build_scheduler(cfg: dict[str, Any]) -> BlockingScheduler:
|
|||
cfg["browse_latest_hours"], max_pages,
|
||||
)
|
||||
|
||||
if cfg.get("clips_hours"):
|
||||
clips_pages = cfg.get("clips_latest_pages") or 5
|
||||
sched.add_job(
|
||||
lambda: _job_clips_latest(clips_pages),
|
||||
IntervalTrigger(hours=cfg["clips_hours"], start_date=INTERVAL_ANCHOR),
|
||||
id="clips_latest",
|
||||
replace_existing=True,
|
||||
max_instances=1,
|
||||
coalesce=True,
|
||||
)
|
||||
log.info("scheduler: clips-latest every %dh (pages=%d)", cfg["clips_hours"], clips_pages)
|
||||
|
||||
if cfg.get("bulk_dedup_hours"):
|
||||
sched.add_job(
|
||||
_job_bulk_dedup_performers,
|
||||
|
|
|
|||
|
|
@ -249,6 +249,9 @@ def run_forever() -> int:
|
|||
# Source ranking, przelicz source_stats (freshness/richness/health per origin)
|
||||
# dla Sites screen. Ciężki agregat → co 6h.
|
||||
"source_stats_hours": getattr(settings, "sched_source_stats_hours", 6) or None,
|
||||
# Klipy do feedu Swipe (RedGifs „najnowsze"), co godzinę, N stron po 100.
|
||||
"clips_hours": getattr(settings, "sched_clips_hours", 1) or None,
|
||||
"clips_latest_pages": getattr(settings, "clips_latest_pages", 5),
|
||||
}
|
||||
sched = build_scheduler(cfg)
|
||||
log.info("worker scheduled mode starting (jobs=%d)", len(sched.get_jobs()))
|
||||
|
|
|
|||
37
scripts/clips_backfill_redgifs.py
Normal file
37
scripts/clips_backfill_redgifs.py
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
"""Backfill klipów RedGifs po niszach (społecznościach). Do uruchamiania ręcznego.
|
||||
|
||||
Scheduler bierze tylko globalne „najnowsze" (co godzinę). Głębszy katalog chodzi się tu:
|
||||
lista nisz z API jest posortowana od najpopularniejszych, dla każdej bierzemy N stron
|
||||
po 100 gifów w wybranym porządku (`latest` = świeże, `top28` = najlepsze z miesiąca).
|
||||
Jedno zapytanie RedGifs widzi maks. 10 000 wyników, więc `--per-niche-pages` ≤ 100.
|
||||
|
||||
Długi przebieg odpalaj przez `docker compose run -d --rm worker python -m scripts...`
|
||||
(nie `exec`, patrz notatka o restarcie workera), z pliku, nie z heredoca.
|
||||
|
||||
python -m scripts.clips_backfill_redgifs --niche-pages 1 --per-niche-pages 1 # ~10k
|
||||
python -m scripts.clips_backfill_redgifs --niche-pages 19 --per-niche-pages 5 --order top28
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
from app.scheduler.clips_ingest import run_redgifs_niches
|
||||
|
||||
|
||||
def main() -> None:
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s")
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--niche-pages", type=int, default=1, help="stron listy nisz (100 nisz/strona)")
|
||||
ap.add_argument("--per-niche-pages", type=int, default=1, help="stron gifów na niszę (100/strona)")
|
||||
ap.add_argument("--order", default="latest", choices=["latest", "top", "top7", "top28", "trending", "score"])
|
||||
ap.add_argument("--skip", type=int, default=0, help="pomiń pierwsze N nisz (wznowienie)")
|
||||
a = ap.parse_args()
|
||||
out = run_redgifs_niches(
|
||||
niche_pages=a.niche_pages, per_niche_pages=a.per_niche_pages, order=a.order, skip=a.skip
|
||||
)
|
||||
print(out)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
83
scripts/clips_reap_dead.py
Normal file
83
scripts/clips_reap_dead.py
Normal file
|
|
@ -0,0 +1,83 @@
|
|||
"""Reaper martwych klipów: twórcy kasują klipy, RedGifs oddaje wtedy 404/410 na pliku.
|
||||
|
||||
Sprawdza HEAD `hd_url` (albo `sd_url`) dla próbki żywych klipów: najpierw te najdawniej
|
||||
sprawdzane (`updated_at`), bo ingest `latest` i tak odświeża świeże. 404/410/403 →
|
||||
`dead_at` + `dead_reason`; błędy sieci nie oznaczają śmierci (patrz notatka o dwóch
|
||||
kontrolach przy przemiale źródeł). Klip, który źródło znów poda w listingu, upsert
|
||||
odżywia (zdejmuje `dead_at`).
|
||||
|
||||
Uruchomienie (kontener worker):
|
||||
python -m scripts.clips_reap_dead --limit 500 # dry-run
|
||||
python -m scripts.clips_reap_dead --limit 500 --yes # zapisz
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import time
|
||||
|
||||
import httpx
|
||||
from sqlalchemy import select, text
|
||||
|
||||
from app.db import session_scope
|
||||
from app.models.clip import Clip
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
_UA = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/152.0.0.0 Safari/537.36"}
|
||||
_DEAD_CODES = {403, 404, 410}
|
||||
|
||||
|
||||
def run(*, limit: int, apply: bool) -> dict[str, int]:
|
||||
checked = dead = errors = 0
|
||||
dead_ids: list = []
|
||||
with session_scope() as session:
|
||||
rows = session.execute(
|
||||
select(Clip.id, Clip.hd_url, Clip.sd_url)
|
||||
.where(Clip.dead_at.is_(None))
|
||||
.order_by(Clip.updated_at.asc())
|
||||
.limit(limit)
|
||||
).all()
|
||||
with httpx.Client(headers=_UA, timeout=20, follow_redirects=True) as http:
|
||||
for cid, hd, sd in rows:
|
||||
url = hd or sd
|
||||
if not url:
|
||||
continue
|
||||
checked += 1
|
||||
try:
|
||||
r = http.head(url)
|
||||
except httpx.HTTPError:
|
||||
errors += 1
|
||||
continue
|
||||
if r.status_code in _DEAD_CODES:
|
||||
dead += 1
|
||||
dead_ids.append((cid, f"http {r.status_code}"))
|
||||
time.sleep(0.2)
|
||||
if apply:
|
||||
with session_scope() as session:
|
||||
for cid, reason in dead_ids:
|
||||
session.execute(
|
||||
text("UPDATE clips SET dead_at = now(), dead_reason = :r WHERE id = :id AND dead_at IS NULL"),
|
||||
{"r": reason, "id": cid},
|
||||
)
|
||||
# Sprawdzone i zywe: odswiez updated_at, zeby kolejny run bral inne.
|
||||
alive = [cid for cid, _hd, _sd in rows if cid not in {d[0] for d in dead_ids}]
|
||||
if alive:
|
||||
session.execute(
|
||||
text("UPDATE clips SET updated_at = now() WHERE id = ANY(:ids)"), {"ids": alive}
|
||||
)
|
||||
out = {"checked": checked, "dead": dead, "errors": errors, "applied": int(apply)}
|
||||
print(out)
|
||||
return out
|
||||
|
||||
|
||||
def main() -> None:
|
||||
logging.basicConfig(level=logging.INFO, format="%(message)s")
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--limit", type=int, default=500)
|
||||
ap.add_argument("--yes", action="store_true", help="zapisz dead_at (bez tego dry-run)")
|
||||
a = ap.parse_args()
|
||||
run(limit=a.limit, apply=a.yes)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
54
tests/test_redgifs_mapping.py
Normal file
54
tests/test_redgifs_mapping.py
Normal file
|
|
@ -0,0 +1,54 @@
|
|||
from datetime import UTC, datetime
|
||||
|
||||
from app.connectors.clips.redgifs import map_gif, tag_slug
|
||||
|
||||
SAMPLE = {
|
||||
"id": "unusualequalaxolotl",
|
||||
"type": 1,
|
||||
"createDate": 1789393316,
|
||||
"contentType": "Solo Female",
|
||||
"duration": 9.4,
|
||||
"hasAudio": True,
|
||||
"height": 1920,
|
||||
"width": 1080,
|
||||
"hls": True,
|
||||
"likes": 12,
|
||||
"views": 340,
|
||||
"niches": ["just-boobs"],
|
||||
"tags": ["Femdom", "Pegging", "Strap On", "Strap On"],
|
||||
"urls": {
|
||||
"sd": "https://media.redgifs.com/UnusualEqualAxolotl-mobile.mp4",
|
||||
"hd": "https://media.redgifs.com/UnusualEqualAxolotl.mp4",
|
||||
"poster": "https://media.redgifs.com/UnusualEqualAxolotl-poster.jpg",
|
||||
"thumbnail": "https://media.redgifs.com/UnusualEqualAxolotl-mobile.jpg",
|
||||
},
|
||||
"userName": "kitsuneroth",
|
||||
"verified": True,
|
||||
}
|
||||
|
||||
|
||||
def test_map_gif_basic():
|
||||
c = map_gif(SAMPLE)
|
||||
assert c is not None
|
||||
assert c.source == "redgifs" and c.source_id == "unusualequalaxolotl"
|
||||
assert c.creator == "kitsuneroth" and c.creator_verified is True
|
||||
assert c.has_audio is True and c.duration_sec == 9.4
|
||||
assert c.width == 1080 and c.height == 1920
|
||||
assert c.hd_url.endswith("UnusualEqualAxolotl.mp4")
|
||||
assert c.published_at == datetime.fromtimestamp(1789393316, tz=UTC)
|
||||
# slugi, bez duplikatow, kolejnosc zachowana
|
||||
assert c.tags == ["femdom", "pegging", "strap-on"]
|
||||
assert c.niches == ["just-boobs"]
|
||||
assert c.likes == 12 and c.views == 340
|
||||
|
||||
|
||||
def test_map_gif_skips_images_and_fileless():
|
||||
assert map_gif({**SAMPLE, "type": 2}) is None
|
||||
assert map_gif({**SAMPLE, "urls": {}}) is None
|
||||
assert map_gif({**SAMPLE, "id": ""}) is None
|
||||
|
||||
|
||||
def test_tag_slug():
|
||||
assert tag_slug("Big Tits") == "big-tits"
|
||||
assert tag_slug(" 18 Years Old!! ") == "18-years-old"
|
||||
assert tag_slug("") == ""
|
||||
Loading…
Add table
Reference in a new issue