09f389100a
Some checks are pending
Backend tests / test (push) Waiting to run
- browse_runs (migracja 0056): jeden wiersz per scraper i przebieg browse-latest - quality_watchdog: listing pusty albo bez nowych, skok lub zanik obsady, sceny bez dlugosci i miniatur, spadek udanych odtworzen per origin; fingerprint per zrodlo i rodzaj - ingest_watchdog: prog swiezosci z mediany odstepow zrodla (3x, max 14 d) zamiast stalej - duration backfill: resolvery perverzija (JSON-LD) i mypornerleak (itemprop), paczka po rowno z originow, 404/410 zamyka wiersz od razu - parsery: galaxyporn data z JSON-LD, hentaimama dlugosc, javflix aktorki, fullmovies i hdporngg data z uploadDate
34 lines
1.6 KiB
Python
34 lines
1.6 KiB
Python
"""Przebieg browse-latest per scraper (sitetag), zeby watchdog odroznial trzy rozne cisze:
|
|
listing pusty (fetch albo parser listingu padl), listing pelny ale wszystko znane (zly
|
|
endpoint, rotacja karuzeli) i zwykly brak nowych publikacji. ingest_runs tego nie daje,
|
|
bo wszystkie tuby dziela jeden Source "tube-scraper".
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import uuid
|
|
from datetime import datetime
|
|
|
|
from sqlalchemy import Boolean, DateTime, Float, Index, Integer, String, Text, func
|
|
from sqlalchemy.dialects.postgresql import UUID
|
|
from sqlalchemy.orm import Mapped, mapped_column
|
|
|
|
from app.models.base import Base
|
|
|
|
|
|
class BrowseRun(Base):
|
|
__tablename__ = "browse_runs"
|
|
__table_args__ = (Index("ix_browse_runs_sitetag_started", "sitetag", "started_at"),)
|
|
|
|
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
|
sitetag: Mapped[str] = mapped_column(String(64), nullable=False)
|
|
started_at: Mapped[datetime] = mapped_column(
|
|
DateTime(timezone=True), server_default=func.now(), nullable=False
|
|
)
|
|
elapsed_s: Mapped[float | None] = mapped_column(Float, nullable=True)
|
|
seen: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
|
new: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
|
updated: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
|
skipped: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
|
errors: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
|
failed: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
|
note: Mapped[str | None] = mapped_column(Text, nullable=True)
|