Compare commits
209 commits
39374c0e49
...
59483951d6
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
59483951d6 | ||
|
|
b4b9b7690a | ||
|
|
7828cfcfa2 | ||
|
|
18057e46d9 | ||
|
|
9fdeaa9327 | ||
|
|
e69de90417 | ||
|
|
fa697726e9 | ||
|
|
28f56d646c | ||
|
|
77f0fb07a1 | ||
|
|
cedabae2d8 | ||
|
|
d7c334be8d | ||
|
|
ad43f4f20d | ||
|
|
426b49fbe9 | ||
|
|
15c1e4e5e1 | ||
|
|
cf0cf75a82 | ||
|
|
c419d3ab4b | ||
|
|
6583fdd185 | ||
|
|
238d03d0c6 | ||
|
|
fa60978564 | ||
|
|
7895148c0f | ||
|
|
8c7fa36437 | ||
|
|
64aee54fb6 | ||
|
|
93f4d05df8 | ||
|
|
137219b961 | ||
|
|
6b972d36ec | ||
|
|
9526d4d9f3 | ||
|
|
2f68b118c6 | ||
|
|
a5f355841a | ||
|
|
b3ddb67ced | ||
|
|
cc163e13a7 | ||
|
|
19483c026b | ||
|
|
05a35955ad | ||
|
|
813bf741b9 | ||
|
|
585e5d59f5 | ||
|
|
9a789a8551 | ||
|
|
1ca503b7be | ||
|
|
2051fc1ded | ||
|
|
55612e262b | ||
|
|
a10c51aebf | ||
|
|
b3ecf7141a | ||
|
|
cbb2390a2a | ||
|
|
2f3e57c0ac | ||
|
|
90e391e255 | ||
|
|
1875604c6d | ||
|
|
db23b63e46 | ||
|
|
c154deab37 | ||
|
|
f34a75f4c6 | ||
|
|
4afebacad8 | ||
|
|
960bc75be4 | ||
|
|
249ad49430 | ||
|
|
b643b2cb77 | ||
|
|
78d26c4bc6 | ||
|
|
ac84da92a4 | ||
|
|
0b6f663528 | ||
|
|
8b216018a2 | ||
|
|
b0e15935c6 | ||
|
|
c524b43fa3 | ||
|
|
f014a901de | ||
|
|
476cbb8d16 | ||
|
|
567a8fb3b5 | ||
|
|
e4cb94bc59 | ||
|
|
b1a530611f | ||
|
|
e77deef667 | ||
|
|
5b67aeeeaf | ||
|
|
e0e69189a8 | ||
|
|
00f4779abe | ||
|
|
bcee5851e9 | ||
|
|
0424cb9138 | ||
|
|
4b71689a95 | ||
|
|
3714afa22f | ||
|
|
8b4783771f | ||
|
|
b5d9473898 | ||
|
|
b66dd99eba | ||
|
|
81d617efc2 | ||
|
|
2a9445fe4a | ||
|
|
08410fddd1 | ||
|
|
29da1fbaa6 | ||
|
|
9269b02a4c | ||
|
|
e512665d26 | ||
|
|
32919d6a6c | ||
|
|
9d4384cef3 | ||
|
|
86b3e88f08 | ||
|
|
e618087eae | ||
|
|
a00acdddfb | ||
|
|
d1f2f035b0 | ||
|
|
aebacc0389 | ||
|
|
1654d78d59 | ||
|
|
16eb633bde | ||
|
|
aa05ce2647 | ||
|
|
072f2608b3 | ||
|
|
a9f0f94321 | ||
|
|
956a0feb22 | ||
|
|
21bc8bf1fe | ||
|
|
80fd83cb4e | ||
|
|
6de986b9a7 | ||
|
|
08079787da | ||
|
|
f7670963df | ||
|
|
fef28ae56b | ||
|
|
bb9e1afc31 | ||
|
|
32c18a6d0f | ||
|
|
adbdce1c75 | ||
|
|
200db33d78 | ||
|
|
c8baa11604 | ||
|
|
953068f0db | ||
|
|
904f8984c8 | ||
|
|
e1c7efb947 | ||
|
|
e98ef6577e | ||
|
|
abddd27856 | ||
|
|
3e8a221981 | ||
|
|
7f36865b5a | ||
|
|
576a424615 | ||
|
|
ffb80c7b60 | ||
|
|
f8b1e801ef | ||
|
|
65b9df073a | ||
|
|
a9545a7ab2 | ||
|
|
e23e2d1f17 | ||
|
|
8f34a3e2f1 | ||
|
|
940d4872e3 | ||
|
|
d4b89f16e3 | ||
|
|
7bf1fd6716 | ||
|
|
edbffc0fa7 | ||
|
|
2b602beea5 | ||
|
|
cd257740be | ||
|
|
bc72515227 | ||
|
|
c5abdc1186 | ||
|
|
43f7e1f7b2 | ||
|
|
d52641774d | ||
|
|
9f46e8dea9 | ||
|
|
4922646011 | ||
|
|
d5409d01ce | ||
|
|
63880feeb1 | ||
|
|
a196fcbcdb | ||
|
|
3339d3cd14 | ||
|
|
b18f07d90e | ||
|
|
8c0edbdf7b | ||
|
|
4d14f3946b | ||
|
|
9d0cb7f26e | ||
|
|
fad72e9cd6 | ||
|
|
3cbfb1d490 | ||
|
|
210aec0536 | ||
|
|
77323d23e6 | ||
|
|
83918e9a8d | ||
|
|
a5ec6ca991 | ||
|
|
e5b6e8968c | ||
|
|
e780e1ae6f | ||
|
|
c05bafb4c7 | ||
|
|
6e3ad870a7 | ||
|
|
c18ed24330 | ||
|
|
4f0fb1636c | ||
|
|
e42217773f | ||
|
|
ee4915770f | ||
|
|
0f19a61789 | ||
|
|
7e46e5ac48 | ||
|
|
5e74195878 | ||
|
|
58b355b6b5 | ||
|
|
d4c4b79e92 | ||
|
|
08f901712c | ||
|
|
24fc790691 | ||
|
|
983bf62416 | ||
|
|
20a8dc8e27 | ||
|
|
c0bb3f403f | ||
|
|
5896b58688 | ||
|
|
4bc594cde3 | ||
|
|
7981ba3408 | ||
|
|
bb5a97e288 | ||
|
|
817b50fbf8 | ||
|
|
cd12348782 | ||
|
|
da7fcda132 | ||
|
|
ee83ae5e97 | ||
|
|
ee1d0c7610 | ||
|
|
7b2f093d85 | ||
|
|
967123d5d6 | ||
|
|
86c9bd438b | ||
|
|
920740b76f | ||
|
|
31d9076f27 | ||
|
|
fbfe2b2bf4 | ||
|
|
00ea8c3fd4 | ||
|
|
bb4a17ae79 | ||
|
|
a3be78373f | ||
|
|
5ae5dbb201 | ||
|
|
2163fee245 | ||
|
|
9c49a69a66 | ||
|
|
b942565a6e | ||
|
|
55503136e6 | ||
|
|
0281e449fe | ||
|
|
64506690df | ||
|
|
05c0f6ef93 | ||
|
|
c19da51aff | ||
|
|
732401cef2 | ||
|
|
d87263dde9 | ||
|
|
a0481060f3 | ||
|
|
177a45eee7 | ||
|
|
aa647dcf97 | ||
|
|
172642d55c | ||
|
|
fa21a53384 | ||
|
|
6eb7cdd320 | ||
|
|
6ee0516e62 | ||
|
|
81090ca8d2 | ||
|
|
49bb65d707 | ||
|
|
aac6b10d77 | ||
|
|
7979d5fa61 | ||
|
|
545fc8f9e3 | ||
|
|
60ece70708 | ||
|
|
b99552fbb7 | ||
|
|
2fad46f934 | ||
|
|
b6e3b1cbb5 | ||
|
|
28273eda02 | ||
|
|
feef312e8d | ||
|
|
642f1ab8b8 |
275 changed files with 31132 additions and 2081 deletions
15
.gitignore
vendored
15
.gitignore
vendored
|
|
@ -71,7 +71,22 @@ mcp-logs.txt
|
||||||
# ADB / development debug artefakty (screenshots, ui dumps)
|
# ADB / development debug artefakty (screenshots, ui dumps)
|
||||||
.tmp_adb/
|
.tmp_adb/
|
||||||
|
|
||||||
|
# Marketing screenshots — kept local, hosted externally for posts/landing.
|
||||||
|
# NOT committed: explicit thumbnails risk GitHub TOS takedown.
|
||||||
|
screenshots/
|
||||||
|
marketing-shots/
|
||||||
|
|
||||||
|
# One-off local backfill/maintenance scripts (underscore prefix = throwaway,
|
||||||
|
# operated manually in the worker container, not part of the shipped codebase).
|
||||||
|
scripts/_*.py
|
||||||
|
|
||||||
|
# Launch / marketing material — local working notes, not part of the codebase.
|
||||||
|
launch/
|
||||||
|
|
||||||
# Operational deploy scripts — moved to a private companion repo. Public repo
|
# Operational deploy scripts — moved to a private companion repo. Public repo
|
||||||
# should NOT contain SSH commands, systemd units, or smoke-test playbooks
|
# should NOT contain SSH commands, systemd units, or smoke-test playbooks
|
||||||
# referencing concrete hosts.
|
# referencing concrete hosts.
|
||||||
deploy/
|
deploy/
|
||||||
|
|
||||||
|
# Local tracker tooling (nimbalyst MCP) — not for OSS repo
|
||||||
|
.nimbalyst/
|
||||||
|
|
|
||||||
86
CLAUDE.md
Normal file
86
CLAUDE.md
Normal file
|
|
@ -0,0 +1,86 @@
|
||||||
|
# Goon
|
||||||
|
|
||||||
|
Self-hosted aggregator of adult-content scene metadata. Multi-source ingest (TPDB, StashDB, 30+ tubes) → dedup → on-demand stream resolution (yt-dlp + JWPlayer unpacker) → API + Expo mobile client.
|
||||||
|
|
||||||
|
Stack: Python 3.12 (FastAPI + SQLAlchemy + Alembic), Postgres 16, React Native (Expo), Docker Compose.
|
||||||
|
Git: github.com/goon-foss/goon (remote: `goonfoss`). Public OSS repo. 18+ — see `DISCLAIMER.md`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Zmiany mobilne (Expo) → publikuj przez OTA, nie zostawiaj w gicie
|
||||||
|
|
||||||
|
Klient `mobile/` (React Native + Expo) aktualizuje się przez **self-hosted Expo Updates (OTA)**, nie przez sklep ani rebuild APK. Zmiany JS (ekrany, komponenty, `api.ts`) lecą silent.
|
||||||
|
|
||||||
|
**KRYTYCZNE: `git commit` NIE publikuje OTA.** Sam commit (ani push) nie zmienia NIC na telefonie. Po zmianie w `mobile/` trzeba zbudować bundle i opublikować:
|
||||||
|
|
||||||
|
```
|
||||||
|
# z roota repo; konkretne wartości env w .env.local / prywatnej pamięci (NIE hardcoduj IP tutaj)
|
||||||
|
GOON_VPS_SSH=<root@vps> GOON_VPS_UPDATES_DIR=/root/goon/app/static/expo-updates \
|
||||||
|
GOON_PUBLIC_UPDATES_URL=https://api.goon-foss.org/expo-updates/asset \
|
||||||
|
python scripts/publish_update.py --runtime 1.1
|
||||||
|
```
|
||||||
|
|
||||||
|
Skrypt: `expo export` (build) → scp bundla na VPS pod `app/static/expo-updates/<runtime>/<update_id>/` → przełącza `current.json` (endpoint `GET /expo-updates/manifest` serwuje aktywny). Apka pobiera przy następnym launchu.
|
||||||
|
|
||||||
|
Reguły:
|
||||||
|
- **runtime = `1.1`** (== `runtimeVersion` w `app.json`/APK). Bump TYLKO przy native change (wtedy też nowy APK przez PackageInstaller). `RUNTIME_DEFAULT` w skrypcie = 1.1.
|
||||||
|
- **Apply dwustopniowy**: 1. launch pobiera w tle, 2. launch aplikuje → po publish **restart apki 2×**.
|
||||||
|
- **Publish odpala AI** (nie człowiek, nie git-hook). Skrypt działa one-shot też na Windows git-bash (fix 2026-06-02: scp source `.`+`cwd=DIST`, `MSYS_NO_PATHCONV`, defensywne odkręcanie zmanglowanej ścieżki).
|
||||||
|
- **Backend (`app/`) to OSOBNY deploy** — scp `app/`→VPS + `docker compose restart` (worker/api). NIE przez OTA. `./app` i `./scripts` są wolumenami w compose, więc restart wystarcza (bez rebuildu obrazu).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Resolve / playback (kluczowe ustalenia)
|
||||||
|
|
||||||
|
Stream resolve jest **on-demand, per-source**. Nic nie jest pre-resolvowane do DB poza miniaturkami i metadanymi.
|
||||||
|
|
||||||
|
- **Registry ekstraktorów**: `app/extractors/__init__.py` `_REGISTRY` mapuje `sitetag` na funkcję. Natywny ekstraktor zwraca `list[StreamSource]`. `_vps_blocked_fallback.extract` = sygnał "resolve tylko w WebView na telefonie" (sprawdzaj `is_vps_blocked_fallback(sitetag)`).
|
||||||
|
- **IP-bound / CAPTCHA hosterzy resolwują się PO STRONIE TELEFONU**, nie na VPS. CDN tokeny i Cloudflare Turnstile wiążą się z IP requestera, więc VPS (Hetzner) dostaje 403 albo Turnstile gate, a residential IP telefonu przechodzi. Resolvery phone-side: `mobile/src/lib/{doodstream,packerHoster,filemoonHoster,getfileResolver,pornxpResolver}.ts`. `PlayerScreen.tsx` routuje URL `type='hoster'`: `isDoodStream`→`resolveDoodStream`, `isPackerHoster`→`resolvePackerHoster`, `isFilemoonHoster`→`resolveFilemoonHoster`.
|
||||||
|
- **DoodStream i klony** (playmogo, doply, dood.*): protokół `pass_md5` + infra `doodcdn.io` + **niewidzialny** Cloudflare Turnstile (przechodzi automatycznie w WebView z residential IP, to NIE interaktywny CAPTCHA). Tube który embeduje playmogo (sxyland, xmoviesforyou, siska) ma iść przez `_embed_iframe.extract` (wyłuskuje iframe, oddaje `type='hoster'`, telefon dood-resolwuje), NIE przez `_vps_blocked_fallback`.
|
||||||
|
- **Image proxy**: WSZYSTKIE miniaturki idą przez `/proxy/img/{token}/...` (backend dokłada Referer, którego expo-image nie wysyła → CDN by zwracał 403). Token MUSI być stabilny (`make_token(..., stable_bucket_sec=...)`), inaczej URL zmienia się co request → expo-image cache miss → re-download co fetch/launch.
|
||||||
|
- **Movies multipart** (paradisehill): backend parsuje `var videoList` i zwraca wszystkie części jako osobne `StreamLink`; mobile pokazuje je w **scrollowalnym Modalu** (nie `Alert.alert` — Androidowy AlertDialog renderuje max 3 buttony, stąd dawne "max 3 z 35").
|
||||||
|
- **Dedup**: `app/scheduler/bulk_dedup.py` (cross-source performer-shared + `phash_exact`). `merge_scenes` (`app/resolve/scene_merge.py`) przenosi refs/performers/tags/fingerprints/**playback_sources** i kasuje drop. Missing-merge bez phash: `scripts/merge_exact_title_duration.py` (ten sam performer + identyczny znormalizowany tytuł + długość co do sekundy). Over-attribution performerów bierze się z luźnego tube-searcha (hqporner wymaga WSZYSTKICH tokenów query w slug).
|
||||||
|
- **Bug reports z apki**: tabela `bug_reports` na prodzie (`screen_name`, `scene_id`, `movie_id`, `message`, `screenshot_b64`, `resolved`). To główne źródło triage'u. Screenshot dekodujesz z base64 i oglądasz.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Lokalny debugging
|
||||||
|
|
||||||
|
Hosta VPS, klucze API i wartości env trzymaj w prywatnej pamięci / `.env.local`. NIE commituj ich tu (repo jest publiczne).
|
||||||
|
|
||||||
|
- **DB / worker na prodzie** (przez ssh na VPS): `docker compose exec -T db psql -U goon -d goon -c "..."` (pamiętaj `-U goon`, bez tego "role root does not exist"). Python w workerze: `docker compose exec -T worker python3 -c "..."` albo heredoc. `from app.db import session_scope` (NIE `app.db.session`).
|
||||||
|
- **Deploy backendu**: `scp app/...` na VPS + `docker compose restart worker api`. Wolumeny `./app` i `./scripts` → restart wystarcza, bez rebuildu.
|
||||||
|
- **Windows: prawdziwy Python**. `python`/`python3` na PATH to STUB ze Sklepu Microsoft (wypisuje "Python was not found" i wychodzi). Użyj realnego interpretera: `%LOCALAPPDATA%\Programs\Python\Python3XX\python.exe` (uruchamiaj przez PowerShell). `publish_update.py` po sukcesie kończy się czysto (UTF-8 stdout wymuszony 2026-06-08, koniec mylącego exit-1 na polskich znakach).
|
||||||
|
- **Emulator Android** (podgląd odtwarzania scen i UI):
|
||||||
|
- adb: `%ANDROID_HOME%\platform-tools\adb.exe` (`ANDROID_HOME` ustawiony; SDK NIE w domyślnym `%LOCALAPPDATA%\Android\Sdk`). AVD: **`goon`**.
|
||||||
|
- Start: `emulator -avd goon -no-snapshot-load -no-boot-anim -gpu swiftshader_indirect` (w tle). Czekaj na boot: `adb wait-for-device` + poll `adb shell getprop sys.boot_completed` == `1`.
|
||||||
|
- Pakiet apki: `com.goon.mobile`. Launch: `adb shell monkey -p com.goon.mobile -c android.intent.category.LAUNCHER 1`.
|
||||||
|
- **Screencap działa bo dev/emulator build ma `FLAG_SECURE` WYŁĄCZONE.** Release build ustawia `FLAG_SECURE` (blokuje screenshoty/nagrywanie ekranu treści NSFW) — w buildzie emulatora zostawiamy off, żeby AI mógł `adb exec-out screencap -p > out.png` i zweryfikować odtwarzanie/UI.
|
||||||
|
- **OTA na emulatorze**: po publish force-stop + launch **2×** (1. pobiera bundle w tle, 2. aplikuje).
|
||||||
|
- Gesty przez adb: tap `adb shell input tap X Y` (współrzędne device, sprawdź `wm size`, zwykle 1080x2400). Long-press: `adb shell input swipe X Y X Y 700` (ten sam punkt, >300ms = `delayLongPress`). Bounds elementów: `adb shell uiautomator dump` (RN czasem nie eksponuje tekstu, natywne Alert/dialogi tak).
|
||||||
|
- **Chrome DevTools (diagnoza hosterów/playbacku — sprawdzaj, nie zgaduj)**: odpal Chrome `--remote-debugging-port=9223 --user-data-dir=<temp>`; podłącza się `chrome-devtools` MCP. KVS/dood tubes **blankują się na wykrycie CDP** (strona → about:blank) → używaj `list_network_requests(includePreservedRequests=true)` zamiast DOM. Sygnatury w trace: `pass_md5` + `doodcdn.io` = DoodStream; `kt_player(` + `license_code` = KVS; `cdn-cgi/challenge-platform` + `turnstile` = Cloudflare (sprawdź czy invisible).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Kanban (auto-aggregation)
|
||||||
|
|
||||||
|
Ten workspace ma własny Kanban (custom tracker type `kanban` w `.nimbalyst/trackers/kanban.yaml`) z kolumnami:
|
||||||
|
|
||||||
|
**Backlog → Następne → W trakcie → Czeka → Zrobione**
|
||||||
|
|
||||||
|
### Reguła auto-agregacji — WAŻNE
|
||||||
|
|
||||||
|
Kiedy podczas sesji pojawi się nowy temat / bug / pomysł / blocker, na który **nie ma teraz czasu** lub jest poboczny względem aktualnego zadania — **utwórz tracker item** zamiast tylko wspomnieć o nim w czacie:
|
||||||
|
|
||||||
|
```
|
||||||
|
mcp__nimbalyst-mcp__tracker_create(
|
||||||
|
type: "kanban",
|
||||||
|
title: "<krótki tytuł>",
|
||||||
|
status: "backlog",
|
||||||
|
description: "<1-2 zdania kontekstu + ścieżka pliku jeśli dotyczy>"
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
Powód: Jan często skupia się na pierwszej rzeczy z planu, a pozostałe znikają. Kanban jest gwarancją, że nic nie wypadnie. Jeśli wspomnisz coś w czacie i nie założysz ticketu — to się zgubi.
|
||||||
|
|
||||||
|
Status `next` = "wybrane na najbliższe dni" (3-5 max). `waiting` = zablokowane na zewnętrzne. `done` ustawia użytkownik, nigdy AI.
|
||||||
47
alembic/versions/20260528_0018_movie_play_progress.py
Normal file
47
alembic/versions/20260528_0018_movie_play_progress.py
Normal file
|
|
@ -0,0 +1,47 @@
|
||||||
|
"""movie_play_progress — pozycja odtwarzania per film (continue watching).
|
||||||
|
|
||||||
|
Revision ID: 0018_movie_play_progress
|
||||||
|
Revises: 0017_drop_realdebrid_cache
|
||||||
|
Create Date: 2026-05-28
|
||||||
|
|
||||||
|
Mirror `scene_play_progress`: pojedyncza tabela, PK=movie_id (single-user app).
|
||||||
|
position_sec + finished + last_played_at. duration_sec mirror z filmu (movies.duration_sec
|
||||||
|
może być None gdy connector go nie wyciągnął — pozwala na progress_pct mimo to).
|
||||||
|
|
||||||
|
User-report 2026-05-26 (b207ff17): "Tutaj też przydałoby się oznaczenie filmów już
|
||||||
|
obejrzanych" — sceny mają watched badge + dim, filmów brakowało.
|
||||||
|
"""
|
||||||
|
from collections.abc import Sequence
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision: str = "0018_movie_play_progress"
|
||||||
|
down_revision: str | None = "0017_drop_realdebrid_cache"
|
||||||
|
branch_labels: str | Sequence[str] | None = None
|
||||||
|
depends_on: str | Sequence[str] | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.create_table(
|
||||||
|
"movie_play_progress",
|
||||||
|
sa.Column(
|
||||||
|
"movie_id",
|
||||||
|
sa.dialects.postgresql.UUID(as_uuid=True),
|
||||||
|
sa.ForeignKey("movies.id", ondelete="CASCADE"),
|
||||||
|
primary_key=True,
|
||||||
|
),
|
||||||
|
sa.Column("position_sec", sa.Integer(), nullable=False, server_default="0"),
|
||||||
|
sa.Column("duration_sec", sa.Integer(), nullable=True),
|
||||||
|
sa.Column("finished", sa.Boolean(), nullable=False, server_default="false"),
|
||||||
|
sa.Column(
|
||||||
|
"last_played_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.func.now(),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_table("movie_play_progress")
|
||||||
54
alembic/versions/20260531_0019_taxonomy_scene_counts.py
Normal file
54
alembic/versions/20260531_0019_taxonomy_scene_counts.py
Normal file
|
|
@ -0,0 +1,54 @@
|
||||||
|
"""taxonomy scene_count denormalization — tags / performers / studios
|
||||||
|
|
||||||
|
Revision ID: 0019_taxonomy_scene_counts
|
||||||
|
Revises: 0018_movie_play_progress
|
||||||
|
Create Date: 2026-05-31
|
||||||
|
|
||||||
|
Perf fix (user-report 2026-05-31 "wolne ładowanie scen/favorites/tags"): baza urosła
|
||||||
|
do 1.69M scen / 6.3M scene_tags, a /tags?order=popular liczył scene_count dla KAŻDEGO
|
||||||
|
tagu na żywo (agregacja 6.3M scene_tags + EXISTS playback, external-merge sort 22MB) —
|
||||||
|
~4.3s, i to razy 2 (total + items). Analogicznie performers/studios + favorites.
|
||||||
|
|
||||||
|
Denormalizujemy `scene_count` na tags/performers/studios. Worker przelicza je w tle
|
||||||
|
(`_job_refresh_taxonomy_counts`, co `GOON_SCHED_TAXONOMY_COUNTS_HOURS`=3h jednym
|
||||||
|
UPDATE...FROM). Endpointy czytają gotową kolumnę + ORDER BY indexed DESC → <20ms.
|
||||||
|
|
||||||
|
scene_count = liczba scen z danym tagiem/performerem/studiem mających ≥1 ŻYWY
|
||||||
|
playback_source (dead_at IS NULL) — dokładnie ta sama definicja co dotychczasowe
|
||||||
|
live-aggregaty (has_live_playback filter w taxonomies.py / favorites.py).
|
||||||
|
|
||||||
|
Counts są do ~3h nieświeże — dla "(123)" przy filtrze i sortu "popular" bez znaczenia.
|
||||||
|
"""
|
||||||
|
from collections.abc import Sequence
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision: str = "0019_taxonomy_scene_counts"
|
||||||
|
down_revision: str | None = "0018_movie_play_progress"
|
||||||
|
branch_labels: str | Sequence[str] | None = None
|
||||||
|
depends_on: str | Sequence[str] | None = None
|
||||||
|
|
||||||
|
_TABLES = ("tags", "performers", "studios")
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
for tbl in _TABLES:
|
||||||
|
op.add_column(
|
||||||
|
tbl,
|
||||||
|
sa.Column(
|
||||||
|
"scene_count", sa.Integer(), nullable=False, server_default="0"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
# DESC index — ORDER BY scene_count DESC (sortowanie "popular").
|
||||||
|
op.create_index(
|
||||||
|
f"ix_{tbl}_scene_count",
|
||||||
|
tbl,
|
||||||
|
[sa.text("scene_count DESC")],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
for tbl in _TABLES:
|
||||||
|
op.drop_index(f"ix_{tbl}_scene_count", table_name=tbl)
|
||||||
|
op.drop_column(tbl, "scene_count")
|
||||||
33
alembic/versions/20260602_0020_scene_tags_tag_id_index.py
Normal file
33
alembic/versions/20260602_0020_scene_tags_tag_id_index.py
Normal file
|
|
@ -0,0 +1,33 @@
|
||||||
|
"""scene_tags (tag_id, scene_id) index — tag-filtered scene lists
|
||||||
|
|
||||||
|
Revision ID: 0020_scene_tags_tag_id_index
|
||||||
|
Revises: 0019_taxonomy_scene_counts
|
||||||
|
Create Date: 2026-06-02
|
||||||
|
|
||||||
|
Perf fix (2026-06-02): `/scenes?tags=<slug>` był ~6s. scene_tags PK to
|
||||||
|
(scene_id, tag_id) — wiodąca kolumna scene_id, więc lookup "sceny z tagiem X"
|
||||||
|
(tag_id → scene_id) nie miał indeksu i robił Parallel Seq Scan po 2.8M scene_tags,
|
||||||
|
materializował wszystkie pasujące sceny i sortował. Indeks (tag_id, scene_id)
|
||||||
|
pozwala planerowi znaleźć sceny danego tagu po indeksie (i z literalnym LIMIT —
|
||||||
|
patrz scenes.py — iść index-walk + early-stop zamiast gather-all+sort).
|
||||||
|
"""
|
||||||
|
from collections.abc import Sequence
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision: str = "0020_scene_tags_tag_id_index"
|
||||||
|
down_revision: str | None = "0019_taxonomy_scene_counts"
|
||||||
|
branch_labels: str | Sequence[str] | None = None
|
||||||
|
depends_on: str | Sequence[str] | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.create_index(
|
||||||
|
"ix_scene_tags_tag_id_scene_id",
|
||||||
|
"scene_tags",
|
||||||
|
["tag_id", "scene_id"],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_index("ix_scene_tags_tag_id_scene_id", table_name="scene_tags")
|
||||||
31
alembic/versions/20260607_0021_scene_tags_tag_id_stats.py
Normal file
31
alembic/versions/20260607_0021_scene_tags_tag_id_stats.py
Normal file
|
|
@ -0,0 +1,31 @@
|
||||||
|
"""scene_tags.tag_id statistics target — MCV captures common tags
|
||||||
|
|
||||||
|
Revision ID: 0021_scene_tags_tag_id_stats
|
||||||
|
Revises: 0020_scene_tags_tag_id_index
|
||||||
|
Create Date: 2026-06-07
|
||||||
|
|
||||||
|
Perf (2026-06-07): dopełnia 0020. Z literalnym tag_id (scenes.py resolvuje slug→id)
|
||||||
|
planner używa statystyk MCV kolumny scene_tags.tag_id by poznać kardynalność tagu i
|
||||||
|
wybrać index-walk po created_at zamiast materializować wszystkie scene_tags
|
||||||
|
(tag-filtered listy 4-12s → ~20ms). Domyślny statistics target (100) łapie tylko
|
||||||
|
~top-100 tagów; podnosimy do 1000 by mid-tier tagi też miały poprawne estymaty.
|
||||||
|
|
||||||
|
ANALYZE NIE w migracji (ciężki/lockujący) — autovacuum przeliczy po zmianie targetu;
|
||||||
|
na prodzie wykonano ANALYZE ręcznie przy wdrożeniu.
|
||||||
|
"""
|
||||||
|
from collections.abc import Sequence
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision: str = "0021_scene_tags_tag_id_stats"
|
||||||
|
down_revision: str | None = "0020_scene_tags_tag_id_index"
|
||||||
|
branch_labels: str | Sequence[str] | None = None
|
||||||
|
depends_on: str | Sequence[str] | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.execute("ALTER TABLE scene_tags ALTER COLUMN tag_id SET STATISTICS 1000")
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.execute("ALTER TABLE scene_tags ALTER COLUMN tag_id SET STATISTICS -1")
|
||||||
58
alembic/versions/20260608_0022_device_scoped_user_state.py
Normal file
58
alembic/versions/20260608_0022_device_scoped_user_state.py
Normal file
|
|
@ -0,0 +1,58 @@
|
||||||
|
"""device-scope user state: favorites / play-progress / blacklists
|
||||||
|
|
||||||
|
Revision ID: 0022_device_scoped_user_state
|
||||||
|
Revises: 0021_scene_tags_tag_id_stats
|
||||||
|
Create Date: 2026-06-08
|
||||||
|
|
||||||
|
Publiczna instancja nie ma kont → stan usera był GLOBALNY, nowi użytkownicy
|
||||||
|
nadpisywali ulubione/blacklisty/progress Jana (bug 2026-06-08). Dodajemy `device_id`
|
||||||
|
(VARCHAR 64) do 9 tabel stanu i przerabiamy PK na composite `(device_id, <entity>)`.
|
||||||
|
|
||||||
|
Istniejące wiersze → `device_id = 'legacy-shared'` (sentinel). Apka po update wysyła
|
||||||
|
`X-Device-Id`; `/me/adopt-legacy` przepina legacy na docelowe device. Zero utraty
|
||||||
|
danych (backfill + composite PK, nie drop).
|
||||||
|
"""
|
||||||
|
from collections.abc import Sequence
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision: str = "0022_device_scoped_user_state"
|
||||||
|
down_revision: str | None = "0021_scene_tags_tag_id_stats"
|
||||||
|
branch_labels: str | Sequence[str] | None = None
|
||||||
|
depends_on: str | Sequence[str] | None = None
|
||||||
|
|
||||||
|
LEGACY = "legacy-shared"
|
||||||
|
|
||||||
|
# (table, entity_pk_column)
|
||||||
|
_TABLES: list[tuple[str, str]] = [
|
||||||
|
("favorite_performers", "performer_id"),
|
||||||
|
("favorite_studios", "studio_id"),
|
||||||
|
("favorite_scenes", "scene_id"),
|
||||||
|
("favorite_movies", "movie_id"),
|
||||||
|
("scene_play_progress", "scene_id"),
|
||||||
|
("movie_play_progress", "movie_id"),
|
||||||
|
("blacklisted_performers", "performer_id"),
|
||||||
|
("blacklisted_studios", "studio_id"),
|
||||||
|
("blacklisted_tags", "tag_id"),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
for table, entity in _TABLES:
|
||||||
|
# 1. dodaj nullable, 2. backfill legacy, 3. NOT NULL, 4. composite PK
|
||||||
|
op.add_column(table, sa.Column("device_id", sa.String(length=64), nullable=True))
|
||||||
|
op.execute(sa.text(f"UPDATE {table} SET device_id = :d").bindparams(d=LEGACY))
|
||||||
|
op.alter_column(table, "device_id", nullable=False)
|
||||||
|
op.drop_constraint(f"pk_{table}", table, type_="primary")
|
||||||
|
op.create_primary_key(f"pk_{table}", table, ["device_id", entity])
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
for table, entity in _TABLES:
|
||||||
|
op.drop_constraint(f"pk_{table}", table, type_="primary")
|
||||||
|
# przywróć single-col PK (zakłada brak duplikatów entity po dropie device_id)
|
||||||
|
op.execute(sa.text(f"DELETE FROM {table} a USING {table} b "
|
||||||
|
f"WHERE a.ctid < b.ctid AND a.{entity} = b.{entity}"))
|
||||||
|
op.create_primary_key(f"pk_{table}", table, [entity])
|
||||||
|
op.drop_column(table, "device_id")
|
||||||
43
alembic/versions/20260612_0023_bug_report_replies.py
Normal file
43
alembic/versions/20260612_0023_bug_report_replies.py
Normal file
|
|
@ -0,0 +1,43 @@
|
||||||
|
"""bug report replies: device_id + admin response back-channel
|
||||||
|
|
||||||
|
Revision ID: 0023_bug_report_replies
|
||||||
|
Revises: 0022_device_scoped_user_state
|
||||||
|
Create Date: 2026-06-12
|
||||||
|
|
||||||
|
Dwukierunkowy kanał na zgłoszenia: `device_id` wiąże zgłoszenie z urządzeniem (z
|
||||||
|
X-Device-Id), `response`/`responded_at` to odpowiedź admina, `response_seen` steruje
|
||||||
|
kropką na FAB (false = nieprzeczytana). Wszystko nullable/default — legacy reports OK.
|
||||||
|
"""
|
||||||
|
from collections.abc import Sequence
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision: str = "0023_bug_report_replies"
|
||||||
|
down_revision: str | None = "0022_device_scoped_user_state"
|
||||||
|
branch_labels: str | Sequence[str] | None = None
|
||||||
|
depends_on: str | Sequence[str] | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.add_column("bug_reports", sa.Column("device_id", sa.String(length=64), nullable=True))
|
||||||
|
op.add_column("bug_reports", sa.Column("response", sa.Text(), nullable=True))
|
||||||
|
op.add_column(
|
||||||
|
"bug_reports",
|
||||||
|
sa.Column("responded_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
)
|
||||||
|
op.add_column(
|
||||||
|
"bug_reports",
|
||||||
|
sa.Column(
|
||||||
|
"response_seen", sa.Boolean(), nullable=False, server_default=sa.false()
|
||||||
|
),
|
||||||
|
)
|
||||||
|
op.create_index("ix_bug_reports_device_id", "bug_reports", ["device_id"])
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_index("ix_bug_reports_device_id", table_name="bug_reports")
|
||||||
|
op.drop_column("bug_reports", "response_seen")
|
||||||
|
op.drop_column("bug_reports", "responded_at")
|
||||||
|
op.drop_column("bug_reports", "response")
|
||||||
|
op.drop_column("bug_reports", "device_id")
|
||||||
43
alembic/versions/20260616_0024_saved_searches.py
Normal file
43
alembic/versions/20260616_0024_saved_searches.py
Normal file
|
|
@ -0,0 +1,43 @@
|
||||||
|
"""saved searches: per-device saved keyword queries
|
||||||
|
|
||||||
|
Revision ID: 0024_saved_searches
|
||||||
|
Revises: 0023_bug_report_replies
|
||||||
|
Create Date: 2026-06-16
|
||||||
|
|
||||||
|
Zapisane słowa kluczowe per urządzenie (user-report mobilism: sceny słabo opisane →
|
||||||
|
dodatkowe strategie wyszukiwania). Scope po device_id (X-Device-Id), unikat na
|
||||||
|
(device_id, query) żeby ten sam zapis był idempotentny.
|
||||||
|
"""
|
||||||
|
from collections.abc import Sequence
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
from sqlalchemy.dialects import postgresql
|
||||||
|
|
||||||
|
revision: str = "0024_saved_searches"
|
||||||
|
down_revision: str | None = "0023_bug_report_replies"
|
||||||
|
branch_labels: str | Sequence[str] | None = None
|
||||||
|
depends_on: str | Sequence[str] | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.create_table(
|
||||||
|
"saved_searches",
|
||||||
|
sa.Column("id", postgresql.UUID(as_uuid=True), nullable=False),
|
||||||
|
sa.Column("device_id", sa.String(length=64), nullable=False),
|
||||||
|
sa.Column("query", sa.String(length=256), nullable=False),
|
||||||
|
sa.Column(
|
||||||
|
"created_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.func.now(),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.PrimaryKeyConstraint("id", name="pk_saved_searches"),
|
||||||
|
sa.UniqueConstraint("device_id", "query", name="uq_saved_searches_device_query"),
|
||||||
|
)
|
||||||
|
op.create_index("ix_saved_searches_device_id", "saved_searches", ["device_id"])
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_index("ix_saved_searches_device_id", table_name="saved_searches")
|
||||||
|
op.drop_table("saved_searches")
|
||||||
66
alembic/versions/20260622_0025_source_ranking.py
Normal file
66
alembic/versions/20260622_0025_source_ranking.py
Normal file
|
|
@ -0,0 +1,66 @@
|
||||||
|
"""source ranking: playback telemetry + per-origin source_stats
|
||||||
|
|
||||||
|
Revision ID: 0025_source_ranking
|
||||||
|
Revises: 0024_saved_searches
|
||||||
|
Create Date: 2026-06-22
|
||||||
|
|
||||||
|
Ranking stron-źródeł na Sites screen (user request): ocena 0-5★ per origin wg
|
||||||
|
częstotliwości odświeżania, bogactwa metadanych i tego czy źródło realnie gra.
|
||||||
|
- playback_events: fire-and-forget telemetria odtwarzania z apki (sygnał health),
|
||||||
|
- source_stats: policzona offline ocena per origin (run_source_stats).
|
||||||
|
"""
|
||||||
|
from collections.abc import Sequence
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
from sqlalchemy.dialects import postgresql
|
||||||
|
|
||||||
|
revision: str = "0025_source_ranking"
|
||||||
|
down_revision: str | None = "0024_saved_searches"
|
||||||
|
branch_labels: str | Sequence[str] | None = None
|
||||||
|
depends_on: str | Sequence[str] | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.create_table(
|
||||||
|
"playback_events",
|
||||||
|
sa.Column("id", postgresql.UUID(as_uuid=True), nullable=False),
|
||||||
|
sa.Column("origin", sa.String(length=64), nullable=False),
|
||||||
|
sa.Column("scene_id", postgresql.UUID(as_uuid=True), nullable=True),
|
||||||
|
sa.Column("status", sa.String(length=16), nullable=False),
|
||||||
|
sa.Column("error_kind", sa.String(length=64), nullable=True),
|
||||||
|
sa.Column("ttff_ms", sa.Integer(), nullable=True),
|
||||||
|
sa.Column("device_id", sa.String(length=64), nullable=True),
|
||||||
|
sa.Column(
|
||||||
|
"created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False
|
||||||
|
),
|
||||||
|
sa.PrimaryKeyConstraint("id", name="pk_playback_events"),
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_playback_events_origin_created", "playback_events", ["origin", "created_at"]
|
||||||
|
)
|
||||||
|
|
||||||
|
op.create_table(
|
||||||
|
"source_stats",
|
||||||
|
sa.Column("origin", sa.String(length=64), nullable=False),
|
||||||
|
sa.Column("stars", sa.SmallInteger(), nullable=False, server_default="0"),
|
||||||
|
sa.Column("freshness", sa.SmallInteger(), nullable=False, server_default="0"),
|
||||||
|
sa.Column("richness", sa.SmallInteger(), nullable=False, server_default="0"),
|
||||||
|
sa.Column("health", sa.SmallInteger(), nullable=True),
|
||||||
|
sa.Column("scenes", sa.Integer(), nullable=False, server_default="0"),
|
||||||
|
sa.Column("new_7d", sa.Integer(), nullable=False, server_default="0"),
|
||||||
|
sa.Column("newest_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column(
|
||||||
|
"components", postgresql.JSONB(), nullable=False, server_default="{}"
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"computed_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False
|
||||||
|
),
|
||||||
|
sa.PrimaryKeyConstraint("origin", name="pk_source_stats"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_table("source_stats")
|
||||||
|
op.drop_index("ix_playback_events_origin_created", table_name="playback_events")
|
||||||
|
op.drop_table("playback_events")
|
||||||
41
alembic/versions/20260702_0026_scene_backfill_flag.py
Normal file
41
alembic/versions/20260702_0026_scene_backfill_flag.py
Normal file
|
|
@ -0,0 +1,41 @@
|
||||||
|
"""scene backfill flag: exclude bulk catalog imports from "new"
|
||||||
|
|
||||||
|
Revision ID: 0026_scene_backfill_flag
|
||||||
|
Revises: 0025_source_ranking
|
||||||
|
Create Date: 2026-07-02
|
||||||
|
|
||||||
|
Tube'y podają datę importu jako release_date, więc masowy backfill starego katalogu
|
||||||
|
(deep-crawl głębokie strony) udawał świeżość i zawyżał licznik "+N nowych" w ulubionych.
|
||||||
|
`scenes.backfill` oznacza takie sceny (pozostają widoczne, ale nie liczą się jako nowe).
|
||||||
|
Dodatkowo indeks na scene_performers(performer_id) pod zapytanie licznika ulubionych
|
||||||
|
(PK to (scene_id, performer_id), więc filtr po samym performer_id był full scanem).
|
||||||
|
|
||||||
|
Idempotentne (IF NOT EXISTS): prod dostał kolumnę/indeksy ręcznym ALTER-em zanim ta
|
||||||
|
migracja powstała, a deploy nie odpala alembic; guard chroni przed DuplicateColumn gdyby
|
||||||
|
`alembic upgrade` puszczono na prodzie albo na dumpie z prod.
|
||||||
|
"""
|
||||||
|
from collections.abc import Sequence
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision: str = "0026_scene_backfill_flag"
|
||||||
|
down_revision: str | None = "0025_source_ranking"
|
||||||
|
branch_labels: str | Sequence[str] | None = None
|
||||||
|
depends_on: str | Sequence[str] | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.execute(
|
||||||
|
"ALTER TABLE scenes ADD COLUMN IF NOT EXISTS backfill boolean NOT NULL DEFAULT false"
|
||||||
|
)
|
||||||
|
op.execute("CREATE INDEX IF NOT EXISTS ix_scenes_backfill ON scenes (backfill)")
|
||||||
|
op.execute(
|
||||||
|
"CREATE INDEX IF NOT EXISTS ix_scene_performers_performer_id "
|
||||||
|
"ON scene_performers (performer_id)"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.execute("DROP INDEX IF EXISTS ix_scene_performers_performer_id")
|
||||||
|
op.execute("DROP INDEX IF EXISTS ix_scenes_backfill")
|
||||||
|
op.execute("ALTER TABLE scenes DROP COLUMN IF EXISTS backfill")
|
||||||
|
|
@ -0,0 +1,33 @@
|
||||||
|
"""playback_events.error_detail: raw native-player error message
|
||||||
|
|
||||||
|
Revision ID: 0027_playback_event_error_detail
|
||||||
|
Revises: 0026_scene_backfill_flag
|
||||||
|
Create Date: 2026-07-05
|
||||||
|
|
||||||
|
`error_kind` grupuje błędy odtwarzania zgrubnie ('player_error'/'gone'/...), ale do
|
||||||
|
diagnozy realnych padów niereprodukowalnych na emulatorze potrzebna jest surowa treść
|
||||||
|
błędu z native playera (ExoPlayer message). `error_detail` (ucięte do 512) zbiera ją
|
||||||
|
z urządzeń przy status='error'.
|
||||||
|
|
||||||
|
Idempotentne (IF NOT EXISTS): prod dostał kolumnę ręcznym ALTER-em zanim ta migracja
|
||||||
|
powstała, a deploy nie odpala alembic; guard chroni przed DuplicateColumn gdyby
|
||||||
|
`alembic upgrade` puszczono na prodzie albo na dumpie z prod.
|
||||||
|
"""
|
||||||
|
from collections.abc import Sequence
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision: str = "0027_playback_event_error_detail"
|
||||||
|
down_revision: str | None = "0026_scene_backfill_flag"
|
||||||
|
branch_labels: str | Sequence[str] | None = None
|
||||||
|
depends_on: str | Sequence[str] | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.execute(
|
||||||
|
"ALTER TABLE playback_events ADD COLUMN IF NOT EXISTS error_detail varchar(512)"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.execute("ALTER TABLE playback_events DROP COLUMN IF EXISTS error_detail")
|
||||||
|
|
@ -20,6 +20,7 @@ from pydantic import BaseModel
|
||||||
from sqlalchemy import select
|
from sqlalchemy import select
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app.api.device import get_device_id
|
||||||
from app.auth import require_api_key
|
from app.auth import require_api_key
|
||||||
from app.db import get_session
|
from app.db import get_session
|
||||||
from app.models.blacklist import (
|
from app.models.blacklist import (
|
||||||
|
|
@ -53,20 +54,24 @@ class BlacklistOut(BaseModel):
|
||||||
@router.get("", response_model=BlacklistOut)
|
@router.get("", response_model=BlacklistOut)
|
||||||
def list_blacklist(
|
def list_blacklist(
|
||||||
session: Annotated[Session, Depends(get_session)],
|
session: Annotated[Session, Depends(get_session)],
|
||||||
|
device_id: Annotated[str, Depends(get_device_id)],
|
||||||
) -> BlacklistOut:
|
) -> BlacklistOut:
|
||||||
perfs = session.execute(
|
perfs = session.execute(
|
||||||
select(BlacklistedPerformer.performer_id, Performer.canonical_name, Performer.slug)
|
select(BlacklistedPerformer.performer_id, Performer.canonical_name, Performer.slug)
|
||||||
.join(Performer, Performer.id == BlacklistedPerformer.performer_id)
|
.join(Performer, Performer.id == BlacklistedPerformer.performer_id)
|
||||||
|
.where(BlacklistedPerformer.device_id == device_id)
|
||||||
.order_by(Performer.canonical_name)
|
.order_by(Performer.canonical_name)
|
||||||
).all()
|
).all()
|
||||||
studios = session.execute(
|
studios = session.execute(
|
||||||
select(BlacklistedStudio.studio_id, Studio.name, Studio.slug)
|
select(BlacklistedStudio.studio_id, Studio.name, Studio.slug)
|
||||||
.join(Studio, Studio.id == BlacklistedStudio.studio_id)
|
.join(Studio, Studio.id == BlacklistedStudio.studio_id)
|
||||||
|
.where(BlacklistedStudio.device_id == device_id)
|
||||||
.order_by(Studio.name)
|
.order_by(Studio.name)
|
||||||
).all()
|
).all()
|
||||||
tags = session.execute(
|
tags = session.execute(
|
||||||
select(BlacklistedTag.tag_id, Tag.name, Tag.slug)
|
select(BlacklistedTag.tag_id, Tag.name, Tag.slug)
|
||||||
.join(Tag, Tag.id == BlacklistedTag.tag_id)
|
.join(Tag, Tag.id == BlacklistedTag.tag_id)
|
||||||
|
.where(BlacklistedTag.device_id == device_id)
|
||||||
.order_by(Tag.name)
|
.order_by(Tag.name)
|
||||||
).all()
|
).all()
|
||||||
return BlacklistOut(
|
return BlacklistOut(
|
||||||
|
|
@ -91,13 +96,14 @@ def add_blacklist(
|
||||||
kind: Kind,
|
kind: Kind,
|
||||||
entity_id: uuid.UUID,
|
entity_id: uuid.UUID,
|
||||||
session: Annotated[Session, Depends(get_session)],
|
session: Annotated[Session, Depends(get_session)],
|
||||||
|
device_id: Annotated[str, Depends(get_device_id)],
|
||||||
) -> dict:
|
) -> dict:
|
||||||
bl_model, parent_model, fk = _kind_to_entity(kind)
|
bl_model, parent_model, fk = _kind_to_entity(kind)
|
||||||
if session.get(parent_model, entity_id) is None:
|
if session.get(parent_model, entity_id) is None:
|
||||||
raise HTTPException(status_code=404, detail=f"{kind} not found")
|
raise HTTPException(status_code=404, detail=f"{kind} not found")
|
||||||
if session.get(bl_model, entity_id) is not None:
|
if session.get(bl_model, (device_id, entity_id)) is not None:
|
||||||
return {"kind": kind, "id": str(entity_id), "created": False}
|
return {"kind": kind, "id": str(entity_id), "created": False}
|
||||||
session.add(bl_model(**{fk: entity_id}))
|
session.add(bl_model(**{"device_id": device_id, fk: entity_id}))
|
||||||
session.commit()
|
session.commit()
|
||||||
return {"kind": kind, "id": str(entity_id), "created": True}
|
return {"kind": kind, "id": str(entity_id), "created": True}
|
||||||
|
|
||||||
|
|
@ -107,9 +113,10 @@ def remove_blacklist(
|
||||||
kind: Kind,
|
kind: Kind,
|
||||||
entity_id: uuid.UUID,
|
entity_id: uuid.UUID,
|
||||||
session: Annotated[Session, Depends(get_session)],
|
session: Annotated[Session, Depends(get_session)],
|
||||||
|
device_id: Annotated[str, Depends(get_device_id)],
|
||||||
) -> None:
|
) -> None:
|
||||||
bl_model, _, _ = _kind_to_entity(kind)
|
bl_model, _, _ = _kind_to_entity(kind)
|
||||||
row = session.get(bl_model, entity_id)
|
row = session.get(bl_model, (device_id, entity_id))
|
||||||
if row is None:
|
if row is None:
|
||||||
return # idempotent
|
return # idempotent
|
||||||
session.delete(row)
|
session.delete(row)
|
||||||
|
|
|
||||||
|
|
@ -11,7 +11,7 @@ Screenshot to PNG/JPEG z react-native-view-shot, base64 — typowe rozmiary:
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import uuid
|
import uuid
|
||||||
from datetime import datetime
|
from datetime import datetime, timezone
|
||||||
from typing import Annotated
|
from typing import Annotated
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends, HTTPException, status
|
from fastapi import APIRouter, Depends, HTTPException, status
|
||||||
|
|
@ -19,6 +19,7 @@ from pydantic import BaseModel, Field
|
||||||
from sqlalchemy import desc, func, select
|
from sqlalchemy import desc, func, select
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app.api.device import get_device_id
|
||||||
from app.auth import require_api_key
|
from app.auth import require_api_key
|
||||||
from app.db import get_session
|
from app.db import get_session
|
||||||
from app.models.bug_report import BugReport
|
from app.models.bug_report import BugReport
|
||||||
|
|
@ -56,10 +57,30 @@ class BugReportListOut(BaseModel):
|
||||||
total: int
|
total: int
|
||||||
|
|
||||||
|
|
||||||
|
class BugReplyIn(BaseModel):
|
||||||
|
response: str = Field(min_length=1, max_length=5000)
|
||||||
|
|
||||||
|
|
||||||
|
class MyBugReportOut(BaseModel):
|
||||||
|
id: uuid.UUID
|
||||||
|
created_at: datetime
|
||||||
|
screen_name: str | None
|
||||||
|
message: str
|
||||||
|
response: str | None
|
||||||
|
responded_at: datetime | None
|
||||||
|
response_seen: bool
|
||||||
|
|
||||||
|
|
||||||
|
class MyBugReportListOut(BaseModel):
|
||||||
|
items: list[MyBugReportOut]
|
||||||
|
unseen: int
|
||||||
|
|
||||||
|
|
||||||
@router.post("/bug-reports", status_code=status.HTTP_201_CREATED)
|
@router.post("/bug-reports", status_code=status.HTTP_201_CREATED)
|
||||||
def create_bug_report(
|
def create_bug_report(
|
||||||
payload: BugReportCreate,
|
payload: BugReportCreate,
|
||||||
session: Annotated[Session, Depends(get_session)],
|
session: Annotated[Session, Depends(get_session)],
|
||||||
|
device_id: Annotated[str, Depends(get_device_id)],
|
||||||
) -> dict[str, str]:
|
) -> dict[str, str]:
|
||||||
# Smart-route entity_id: mobile Player używa `sceneId` param zarówno dla
|
# Smart-route entity_id: mobile Player używa `sceneId` param zarówno dla
|
||||||
# scen jak i movies (legacy progress tracking hack). Bez tego INSERT FK
|
# scen jak i movies (legacy progress tracking hack). Bez tego INSERT FK
|
||||||
|
|
@ -75,6 +96,7 @@ def create_bug_report(
|
||||||
|
|
||||||
br = BugReport(
|
br = BugReport(
|
||||||
id=uuid.uuid4(),
|
id=uuid.uuid4(),
|
||||||
|
device_id=device_id,
|
||||||
message=payload.message.strip(),
|
message=payload.message.strip(),
|
||||||
screen_name=payload.screen_name,
|
screen_name=payload.screen_name,
|
||||||
app_version=payload.app_version,
|
app_version=payload.app_version,
|
||||||
|
|
@ -87,6 +109,72 @@ def create_bug_report(
|
||||||
return {"id": str(br.id)}
|
return {"id": str(br.id)}
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/bug-reports/mine", response_model=MyBugReportListOut)
|
||||||
|
def list_my_bug_reports(
|
||||||
|
session: Annotated[Session, Depends(get_session)],
|
||||||
|
device_id: Annotated[str, Depends(get_device_id)],
|
||||||
|
) -> MyBugReportListOut:
|
||||||
|
"""Zgłoszenia TEGO urządzenia (+ ewentualna odpowiedź admina). Apka pokazuje je
|
||||||
|
w 'Your messages' pod FAB; `unseen` = liczba nieprzeczytanych odpowiedzi → kropka."""
|
||||||
|
rows = session.scalars(
|
||||||
|
select(BugReport)
|
||||||
|
.where(BugReport.device_id == device_id)
|
||||||
|
.order_by(desc(BugReport.created_at))
|
||||||
|
.limit(50)
|
||||||
|
).all()
|
||||||
|
items = [
|
||||||
|
MyBugReportOut(
|
||||||
|
id=r.id,
|
||||||
|
created_at=r.created_at,
|
||||||
|
screen_name=r.screen_name,
|
||||||
|
message=r.message,
|
||||||
|
response=r.response,
|
||||||
|
responded_at=r.responded_at,
|
||||||
|
response_seen=r.response_seen,
|
||||||
|
)
|
||||||
|
for r in rows
|
||||||
|
]
|
||||||
|
unseen = sum(1 for r in rows if r.response is not None and not r.response_seen)
|
||||||
|
return MyBugReportListOut(items=items, unseen=unseen)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/bug-reports/mine/seen")
|
||||||
|
def mark_my_replies_seen(
|
||||||
|
session: Annotated[Session, Depends(get_session)],
|
||||||
|
device_id: Annotated[str, Depends(get_device_id)],
|
||||||
|
) -> dict[str, int]:
|
||||||
|
"""Oznacz wszystkie odpowiedzi dla tego urządzenia jako przeczytane (gasi kropkę)."""
|
||||||
|
rows = session.scalars(
|
||||||
|
select(BugReport).where(
|
||||||
|
BugReport.device_id == device_id,
|
||||||
|
BugReport.response.is_not(None),
|
||||||
|
BugReport.response_seen.is_(False),
|
||||||
|
)
|
||||||
|
).all()
|
||||||
|
for r in rows:
|
||||||
|
r.response_seen = True
|
||||||
|
session.commit()
|
||||||
|
return {"marked": len(rows)}
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/bug-reports/{bug_id}/reply")
|
||||||
|
def reply_bug_report(
|
||||||
|
bug_id: uuid.UUID,
|
||||||
|
payload: BugReplyIn,
|
||||||
|
session: Annotated[Session, Depends(get_session)],
|
||||||
|
) -> dict[str, str]:
|
||||||
|
"""Admin/triage: zapisz odpowiedź do usera. Pojawi się w jego 'Your messages'
|
||||||
|
(kropka na FAB). Resetuje response_seen, żeby user dostał notyfikację."""
|
||||||
|
br = session.get(BugReport, bug_id)
|
||||||
|
if br is None:
|
||||||
|
raise HTTPException(status_code=404, detail="not found")
|
||||||
|
br.response = payload.response.strip()
|
||||||
|
br.responded_at = datetime.now(timezone.utc)
|
||||||
|
br.response_seen = False
|
||||||
|
session.commit()
|
||||||
|
return {"status": "replied"}
|
||||||
|
|
||||||
|
|
||||||
@router.get("/bug-reports", response_model=BugReportListOut)
|
@router.get("/bug-reports", response_model=BugReportListOut)
|
||||||
def list_bug_reports(
|
def list_bug_reports(
|
||||||
session: Annotated[Session, Depends(get_session)],
|
session: Annotated[Session, Depends(get_session)],
|
||||||
|
|
|
||||||
29
app/api/device.py
Normal file
29
app/api/device.py
Normal file
|
|
@ -0,0 +1,29 @@
|
||||||
|
"""Per-device scoping stanu usera (favorites / play-progress / blacklisty).
|
||||||
|
|
||||||
|
Publiczna instancja Goon nie ma kont. Wcześniej cały stan usera był GLOBALNY w DB →
|
||||||
|
nowi użytkownicy widzieli/nadpisywali ulubione, watched-badge i blacklisty Jana
|
||||||
|
(bug 2026-06-08). Apka generuje raz UUID instalacji (SecureStore) i wysyła go w
|
||||||
|
nagłówku `X-Device-Id`; backend scope'uje wszystkie tabele stanu po `device_id`.
|
||||||
|
|
||||||
|
Stare wiersze (sprzed migracji) mają `device_id = LEGACY_DEVICE`. Klient bez nagłówka
|
||||||
|
(stara wersja apki przed OTA) trafia również na LEGACY_DEVICE — czyli dostaje dawną
|
||||||
|
współdzieloną pulę, dopóki nie zaktualizuje bundla. Endpoint `/me/adopt-legacy`
|
||||||
|
przepina LEGACY rows na konkretne device (Jan robi to raz po update).
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import Annotated
|
||||||
|
|
||||||
|
from fastapi import Header
|
||||||
|
|
||||||
|
# Sentinel dla wierszy sprzed device-scopingu + klientów bez nagłówka.
|
||||||
|
LEGACY_DEVICE = "legacy-shared"
|
||||||
|
|
||||||
|
|
||||||
|
def get_device_id(
|
||||||
|
x_device_id: Annotated[str | None, Header(alias="X-Device-Id")] = None,
|
||||||
|
) -> str:
|
||||||
|
"""Zwraca device_id z nagłówka `X-Device-Id` (przycięty do 64 znaków).
|
||||||
|
Brak/empty → LEGACY_DEVICE (kompat ze starymi klientami)."""
|
||||||
|
v = (x_device_id or "").strip()
|
||||||
|
return v[:64] if v else LEGACY_DEVICE
|
||||||
|
|
@ -91,6 +91,12 @@ def get_asset(
|
||||||
zwykle `<update_id>/_expo/static/js/android/<hash>.js` lub
|
zwykle `<update_id>/_expo/static/js/android/<hash>.js` lub
|
||||||
`<update_id>/assets/<hash>`. Path traversal blocked przez resolve+is_relative.
|
`<update_id>/assets/<hash>`. Path traversal blocked przez resolve+is_relative.
|
||||||
"""
|
"""
|
||||||
|
# Windows publish quirk: Expo metadata.json zapisuje assets[].path z backslashami
|
||||||
|
# (os.sep) na Windowsie. publish_update.py kopiuje to do URL → manifest zawiera
|
||||||
|
# `?asset=<update>/assets\<hash>`. Na Linux backslash nie jest separatorem path-a,
|
||||||
|
# więc Path resolve nie znalazłby pliku (404 na każdy asset → mobile odrzuca cały
|
||||||
|
# update). Normalizujemy tutaj zamiast wymagać re-publishu starych bundle'i.
|
||||||
|
asset = asset.replace("\\", "/")
|
||||||
runtime_dir = (_STATIC_DIR / runtimeVersion).resolve()
|
runtime_dir = (_STATIC_DIR / runtimeVersion).resolve()
|
||||||
target = (runtime_dir / asset).resolve()
|
target = (runtime_dir / asset).resolve()
|
||||||
if not str(target).startswith(str(runtime_dir)):
|
if not str(target).startswith(str(runtime_dir)):
|
||||||
|
|
|
||||||
|
|
@ -27,9 +27,10 @@ from typing import Annotated
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends, HTTPException, status
|
from fastapi import APIRouter, Depends, HTTPException, status
|
||||||
from pydantic import BaseModel
|
from pydantic import BaseModel
|
||||||
from sqlalchemy import func, select
|
from sqlalchemy import select
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app.api.device import get_device_id
|
||||||
from app.auth import require_api_key
|
from app.auth import require_api_key
|
||||||
from app.db import get_session
|
from app.db import get_session
|
||||||
from app.models.favorite_movie import FavoriteMovie
|
from app.models.favorite_movie import FavoriteMovie
|
||||||
|
|
@ -37,7 +38,6 @@ from app.models.favorite_performer import FavoritePerformer
|
||||||
from app.models.favorite_studio import FavoriteStudio
|
from app.models.favorite_studio import FavoriteStudio
|
||||||
from app.models.movie import Movie
|
from app.models.movie import Movie
|
||||||
from app.models.performer import Performer
|
from app.models.performer import Performer
|
||||||
from app.models.playback_source import PlaybackSource
|
|
||||||
from app.models.scene import Scene, ScenePerformer
|
from app.models.scene import Scene, ScenePerformer
|
||||||
from app.models.studio import Studio
|
from app.models.studio import Studio
|
||||||
|
|
||||||
|
|
@ -45,6 +45,88 @@ router = APIRouter(
|
||||||
prefix="/favorites", tags=["favorites"], dependencies=[Depends(require_api_key)]
|
prefix="/favorites", tags=["favorites"], dependencies=[Depends(require_api_key)]
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Licznik "+N nowych" MUSI liczyć to samo, co user zobaczy po wejściu na listę scen
|
||||||
|
# performerki/studia, inaczej pokazuje +6 a lista ma 0 (report: „+6 a nic nowego").
|
||||||
|
# Rozjazdy które to powodowały: (1) licznik nie nakładał blacklist device (gay-filter/
|
||||||
|
# ukryte tagi) którą lista nakłada, (2) nie ograniczał do pierwszej strony listy
|
||||||
|
# (per_page=200, sort release_date desc), nowa scena o starej dacie wydania wpadała
|
||||||
|
# poza top-200 albo w ogóle poza to co widać, (3) dla studiów nie odsiewał stub-scen.
|
||||||
|
# Poniższe replikuje filtry i okno listy (app/api/scenes.py list_scenes, domyślne paramy
|
||||||
|
# mobile: has_playback=true, include_stubs=false). Trzymane ręcznie w zgodzie z tamtym.
|
||||||
|
_FAVORITES_PAGE_CAP = 200 # == per_page w PerformerScenesScreen/StudioScenesScreen
|
||||||
|
|
||||||
|
|
||||||
|
def _new_counts(session: Session, device_id: str, *, kind: str) -> dict:
|
||||||
|
"""Policz per-favorite "+N nowych" = sceny created_at > last_seen_at, ale TYLKO wśród
|
||||||
|
tych które user zobaczy na liście: te same filtry widoczności co list_scenes (żywy
|
||||||
|
playback + blacklist + stub) i to samo okno (top-_FAVORITES_PAGE_CAP pod sortem
|
||||||
|
release_date desc), z pominięciem backfillu. Liczy w SQL (count(*) FILTER), zwraca
|
||||||
|
{group_id: n} — bez streamowania N*200 wierszy do Pythona.
|
||||||
|
|
||||||
|
kind="performer": grupuje po ScenePerformer.performer_id, join favorite_performers.
|
||||||
|
kind="studio": grupuje po Scene.studio_id, join favorite_studios. Studia potrzebują
|
||||||
|
odsiewu stub (mogą nie mieć performera); performerki nie (mają).
|
||||||
|
"""
|
||||||
|
from sqlalchemy import and_, func
|
||||||
|
|
||||||
|
from app.api.scenes import blacklist_clauses, live_playback_exists, stub_exclusion_clause
|
||||||
|
|
||||||
|
clauses = [live_playback_exists(), *blacklist_clauses(session, device_id)]
|
||||||
|
|
||||||
|
if kind == "performer":
|
||||||
|
gid = ScenePerformer.performer_id
|
||||||
|
last_seen = FavoritePerformer.last_seen_at
|
||||||
|
base = (
|
||||||
|
select(
|
||||||
|
gid.label("gid"),
|
||||||
|
Scene.created_at.label("created_at"),
|
||||||
|
Scene.backfill.label("backfill"),
|
||||||
|
last_seen.label("last_seen"),
|
||||||
|
)
|
||||||
|
.select_from(FavoritePerformer)
|
||||||
|
.join(ScenePerformer, ScenePerformer.performer_id == FavoritePerformer.performer_id)
|
||||||
|
.join(Scene, Scene.id == ScenePerformer.scene_id)
|
||||||
|
.where(FavoritePerformer.device_id == device_id)
|
||||||
|
)
|
||||||
|
partition = ScenePerformer.performer_id
|
||||||
|
else:
|
||||||
|
clauses.append(stub_exclusion_clause())
|
||||||
|
gid = Scene.studio_id
|
||||||
|
last_seen = FavoriteStudio.last_seen_at
|
||||||
|
base = (
|
||||||
|
select(
|
||||||
|
gid.label("gid"),
|
||||||
|
Scene.created_at.label("created_at"),
|
||||||
|
Scene.backfill.label("backfill"),
|
||||||
|
last_seen.label("last_seen"),
|
||||||
|
)
|
||||||
|
.select_from(FavoriteStudio)
|
||||||
|
.join(Scene, Scene.studio_id == FavoriteStudio.studio_id)
|
||||||
|
.where(FavoriteStudio.device_id == device_id)
|
||||||
|
)
|
||||||
|
partition = Scene.studio_id
|
||||||
|
|
||||||
|
rn = func.row_number().over(
|
||||||
|
partition_by=partition,
|
||||||
|
order_by=(Scene.release_date.desc().nullslast(), Scene.created_at.desc()),
|
||||||
|
).label("rn")
|
||||||
|
inner = base.add_columns(rn).where(*clauses).subquery()
|
||||||
|
rows = session.execute(
|
||||||
|
select(
|
||||||
|
inner.c.gid,
|
||||||
|
func.count()
|
||||||
|
.filter(
|
||||||
|
and_(
|
||||||
|
inner.c.rn <= _FAVORITES_PAGE_CAP,
|
||||||
|
inner.c.backfill.is_(False),
|
||||||
|
inner.c.created_at > inner.c.last_seen,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
.label("n"),
|
||||||
|
).group_by(inner.c.gid)
|
||||||
|
).all()
|
||||||
|
return {gid_val: int(n) for gid_val, n in rows}
|
||||||
|
|
||||||
|
|
||||||
class FavoriteOut(BaseModel):
|
class FavoriteOut(BaseModel):
|
||||||
performer_id: uuid.UUID
|
performer_id: uuid.UUID
|
||||||
|
|
@ -65,71 +147,22 @@ class FavoriteListOut(BaseModel):
|
||||||
@router.get("", response_model=FavoriteListOut)
|
@router.get("", response_model=FavoriteListOut)
|
||||||
def list_favorites(
|
def list_favorites(
|
||||||
session: Annotated[Session, Depends(get_session)],
|
session: Annotated[Session, Depends(get_session)],
|
||||||
|
device_id: Annotated[str, Depends(get_device_id)],
|
||||||
) -> FavoriteListOut:
|
) -> FavoriteListOut:
|
||||||
rows = session.execute(
|
rows = session.execute(
|
||||||
select(FavoritePerformer, Performer)
|
select(FavoritePerformer, Performer)
|
||||||
.join(Performer, Performer.id == FavoritePerformer.performer_id)
|
.join(Performer, Performer.id == FavoritePerformer.performer_id)
|
||||||
|
.where(FavoritePerformer.device_id == device_id)
|
||||||
.order_by(Performer.canonical_name)
|
.order_by(Performer.canonical_name)
|
||||||
).all()
|
).all()
|
||||||
if not rows:
|
if not rows:
|
||||||
return FavoriteListOut(items=[], total=0, new_total=0)
|
return FavoriteListOut(items=[], total=0, new_total=0)
|
||||||
|
|
||||||
perf_ids = [perf.id for _, perf in rows]
|
# scene_count: czytamy zdenormalizowany Performer.scene_count (refresh w tle przez
|
||||||
last_seen_by_perf = {fav.performer_id: fav.last_seen_at for fav, _ in rows}
|
# _job_refresh_taxonomy_counts) — ta sama definicja co przed (sceny z żywym
|
||||||
|
# playback). Wcześniej grouped count z EXISTS playback per-request. Migracja 0019.
|
||||||
# Batch: scene_count per performer — filtrujemy `has_live_playback` żeby badge
|
scene_counts: dict = {perf.id: perf.scene_count for _, perf in rows}
|
||||||
# `N scenes` zgadzał się z tym co widać w PerformerScenes (mobile filtruje
|
new_counts = _new_counts(session, device_id, kind="performer")
|
||||||
# `has_playback=true`). TPDB/StashDB sync wstawia metadata-only stubs które wlicz
|
|
||||||
# by się w 2062 dla Aletta Ocean ale w profilu pokazuje tylko 499 oglądalnych.
|
|
||||||
from sqlalchemy import and_, exists
|
|
||||||
_scene_count_live_playback = exists().where(
|
|
||||||
and_(
|
|
||||||
PlaybackSource.scene_id == ScenePerformer.scene_id,
|
|
||||||
PlaybackSource.dead_at.is_(None),
|
|
||||||
)
|
|
||||||
)
|
|
||||||
scene_counts: dict = dict(
|
|
||||||
session.execute(
|
|
||||||
select(ScenePerformer.performer_id, func.count(ScenePerformer.scene_id))
|
|
||||||
.where(ScenePerformer.performer_id.in_(perf_ids))
|
|
||||||
.where(_scene_count_live_playback)
|
|
||||||
.group_by(ScenePerformer.performer_id)
|
|
||||||
).all()
|
|
||||||
)
|
|
||||||
|
|
||||||
# Batch: new_count per performer — sceny z created_at > last_seen_at favorite'a.
|
|
||||||
# Każda performerka ma INNY last_seen_at, więc warunek per-row. Trick: GREATEST jest
|
|
||||||
# nieważny — robimy CASE per row z mapowaniem perf_id → last_seen przez VALUES list.
|
|
||||||
# Prościej: jeden join + WHERE z OR po wszystkich (perf_id=X AND created_at>ts_X) —
|
|
||||||
# ale to N OR-ów. Najczystsze rozwiązanie: zapytaj per-row ale wszystkie naraz w
|
|
||||||
# SQL używając IN tuple lub sub-query. Tu korzystamy z faktu że N=14 typowo, więc
|
|
||||||
# robimy unionall albo prosty (perf_id, last_seen_at) JOIN.
|
|
||||||
new_counts: dict = {}
|
|
||||||
if perf_ids:
|
|
||||||
# Liczymy TYLKO sceny z żywym playback_source (has_live_playback). Powód:
|
|
||||||
# TPDB/StashDB sync wstawia metadata-only stubs (52 scen Danielle Renae jednego
|
|
||||||
# dnia z 0 playback) — bumpują created_at, badge `+N`, ale w PerformerScenes
|
|
||||||
# mobile filtruje `has_playback=true` → 0 widocznych. Result: user widzi +48
|
|
||||||
# ale w profilu nic nowego. Filter aligns count z faktycznie oglądalnym
|
|
||||||
# contentem ("new znalezisko" = scena którą da się odtworzyć).
|
|
||||||
from sqlalchemy import and_, exists
|
|
||||||
live_playback = exists().where(
|
|
||||||
and_(
|
|
||||||
PlaybackSource.scene_id == Scene.id,
|
|
||||||
PlaybackSource.dead_at.is_(None),
|
|
||||||
)
|
|
||||||
)
|
|
||||||
per_scene_rows = session.execute(
|
|
||||||
select(ScenePerformer.performer_id, Scene.created_at)
|
|
||||||
.join(Scene, Scene.id == ScenePerformer.scene_id)
|
|
||||||
.where(ScenePerformer.performer_id.in_(perf_ids))
|
|
||||||
.where(live_playback)
|
|
||||||
).all()
|
|
||||||
for pid, created_at in per_scene_rows:
|
|
||||||
if created_at is None:
|
|
||||||
continue
|
|
||||||
if created_at > last_seen_by_perf.get(pid):
|
|
||||||
new_counts[pid] = new_counts.get(pid, 0) + 1
|
|
||||||
|
|
||||||
items: list[FavoriteOut] = []
|
items: list[FavoriteOut] = []
|
||||||
new_total = 0
|
new_total = 0
|
||||||
|
|
@ -163,14 +196,15 @@ class FavoriteAddOut(BaseModel):
|
||||||
def add_favorite(
|
def add_favorite(
|
||||||
performer_id: uuid.UUID,
|
performer_id: uuid.UUID,
|
||||||
session: Annotated[Session, Depends(get_session)],
|
session: Annotated[Session, Depends(get_session)],
|
||||||
|
device_id: Annotated[str, Depends(get_device_id)],
|
||||||
) -> FavoriteAddOut:
|
) -> FavoriteAddOut:
|
||||||
perf = session.get(Performer, performer_id)
|
perf = session.get(Performer, performer_id)
|
||||||
if perf is None:
|
if perf is None:
|
||||||
raise HTTPException(status_code=404, detail="performer not found")
|
raise HTTPException(status_code=404, detail="performer not found")
|
||||||
existing = session.get(FavoritePerformer, performer_id)
|
existing = session.get(FavoritePerformer, (device_id, performer_id))
|
||||||
if existing is not None:
|
if existing is not None:
|
||||||
return FavoriteAddOut(performer_id=performer_id, created=False)
|
return FavoriteAddOut(performer_id=performer_id, created=False)
|
||||||
session.add(FavoritePerformer(performer_id=performer_id))
|
session.add(FavoritePerformer(device_id=device_id, performer_id=performer_id))
|
||||||
session.commit()
|
session.commit()
|
||||||
return FavoriteAddOut(performer_id=performer_id, created=True)
|
return FavoriteAddOut(performer_id=performer_id, created=True)
|
||||||
|
|
||||||
|
|
@ -179,8 +213,9 @@ def add_favorite(
|
||||||
def remove_favorite(
|
def remove_favorite(
|
||||||
performer_id: uuid.UUID,
|
performer_id: uuid.UUID,
|
||||||
session: Annotated[Session, Depends(get_session)],
|
session: Annotated[Session, Depends(get_session)],
|
||||||
|
device_id: Annotated[str, Depends(get_device_id)],
|
||||||
) -> None:
|
) -> None:
|
||||||
fav = session.get(FavoritePerformer, performer_id)
|
fav = session.get(FavoritePerformer, (device_id, performer_id))
|
||||||
if fav is None:
|
if fav is None:
|
||||||
# idempotent — brak ulubionego = nie ma nic do usunięcia, success
|
# idempotent — brak ulubionego = nie ma nic do usunięcia, success
|
||||||
return
|
return
|
||||||
|
|
@ -197,8 +232,9 @@ class SeenOut(BaseModel):
|
||||||
def mark_seen(
|
def mark_seen(
|
||||||
performer_id: uuid.UUID,
|
performer_id: uuid.UUID,
|
||||||
session: Annotated[Session, Depends(get_session)],
|
session: Annotated[Session, Depends(get_session)],
|
||||||
|
device_id: Annotated[str, Depends(get_device_id)],
|
||||||
) -> SeenOut:
|
) -> SeenOut:
|
||||||
fav = session.get(FavoritePerformer, performer_id)
|
fav = session.get(FavoritePerformer, (device_id, performer_id))
|
||||||
if fav is None:
|
if fav is None:
|
||||||
raise HTTPException(status_code=404, detail="not in favorites")
|
raise HTTPException(status_code=404, detail="not in favorites")
|
||||||
fav.last_seen_at = datetime.now(UTC)
|
fav.last_seen_at = datetime.now(UTC)
|
||||||
|
|
@ -228,55 +264,20 @@ class FavoriteStudioListOut(BaseModel):
|
||||||
@router.get("/studios", response_model=FavoriteStudioListOut)
|
@router.get("/studios", response_model=FavoriteStudioListOut)
|
||||||
def list_favorite_studios(
|
def list_favorite_studios(
|
||||||
session: Annotated[Session, Depends(get_session)],
|
session: Annotated[Session, Depends(get_session)],
|
||||||
|
device_id: Annotated[str, Depends(get_device_id)],
|
||||||
) -> FavoriteStudioListOut:
|
) -> FavoriteStudioListOut:
|
||||||
rows = session.execute(
|
rows = session.execute(
|
||||||
select(FavoriteStudio, Studio)
|
select(FavoriteStudio, Studio)
|
||||||
.join(Studio, Studio.id == FavoriteStudio.studio_id)
|
.join(Studio, Studio.id == FavoriteStudio.studio_id)
|
||||||
|
.where(FavoriteStudio.device_id == device_id)
|
||||||
.order_by(Studio.name)
|
.order_by(Studio.name)
|
||||||
).all()
|
).all()
|
||||||
if not rows:
|
if not rows:
|
||||||
return FavoriteStudioListOut(items=[], total=0, new_total=0)
|
return FavoriteStudioListOut(items=[], total=0, new_total=0)
|
||||||
|
|
||||||
studio_ids = [st.id for _, st in rows]
|
# scene_count: zdenormalizowany Studio.scene_count (refresh w tle, migracja 0019).
|
||||||
last_seen_by_studio = {fav.studio_id: fav.last_seen_at for fav, _ in rows}
|
scene_counts: dict = {st.id: st.scene_count for _, st in rows}
|
||||||
|
new_counts = _new_counts(session, device_id, kind="studio")
|
||||||
# has_live_playback filter — patrz `list_favorites` (performers) wyżej.
|
|
||||||
from sqlalchemy import and_, exists
|
|
||||||
_studio_count_live_playback = exists().where(
|
|
||||||
and_(
|
|
||||||
PlaybackSource.scene_id == Scene.id,
|
|
||||||
PlaybackSource.dead_at.is_(None),
|
|
||||||
)
|
|
||||||
)
|
|
||||||
scene_counts: dict = dict(
|
|
||||||
session.execute(
|
|
||||||
select(Scene.studio_id, func.count(Scene.id))
|
|
||||||
.where(Scene.studio_id.in_(studio_ids))
|
|
||||||
.where(_studio_count_live_playback)
|
|
||||||
.group_by(Scene.studio_id)
|
|
||||||
).all()
|
|
||||||
)
|
|
||||||
|
|
||||||
new_counts: dict = {}
|
|
||||||
if studio_ids:
|
|
||||||
# has_live_playback filter — patrz `list_favorites` (performers) wyżej.
|
|
||||||
from sqlalchemy import and_, exists
|
|
||||||
live_playback = exists().where(
|
|
||||||
and_(
|
|
||||||
PlaybackSource.scene_id == Scene.id,
|
|
||||||
PlaybackSource.dead_at.is_(None),
|
|
||||||
)
|
|
||||||
)
|
|
||||||
per_scene_rows = session.execute(
|
|
||||||
select(Scene.studio_id, Scene.created_at)
|
|
||||||
.where(Scene.studio_id.in_(studio_ids))
|
|
||||||
.where(live_playback)
|
|
||||||
).all()
|
|
||||||
for sid, created_at in per_scene_rows:
|
|
||||||
if created_at is None:
|
|
||||||
continue
|
|
||||||
if created_at > last_seen_by_studio.get(sid):
|
|
||||||
new_counts[sid] = new_counts.get(sid, 0) + 1
|
|
||||||
|
|
||||||
items: list[FavoriteStudioOut] = []
|
items: list[FavoriteStudioOut] = []
|
||||||
new_total = 0
|
new_total = 0
|
||||||
|
|
@ -311,14 +312,15 @@ class FavoriteStudioAddOut(BaseModel):
|
||||||
def add_favorite_studio(
|
def add_favorite_studio(
|
||||||
studio_id: uuid.UUID,
|
studio_id: uuid.UUID,
|
||||||
session: Annotated[Session, Depends(get_session)],
|
session: Annotated[Session, Depends(get_session)],
|
||||||
|
device_id: Annotated[str, Depends(get_device_id)],
|
||||||
) -> FavoriteStudioAddOut:
|
) -> FavoriteStudioAddOut:
|
||||||
st = session.get(Studio, studio_id)
|
st = session.get(Studio, studio_id)
|
||||||
if st is None:
|
if st is None:
|
||||||
raise HTTPException(status_code=404, detail="studio not found")
|
raise HTTPException(status_code=404, detail="studio not found")
|
||||||
existing = session.get(FavoriteStudio, studio_id)
|
existing = session.get(FavoriteStudio, (device_id, studio_id))
|
||||||
if existing is not None:
|
if existing is not None:
|
||||||
return FavoriteStudioAddOut(studio_id=studio_id, created=False)
|
return FavoriteStudioAddOut(studio_id=studio_id, created=False)
|
||||||
session.add(FavoriteStudio(studio_id=studio_id))
|
session.add(FavoriteStudio(device_id=device_id, studio_id=studio_id))
|
||||||
session.commit()
|
session.commit()
|
||||||
return FavoriteStudioAddOut(studio_id=studio_id, created=True)
|
return FavoriteStudioAddOut(studio_id=studio_id, created=True)
|
||||||
|
|
||||||
|
|
@ -327,8 +329,9 @@ def add_favorite_studio(
|
||||||
def remove_favorite_studio(
|
def remove_favorite_studio(
|
||||||
studio_id: uuid.UUID,
|
studio_id: uuid.UUID,
|
||||||
session: Annotated[Session, Depends(get_session)],
|
session: Annotated[Session, Depends(get_session)],
|
||||||
|
device_id: Annotated[str, Depends(get_device_id)],
|
||||||
) -> None:
|
) -> None:
|
||||||
fav = session.get(FavoriteStudio, studio_id)
|
fav = session.get(FavoriteStudio, (device_id, studio_id))
|
||||||
if fav is None:
|
if fav is None:
|
||||||
return
|
return
|
||||||
session.delete(fav)
|
session.delete(fav)
|
||||||
|
|
@ -344,8 +347,9 @@ class SeenStudioOut(BaseModel):
|
||||||
def mark_studio_seen(
|
def mark_studio_seen(
|
||||||
studio_id: uuid.UUID,
|
studio_id: uuid.UUID,
|
||||||
session: Annotated[Session, Depends(get_session)],
|
session: Annotated[Session, Depends(get_session)],
|
||||||
|
device_id: Annotated[str, Depends(get_device_id)],
|
||||||
) -> SeenStudioOut:
|
) -> SeenStudioOut:
|
||||||
fav = session.get(FavoriteStudio, studio_id)
|
fav = session.get(FavoriteStudio, (device_id, studio_id))
|
||||||
if fav is None:
|
if fav is None:
|
||||||
raise HTTPException(status_code=404, detail="not in favorites")
|
raise HTTPException(status_code=404, detail="not in favorites")
|
||||||
fav.last_seen_at = datetime.now(UTC)
|
fav.last_seen_at = datetime.now(UTC)
|
||||||
|
|
@ -379,11 +383,13 @@ class FavoriteMovieListOut(BaseModel):
|
||||||
@router.get("/movies", response_model=FavoriteMovieListOut)
|
@router.get("/movies", response_model=FavoriteMovieListOut)
|
||||||
def list_favorite_movies(
|
def list_favorite_movies(
|
||||||
session: Annotated[Session, Depends(get_session)],
|
session: Annotated[Session, Depends(get_session)],
|
||||||
|
device_id: Annotated[str, Depends(get_device_id)],
|
||||||
) -> FavoriteMovieListOut:
|
) -> FavoriteMovieListOut:
|
||||||
rows = session.execute(
|
rows = session.execute(
|
||||||
select(FavoriteMovie, Movie, Studio)
|
select(FavoriteMovie, Movie, Studio)
|
||||||
.join(Movie, Movie.id == FavoriteMovie.movie_id)
|
.join(Movie, Movie.id == FavoriteMovie.movie_id)
|
||||||
.outerjoin(Studio, Studio.id == Movie.studio_id)
|
.outerjoin(Studio, Studio.id == Movie.studio_id)
|
||||||
|
.where(FavoriteMovie.device_id == device_id)
|
||||||
.order_by(Movie.title)
|
.order_by(Movie.title)
|
||||||
).all()
|
).all()
|
||||||
items = [
|
items = [
|
||||||
|
|
@ -415,14 +421,15 @@ class FavoriteMovieAddOut(BaseModel):
|
||||||
def add_favorite_movie(
|
def add_favorite_movie(
|
||||||
movie_id: uuid.UUID,
|
movie_id: uuid.UUID,
|
||||||
session: Annotated[Session, Depends(get_session)],
|
session: Annotated[Session, Depends(get_session)],
|
||||||
|
device_id: Annotated[str, Depends(get_device_id)],
|
||||||
) -> FavoriteMovieAddOut:
|
) -> FavoriteMovieAddOut:
|
||||||
movie = session.get(Movie, movie_id)
|
movie = session.get(Movie, movie_id)
|
||||||
if movie is None:
|
if movie is None:
|
||||||
raise HTTPException(status_code=404, detail="movie not found")
|
raise HTTPException(status_code=404, detail="movie not found")
|
||||||
existing = session.get(FavoriteMovie, movie_id)
|
existing = session.get(FavoriteMovie, (device_id, movie_id))
|
||||||
if existing is not None:
|
if existing is not None:
|
||||||
return FavoriteMovieAddOut(movie_id=movie_id, created=False)
|
return FavoriteMovieAddOut(movie_id=movie_id, created=False)
|
||||||
session.add(FavoriteMovie(movie_id=movie_id))
|
session.add(FavoriteMovie(device_id=device_id, movie_id=movie_id))
|
||||||
session.commit()
|
session.commit()
|
||||||
return FavoriteMovieAddOut(movie_id=movie_id, created=True)
|
return FavoriteMovieAddOut(movie_id=movie_id, created=True)
|
||||||
|
|
||||||
|
|
@ -431,8 +438,9 @@ def add_favorite_movie(
|
||||||
def remove_favorite_movie(
|
def remove_favorite_movie(
|
||||||
movie_id: uuid.UUID,
|
movie_id: uuid.UUID,
|
||||||
session: Annotated[Session, Depends(get_session)],
|
session: Annotated[Session, Depends(get_session)],
|
||||||
|
device_id: Annotated[str, Depends(get_device_id)],
|
||||||
) -> None:
|
) -> None:
|
||||||
fav = session.get(FavoriteMovie, movie_id)
|
fav = session.get(FavoriteMovie, (device_id, movie_id))
|
||||||
if fav is None:
|
if fav is None:
|
||||||
return
|
return
|
||||||
session.delete(fav)
|
session.delete(fav)
|
||||||
|
|
@ -448,8 +456,9 @@ class SeenMovieOut(BaseModel):
|
||||||
def mark_movie_seen(
|
def mark_movie_seen(
|
||||||
movie_id: uuid.UUID,
|
movie_id: uuid.UUID,
|
||||||
session: Annotated[Session, Depends(get_session)],
|
session: Annotated[Session, Depends(get_session)],
|
||||||
|
device_id: Annotated[str, Depends(get_device_id)],
|
||||||
) -> SeenMovieOut:
|
) -> SeenMovieOut:
|
||||||
fav = session.get(FavoriteMovie, movie_id)
|
fav = session.get(FavoriteMovie, (device_id, movie_id))
|
||||||
if fav is None:
|
if fav is None:
|
||||||
raise HTTPException(status_code=404, detail="not in favorites")
|
raise HTTPException(status_code=404, detail="not in favorites")
|
||||||
fav.last_seen_at = datetime.now(UTC)
|
fav.last_seen_at = datetime.now(UTC)
|
||||||
|
|
|
||||||
68
app/api/me.py
Normal file
68
app/api/me.py
Normal file
|
|
@ -0,0 +1,68 @@
|
||||||
|
"""Per-device self-service: przejęcie legacy stanu usera.
|
||||||
|
|
||||||
|
Po migracji device-scopingu (0022) stare wiersze mają `device_id='legacy-shared'`.
|
||||||
|
`POST /me/adopt-legacy` przepina WSZYSTKIE legacy wiersze (favorites/progress/blacklisty)
|
||||||
|
na device wołającego. Robi to JEDEN raz właściciel instancji po update apki — kolejne
|
||||||
|
wywołania nic nie znajdą (legacy już puste). Jeśli na keep-device istnieje już wiersz
|
||||||
|
dla tej samej encji, legacy duplikat jest pomijany (ON CONFLICT DO NOTHING → potem
|
||||||
|
kasujemy resztki legacy).
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import Annotated
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Depends
|
||||||
|
from pydantic import BaseModel
|
||||||
|
from sqlalchemy import text
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app.api.device import LEGACY_DEVICE, get_device_id
|
||||||
|
from app.auth import require_api_key
|
||||||
|
from app.db import get_session
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/me", tags=["me"], dependencies=[Depends(require_api_key)])
|
||||||
|
|
||||||
|
# (tabela, kolumna-encji) — wszystkie tabele device-scoped (migracja 0022).
|
||||||
|
_TABLES: list[tuple[str, str]] = [
|
||||||
|
("favorite_performers", "performer_id"),
|
||||||
|
("favorite_studios", "studio_id"),
|
||||||
|
("favorite_scenes", "scene_id"),
|
||||||
|
("favorite_movies", "movie_id"),
|
||||||
|
("scene_play_progress", "scene_id"),
|
||||||
|
("movie_play_progress", "movie_id"),
|
||||||
|
("blacklisted_performers", "performer_id"),
|
||||||
|
("blacklisted_studios", "studio_id"),
|
||||||
|
("blacklisted_tags", "tag_id"),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
class AdoptLegacyOut(BaseModel):
|
||||||
|
device_id: str
|
||||||
|
moved: dict[str, int]
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/adopt-legacy", response_model=AdoptLegacyOut)
|
||||||
|
def adopt_legacy(
|
||||||
|
session: Annotated[Session, Depends(get_session)],
|
||||||
|
device_id: Annotated[str, Depends(get_device_id)],
|
||||||
|
) -> AdoptLegacyOut:
|
||||||
|
moved: dict[str, int] = {}
|
||||||
|
if device_id == LEGACY_DEVICE:
|
||||||
|
# Wołający bez X-Device-Id == legacy → nie ma czego przepinać.
|
||||||
|
return AdoptLegacyOut(device_id=device_id, moved={})
|
||||||
|
for table, entity in _TABLES:
|
||||||
|
# Przepnij legacy → device dla encji których device JESZCZE nie ma.
|
||||||
|
res = session.execute(
|
||||||
|
text(
|
||||||
|
f"UPDATE {table} SET device_id = :dev "
|
||||||
|
f"WHERE device_id = :legacy AND {entity} NOT IN "
|
||||||
|
f"(SELECT {entity} FROM {table} WHERE device_id = :dev)"
|
||||||
|
).bindparams(dev=device_id, legacy=LEGACY_DEVICE)
|
||||||
|
)
|
||||||
|
moved[table] = res.rowcount or 0
|
||||||
|
# Resztki legacy (encje które device już miał) — skasuj, żeby nie wisiały.
|
||||||
|
session.execute(
|
||||||
|
text(f"DELETE FROM {table} WHERE device_id = :legacy").bindparams(legacy=LEGACY_DEVICE)
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
return AdoptLegacyOut(device_id=device_id, moved=moved)
|
||||||
|
|
@ -27,9 +27,11 @@ from app.models.movie import (
|
||||||
MoviePerformer,
|
MoviePerformer,
|
||||||
MovieTag,
|
MovieTag,
|
||||||
)
|
)
|
||||||
|
from app.api.device import LEGACY_DEVICE, get_device_id
|
||||||
from app.models.favorite_movie import FavoriteMovie
|
from app.models.favorite_movie import FavoriteMovie
|
||||||
from app.models.movie_playback_source import MoviePlaybackSource
|
from app.models.movie_playback_source import MoviePlaybackSource
|
||||||
from app.models.performer import Performer
|
from app.models.performer import Performer
|
||||||
|
from app.models.play_progress import MoviePlayProgress
|
||||||
from app.models.source import Source
|
from app.models.source import Source
|
||||||
from app.models.studio import Studio
|
from app.models.studio import Studio
|
||||||
from app.models.tag import Tag
|
from app.models.tag import Tag
|
||||||
|
|
@ -48,6 +50,7 @@ def _split_csv(raw: str | None) -> list[str]:
|
||||||
@router.get("", response_model=MovieListOut)
|
@router.get("", response_model=MovieListOut)
|
||||||
def list_movies(
|
def list_movies(
|
||||||
session: Annotated[Session, Depends(get_session)],
|
session: Annotated[Session, Depends(get_session)],
|
||||||
|
device_id: Annotated[str, Depends(get_device_id)],
|
||||||
q: str | None = Query(default=None, description="Title search (trgm)"),
|
q: str | None = Query(default=None, description="Title search (trgm)"),
|
||||||
studio_slugs: str | None = Query(default=None, description="Comma-separated studio slugs (OR)"),
|
studio_slugs: str | None = Query(default=None, description="Comma-separated studio slugs (OR)"),
|
||||||
tags: str | None = Query(default=None, description="Comma-separated tag slugs (AND)"),
|
tags: str | None = Query(default=None, description="Comma-separated tag slugs (AND)"),
|
||||||
|
|
@ -130,7 +133,7 @@ def list_movies(
|
||||||
base = base.limit(per_page).offset((page - 1) * per_page)
|
base = base.limit(per_page).offset((page - 1) * per_page)
|
||||||
|
|
||||||
movies = session.execute(base).scalars().all()
|
movies = session.execute(base).scalars().all()
|
||||||
items = [_movie_to_out(session, m) for m in movies]
|
items = [_movie_to_out(session, m, device_id=device_id) for m in movies]
|
||||||
|
|
||||||
return MovieListOut(items=items, total=total, page=page, per_page=per_page)
|
return MovieListOut(items=items, total=total, page=page, per_page=per_page)
|
||||||
|
|
||||||
|
|
@ -171,14 +174,15 @@ def _movie_origin_priority(origin: str) -> int:
|
||||||
def get_movie(
|
def get_movie(
|
||||||
movie_id: uuid.UUID,
|
movie_id: uuid.UUID,
|
||||||
session: Annotated[Session, Depends(get_session)],
|
session: Annotated[Session, Depends(get_session)],
|
||||||
|
device_id: Annotated[str, Depends(get_device_id)],
|
||||||
) -> MovieOut:
|
) -> MovieOut:
|
||||||
movie = session.get(Movie, movie_id)
|
movie = session.get(Movie, movie_id)
|
||||||
if movie is None:
|
if movie is None:
|
||||||
raise HTTPException(status_code=404, detail="movie not found")
|
raise HTTPException(status_code=404, detail="movie not found")
|
||||||
return _movie_to_out(session, movie)
|
return _movie_to_out(session, movie, device_id=device_id)
|
||||||
|
|
||||||
|
|
||||||
def _movie_to_out(session: Session, movie: Movie) -> MovieOut:
|
def _movie_to_out(session: Session, movie: Movie, *, device_id: str = LEGACY_DEVICE) -> MovieOut:
|
||||||
studio_out: StudioOut | None = None
|
studio_out: StudioOut | None = None
|
||||||
if movie.studio_id:
|
if movie.studio_id:
|
||||||
studio = session.get(Studio, movie.studio_id)
|
studio = session.get(Studio, movie.studio_id)
|
||||||
|
|
@ -247,9 +251,24 @@ def _movie_to_out(session: Session, movie: Movie) -> MovieOut:
|
||||||
if has_subhost:
|
if has_subhost:
|
||||||
pb_rows = [p for p in pb_rows if p.origin not in _MOVIE_LANDING_HIDE]
|
pb_rows = [p for p in pb_rows if p.origin not in _MOVIE_LANDING_HIDE]
|
||||||
pb_rows = sorted(pb_rows, key=lambda p: _movie_origin_priority(p.origin))
|
pb_rows = sorted(pb_rows, key=lambda p: _movie_origin_priority(p.origin))
|
||||||
|
# Dedup po realnym celu: 3 mirrory dooplay (mangoporn/pandamovies/streamporn)
|
||||||
|
# zapisują TEN SAM embed hostera jako osobne wiersze (np. luluvid/e/X z każdego
|
||||||
|
# mirrora) → film miał ~100 linków, z czego ~połowa to ten sam film (bug-report
|
||||||
|
# 41ca1fa4). Zostawiamy pierwszy = najwyższy priorytet (sort wyżej).
|
||||||
|
seen_targets: set[str] = set()
|
||||||
|
deduped: list[MoviePlaybackSource] = []
|
||||||
|
for p in pb_rows:
|
||||||
|
target = (p.embed_url or p.stream_url or p.page_url or "").strip()
|
||||||
|
if target and target in seen_targets:
|
||||||
|
continue
|
||||||
|
if target:
|
||||||
|
seen_targets.add(target)
|
||||||
|
deduped.append(p)
|
||||||
|
pb_rows = deduped
|
||||||
playback_sources = [PlaybackSourceOut.model_validate(p) for p in pb_rows]
|
playback_sources = [PlaybackSourceOut.model_validate(p) for p in pb_rows]
|
||||||
|
|
||||||
is_fav = session.get(FavoriteMovie, movie.id) is not None
|
is_fav = session.get(FavoriteMovie, (device_id, movie.id)) is not None
|
||||||
|
progress = session.get(MoviePlayProgress, (device_id, movie.id))
|
||||||
|
|
||||||
return MovieOut(
|
return MovieOut(
|
||||||
id=movie.id,
|
id=movie.id,
|
||||||
|
|
@ -272,4 +291,7 @@ def _movie_to_out(session: Session, movie: Movie) -> MovieOut:
|
||||||
playback_sources=playback_sources,
|
playback_sources=playback_sources,
|
||||||
created_at=movie.created_at,
|
created_at=movie.created_at,
|
||||||
is_favorite=is_fav,
|
is_favorite=is_fav,
|
||||||
|
last_played_at=progress.last_played_at if progress else None,
|
||||||
|
finished=progress.finished if progress else False,
|
||||||
|
position_sec=progress.position_sec if progress else 0,
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -148,7 +148,33 @@ def resolve_movie_playback(
|
||||||
links: list[StreamLink] = []
|
links: list[StreamLink] = []
|
||||||
|
|
||||||
if pb.origin == "paradisehill":
|
if pb.origin == "paradisehill":
|
||||||
# Tylko WebView fallback — paradisehill player wymaga session login dla streamu.
|
# Paradisehill: pobierz page, parsuj `var videoList = [...]` żeby dostać N parts.
|
||||||
|
# Każdy part to direct mp4 z paradisehill CDN (v1.paradisehill.cc), serwowane
|
||||||
|
# bez auth — 200 OK z plain User-Agent + Referer.
|
||||||
|
# Bug-reports `c5693926`/`418270e4`/`3c999b27` 2026-05-21 ("ładuje tylko 1 z N").
|
||||||
|
# Poprzednio: tylko WebView fallback → mobile gra 1. part w playerze paradisehilla,
|
||||||
|
# nie ma sposobu przejść do następnego.
|
||||||
|
try:
|
||||||
|
from app.connectors.paradisehill import fetch_and_extract_parts
|
||||||
|
parts = fetch_and_extract_parts(pb.page_url)
|
||||||
|
except Exception as e:
|
||||||
|
log.warning("paradisehill parts extract failed for %s: %s", pb.page_url, e)
|
||||||
|
parts = []
|
||||||
|
if parts:
|
||||||
|
for url, label in parts:
|
||||||
|
# NIE proxifikujemy tutaj — outer `_proxify_link` poniżej (linia 247) opakuje
|
||||||
|
# wszystkie linki. Double-wrap → token wewnątrz tokena (broken proxy URL).
|
||||||
|
links.append(
|
||||||
|
StreamLink(
|
||||||
|
stream_url=url,
|
||||||
|
embed_url=None,
|
||||||
|
quality=label,
|
||||||
|
type="mp4",
|
||||||
|
raw={"origin": pb.origin, "part_label": label},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
# Fallback: brak videoList (np. login-only movie) — WebView na całość.
|
||||||
links = [
|
links = [
|
||||||
StreamLink(
|
StreamLink(
|
||||||
stream_url=None,
|
stream_url=None,
|
||||||
|
|
@ -184,21 +210,33 @@ def resolve_movie_playback(
|
||||||
pb.id,
|
pb.id,
|
||||||
)
|
)
|
||||||
stream = None
|
stream = None
|
||||||
|
# Mixdrop mxcontent CDN wymaga curl_cffi JA3 → wymusza VPS proxy.
|
||||||
|
# Pre-2026-05-25 skipowaliśmy ten path "Bandwidth + anonimowość > UX",
|
||||||
|
# ale bug-report b1b5e1a2 zgłosił że Mixdrop WebView fallback = czarny
|
||||||
|
# ekran (recaptcha/adblock-detect blokują player init w in-app WebView).
|
||||||
|
# Movie ma zwykle 10+ alt hosterów, ale jeśli WebView fallback nie
|
||||||
|
# działa, user widzi tylko czarny ekran zamiast jakiejkolwiek alternatywy.
|
||||||
|
# Backend ma pełen pipeline: mixdrop.py extract → raw={proxy_impersonate:
|
||||||
|
# True, refetch_url} → stream_proxy._curl_cffi_stream z Chrome JA3 +
|
||||||
|
# auto-refetch on token expire. Włączamy go z powrotem.
|
||||||
|
# Bandwidth cost: ~485 MB/movie play; przy ~3 plays/day = 1.5 GB/day
|
||||||
|
# (acceptable na 8GB/m Hetzner plan z 20 TB transfer).
|
||||||
if stream:
|
if stream:
|
||||||
type_hint = "m3u8" if ".m3u8" in stream.lower() else "mp4"
|
type_hint = "m3u8" if ".m3u8" in stream.lower() else "mp4"
|
||||||
# Hostery których CDN wymaga Chrome JA3 (mxcontent dla mixdrop):
|
|
||||||
# proxy MUSI użyć curl_cffi impersonate inaczej 403. `proxy_impersonate=True`
|
|
||||||
# idzie przez `raw` → `_proxify_link` ustawi token `i=1`.
|
|
||||||
cdn_needs_impersonate = "mxcontent.net" in stream.lower()
|
|
||||||
raw_meta: dict = {"origin": pb.origin, "host": target}
|
raw_meta: dict = {"origin": pb.origin, "host": target}
|
||||||
if cdn_needs_impersonate:
|
# seekplayer-engine (#hash family: easyvidplayer/player4me/seekplayer/
|
||||||
raw_meta["proxy_impersonate"] = True
|
# embedseek/upns — ~322k sources) zwraca master.m3u8 na raw-IP CDN
|
||||||
# Mixdrop: same-session cookies + chrome JA3 wymagane dla mp4.
|
# (185.237.x/203.188.x/45.156.x `/v4/<tok>/<exp>/pp/<hash>/master.m3u8`).
|
||||||
# Backend extract zamknął sesję — proxy musi re-fetchować
|
# Zweryfikowane cross-IP (curl_cffi chrome + Bright Data, 2026-06-06):
|
||||||
# embed page w fresh curl_cffi session żeby re-extract mp4
|
# manifest + variant + fMP4 segment WSZYSTKIE 200 z innego IP, a cert jest
|
||||||
# z aktualnymi cookies.
|
# VALID (verify=True OK — IP-SAN, nie self-signed jak głosił stary docstring).
|
||||||
raw_meta["refetch_url"] = target
|
# Token jest TIME-bound (`<exp>` unix ts), NIE IP-bound. → mobile ExoPlayer
|
||||||
raw_meta["refetch_hoster"] = "mixdrop"
|
# gra direct z CDN, zero VPS bandwidth (był to największy movie proxy-sink).
|
||||||
|
# Proxy (`stream_url`) zostaje jako fallback (stream_proxy IP-host gałąź
|
||||||
|
# robi verify=False). Device-verified na emulatorze przed deployem.
|
||||||
|
from app.extractors.hosters import seekplayer_engine
|
||||||
|
if seekplayer_engine.matches(target):
|
||||||
|
raw_meta["mobile_direct_ok"] = True
|
||||||
links.append(
|
links.append(
|
||||||
StreamLink(
|
StreamLink(
|
||||||
stream_url=stream,
|
stream_url=stream,
|
||||||
|
|
@ -224,6 +262,13 @@ def resolve_movie_playback(
|
||||||
raise HTTPException(status_code=502, detail="no playable links")
|
raise HTTPException(status_code=502, detail="no playable links")
|
||||||
|
|
||||||
links = [_proxify_link(link, referer) for link in links]
|
links = [_proxify_link(link, referer) for link in links]
|
||||||
|
# Dla paradisehill multipart: `_pick_best` wybiera "Part N" z najwyższą cyfrą (parsuje
|
||||||
|
# quality jako int), ale user chce zacząć od Part 1. Override: zawsze links[0].
|
||||||
|
if pb.origin == "paradisehill" and len(links) > 1 and any(
|
||||||
|
(link.raw or {}).get("part_label") for link in links
|
||||||
|
):
|
||||||
|
best = links[0]
|
||||||
|
else:
|
||||||
best = _pick_best(links) if links else None
|
best = _pick_best(links) if links else None
|
||||||
return ResolveOut(
|
return ResolveOut(
|
||||||
source=PlaybackSourceOut.model_validate(pb),
|
source=PlaybackSourceOut.model_validate(pb),
|
||||||
|
|
@ -462,8 +507,19 @@ def _proxify_link(link: StreamLink, referer: str) -> StreamLink:
|
||||||
# mobile_direct_ok overrides m3u8 default-to-proxy: gdy CDN ma time-bound token
|
# mobile_direct_ok overrides m3u8 default-to-proxy: gdy CDN ma time-bound token
|
||||||
# (nie IP-bound), mobile ExoPlayer może pobrać manifest direct bez VPS proxy.
|
# (nie IP-bound), mobile ExoPlayer może pobrać manifest direct bez VPS proxy.
|
||||||
is_manifest_type = type_lower in {"m3u8", "hls", "mpd"}
|
is_manifest_type = type_lower in {"m3u8", "hls", "mpd"}
|
||||||
|
is_hls = type_lower in {"m3u8", "hls"}
|
||||||
|
# ExoPlayer (expo-video 2.0.6, brak contentType) zgaduje typ z URL: ścieżka kończąca
|
||||||
|
# się `.m3u8` → HLS (gra direct, 0 VPS), inaczej → progressive Mp4Extractor → fail
|
||||||
|
# → fallback na pełny proxy → CAŁE wideo przez VPS (pornhat `...mp4,?...`: 466 hitów
|
||||||
|
# /48h, audit 2026-06-11). Dla takich time-bound HLS dajemy manifest-passthrough pod
|
||||||
|
# `play.m3u8`: segmenty zostają absolutne na CDN → telefon ciągnie je direct, przez
|
||||||
|
# VPS leci tylko ~1KB manifestu. Manifesty z poprawnym `.m3u8` zostają w pełni direct.
|
||||||
|
raw_path = raw_url.split("?", 1)[0].lower()
|
||||||
|
hls_needs_passthrough = is_hls and mobile_direct_ok and not raw_path.endswith(".m3u8")
|
||||||
if use_impersonate or force_proxy or (is_manifest_type and not mobile_direct_ok):
|
if use_impersonate or force_proxy or (is_manifest_type and not mobile_direct_ok):
|
||||||
direct_for_mobile = proxied
|
direct_for_mobile = proxied
|
||||||
|
elif hls_needs_passthrough:
|
||||||
|
direct_for_mobile = f"/proxy/hls/{token}/play.m3u8"
|
||||||
else:
|
else:
|
||||||
direct_for_mobile = raw_url
|
direct_for_mobile = raw_url
|
||||||
return StreamLink(
|
return StreamLink(
|
||||||
|
|
|
||||||
53
app/api/playback_events.py
Normal file
53
app/api/playback_events.py
Normal file
|
|
@ -0,0 +1,53 @@
|
||||||
|
"""Playback telemetry — POST /playback-events (fire-and-forget z apki).
|
||||||
|
|
||||||
|
Apka po każdej próbie odtwarzania woła to z origin sceny + wynikiem (success/error).
|
||||||
|
Sygnał zasila health/szybkość w rankingu źródeł (run_source_stats agreguje okno 7d).
|
||||||
|
Best-effort: apka NIE czeka i ignoruje błąd — telemetria nie może psuć playbacku.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import uuid
|
||||||
|
from typing import Annotated, Literal
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Depends
|
||||||
|
from pydantic import BaseModel, Field
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app.api.device import get_device_id
|
||||||
|
from app.auth import require_api_key
|
||||||
|
from app.db import get_session
|
||||||
|
from app.models.playback_event import PlaybackEvent
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/playback-events", tags=["telemetry"], dependencies=[Depends(require_api_key)])
|
||||||
|
|
||||||
|
|
||||||
|
class PlaybackEventIn(BaseModel):
|
||||||
|
# 'tube:<sitetag>' — zgodne z playback_sources.origin. Bez tego ping bezużyteczny.
|
||||||
|
origin: str = Field(min_length=1, max_length=64)
|
||||||
|
status: Literal["success", "error"]
|
||||||
|
scene_id: uuid.UUID | None = None
|
||||||
|
error_kind: str | None = Field(default=None, max_length=64)
|
||||||
|
# Surowa treść błędu z native playera (ExoPlayer message) — diagnostyka realnych
|
||||||
|
# padów których nie da się odtworzyć na emulatorze (np. watchporn player_error).
|
||||||
|
error_detail: str | None = Field(default=None, max_length=512)
|
||||||
|
ttff_ms: int | None = Field(default=None, ge=0, le=600_000)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("", status_code=204)
|
||||||
|
def post_playback_event(
|
||||||
|
body: PlaybackEventIn,
|
||||||
|
session: Annotated[Session, Depends(get_session)],
|
||||||
|
device_id: Annotated[str, Depends(get_device_id)],
|
||||||
|
) -> None:
|
||||||
|
session.add(
|
||||||
|
PlaybackEvent(
|
||||||
|
origin=body.origin,
|
||||||
|
scene_id=body.scene_id,
|
||||||
|
status=body.status,
|
||||||
|
error_kind=body.error_kind,
|
||||||
|
error_detail=body.error_detail if body.status == "error" else None,
|
||||||
|
ttff_ms=body.ttff_ms if body.status == "success" else None,
|
||||||
|
device_id=device_id,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
108
app/api/saved_searches.py
Normal file
108
app/api/saved_searches.py
Normal file
|
|
@ -0,0 +1,108 @@
|
||||||
|
"""Saved searches — zapisane słowa kluczowe per urządzenie.
|
||||||
|
|
||||||
|
User-report mobilism: sceny słabo opisane → user chce zapisywać często używane
|
||||||
|
zapytania i szybko je odpalać. Scope po device_id (X-Device-Id), jak reszta stanu.
|
||||||
|
|
||||||
|
Endpointy:
|
||||||
|
GET /saved-searches — lista zapisów tego urządzenia (najnowsze pierwsze)
|
||||||
|
POST /saved-searches — dodaj {query} (idempotent na device+query)
|
||||||
|
DELETE /saved-searches/{id} — usuń
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import uuid
|
||||||
|
from datetime import datetime
|
||||||
|
from typing import Annotated
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Depends, HTTPException, status
|
||||||
|
from pydantic import BaseModel, field_validator
|
||||||
|
from sqlalchemy import select
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app.api.device import get_device_id
|
||||||
|
from app.auth import require_api_key
|
||||||
|
from app.db import get_session
|
||||||
|
from app.models.saved_search import SavedSearch
|
||||||
|
|
||||||
|
router = APIRouter(
|
||||||
|
prefix="/saved-searches",
|
||||||
|
tags=["saved-searches"],
|
||||||
|
dependencies=[Depends(require_api_key)],
|
||||||
|
)
|
||||||
|
|
||||||
|
# Limit zapisów per urządzenie — to wygodny skrót, nie nieograniczona kolekcja.
|
||||||
|
MAX_PER_DEVICE = 50
|
||||||
|
|
||||||
|
|
||||||
|
class SavedSearchOut(BaseModel):
|
||||||
|
id: uuid.UUID
|
||||||
|
query: str
|
||||||
|
created_at: datetime
|
||||||
|
|
||||||
|
|
||||||
|
class SavedSearchIn(BaseModel):
|
||||||
|
query: str
|
||||||
|
|
||||||
|
@field_validator("query")
|
||||||
|
@classmethod
|
||||||
|
def _clean(cls, v: str) -> str:
|
||||||
|
v = (v or "").strip()
|
||||||
|
if not v:
|
||||||
|
raise ValueError("query empty")
|
||||||
|
return v[:256]
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("", response_model=list[SavedSearchOut])
|
||||||
|
def list_saved_searches(
|
||||||
|
session: Annotated[Session, Depends(get_session)],
|
||||||
|
device_id: Annotated[str, Depends(get_device_id)],
|
||||||
|
) -> list[SavedSearch]:
|
||||||
|
rows = session.execute(
|
||||||
|
select(SavedSearch)
|
||||||
|
.where(SavedSearch.device_id == device_id)
|
||||||
|
.order_by(SavedSearch.created_at.desc())
|
||||||
|
).scalars().all()
|
||||||
|
return list(rows)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("", response_model=SavedSearchOut, status_code=status.HTTP_200_OK)
|
||||||
|
def add_saved_search(
|
||||||
|
payload: SavedSearchIn,
|
||||||
|
session: Annotated[Session, Depends(get_session)],
|
||||||
|
device_id: Annotated[str, Depends(get_device_id)],
|
||||||
|
) -> SavedSearch:
|
||||||
|
# Idempotent — ten sam (device, query) zwraca istniejący wiersz.
|
||||||
|
existing = session.execute(
|
||||||
|
select(SavedSearch).where(
|
||||||
|
SavedSearch.device_id == device_id, SavedSearch.query == payload.query
|
||||||
|
)
|
||||||
|
).scalar_one_or_none()
|
||||||
|
if existing is not None:
|
||||||
|
return existing
|
||||||
|
count = session.execute(
|
||||||
|
select(SavedSearch.id).where(SavedSearch.device_id == device_id)
|
||||||
|
).scalars().all()
|
||||||
|
if len(count) >= MAX_PER_DEVICE:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_409_CONFLICT,
|
||||||
|
detail=f"saved-search limit ({MAX_PER_DEVICE}) reached",
|
||||||
|
)
|
||||||
|
row = SavedSearch(device_id=device_id, query=payload.query)
|
||||||
|
session.add(row)
|
||||||
|
session.commit()
|
||||||
|
session.refresh(row)
|
||||||
|
return row
|
||||||
|
|
||||||
|
|
||||||
|
@router.delete("/{search_id}", status_code=status.HTTP_204_NO_CONTENT)
|
||||||
|
def remove_saved_search(
|
||||||
|
search_id: uuid.UUID,
|
||||||
|
session: Annotated[Session, Depends(get_session)],
|
||||||
|
device_id: Annotated[str, Depends(get_device_id)],
|
||||||
|
) -> None:
|
||||||
|
row = session.get(SavedSearch, search_id)
|
||||||
|
# Tylko własne urządzenie może usuwać; cudze/nieistniejące → idempotentny no-op.
|
||||||
|
if row is None or row.device_id != device_id:
|
||||||
|
return
|
||||||
|
session.delete(row)
|
||||||
|
session.commit()
|
||||||
|
|
@ -15,6 +15,7 @@ from pydantic import BaseModel
|
||||||
from sqlalchemy import select
|
from sqlalchemy import select
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app.api.device import get_device_id
|
||||||
from app.api.scenes import _build_scene_out
|
from app.api.scenes import _build_scene_out
|
||||||
from app.api.schemas import SceneOut
|
from app.api.schemas import SceneOut
|
||||||
from app.auth import require_api_key
|
from app.auth import require_api_key
|
||||||
|
|
@ -42,16 +43,18 @@ class SceneFavoriteToggleOut(BaseModel):
|
||||||
@router.get("", response_model=SceneFavoriteListOut)
|
@router.get("", response_model=SceneFavoriteListOut)
|
||||||
def list_scene_favorites(
|
def list_scene_favorites(
|
||||||
session: Annotated[Session, Depends(get_session)],
|
session: Annotated[Session, Depends(get_session)],
|
||||||
|
device_id: Annotated[str, Depends(get_device_id)],
|
||||||
) -> SceneFavoriteListOut:
|
) -> SceneFavoriteListOut:
|
||||||
rows = (
|
rows = (
|
||||||
session.execute(
|
session.execute(
|
||||||
select(Scene, FavoriteScene)
|
select(Scene, FavoriteScene)
|
||||||
.join(FavoriteScene, FavoriteScene.scene_id == Scene.id)
|
.join(FavoriteScene, FavoriteScene.scene_id == Scene.id)
|
||||||
|
.where(FavoriteScene.device_id == device_id)
|
||||||
.order_by(FavoriteScene.created_at.desc())
|
.order_by(FavoriteScene.created_at.desc())
|
||||||
)
|
)
|
||||||
.all()
|
.all()
|
||||||
)
|
)
|
||||||
items = [_build_scene_out(session, scene) for scene, _ in rows]
|
items = [_build_scene_out(session, scene, device_id=device_id) for scene, _ in rows]
|
||||||
return SceneFavoriteListOut(items=items, total=len(items))
|
return SceneFavoriteListOut(items=items, total=len(items))
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -63,13 +66,14 @@ def list_scene_favorites(
|
||||||
def add_scene_favorite(
|
def add_scene_favorite(
|
||||||
scene_id: uuid.UUID,
|
scene_id: uuid.UUID,
|
||||||
session: Annotated[Session, Depends(get_session)],
|
session: Annotated[Session, Depends(get_session)],
|
||||||
|
device_id: Annotated[str, Depends(get_device_id)],
|
||||||
) -> SceneFavoriteToggleOut:
|
) -> SceneFavoriteToggleOut:
|
||||||
scene = session.get(Scene, scene_id)
|
scene = session.get(Scene, scene_id)
|
||||||
if scene is None:
|
if scene is None:
|
||||||
raise HTTPException(status_code=404, detail="scene not found")
|
raise HTTPException(status_code=404, detail="scene not found")
|
||||||
existing = session.get(FavoriteScene, scene_id)
|
existing = session.get(FavoriteScene, (device_id, scene_id))
|
||||||
if existing is None:
|
if existing is None:
|
||||||
session.add(FavoriteScene(scene_id=scene_id))
|
session.add(FavoriteScene(device_id=device_id, scene_id=scene_id))
|
||||||
return SceneFavoriteToggleOut(scene_id=scene_id, favorited=True)
|
return SceneFavoriteToggleOut(scene_id=scene_id, favorited=True)
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -77,7 +81,8 @@ def add_scene_favorite(
|
||||||
def remove_scene_favorite(
|
def remove_scene_favorite(
|
||||||
scene_id: uuid.UUID,
|
scene_id: uuid.UUID,
|
||||||
session: Annotated[Session, Depends(get_session)],
|
session: Annotated[Session, Depends(get_session)],
|
||||||
|
device_id: Annotated[str, Depends(get_device_id)],
|
||||||
) -> None:
|
) -> None:
|
||||||
fav = session.get(FavoriteScene, scene_id)
|
fav = session.get(FavoriteScene, (device_id, scene_id))
|
||||||
if fav is not None:
|
if fav is not None:
|
||||||
session.delete(fav)
|
session.delete(fav)
|
||||||
|
|
|
||||||
|
|
@ -8,7 +8,7 @@ from typing import Annotated
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends, HTTPException, Query, status
|
from fastapi import APIRouter, Depends, HTTPException, Query, status
|
||||||
from pydantic import BaseModel
|
from pydantic import BaseModel
|
||||||
from sqlalchemy import distinct, exists, func, select
|
from sqlalchemy import distinct, exists, false, func, literal_column, select
|
||||||
from sqlalchemy.exc import IntegrityError
|
from sqlalchemy.exc import IntegrityError
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
|
@ -24,6 +24,7 @@ from app.api.schemas import (
|
||||||
TagOut,
|
TagOut,
|
||||||
)
|
)
|
||||||
from app.db import get_session
|
from app.db import get_session
|
||||||
|
from app.api.device import LEGACY_DEVICE, get_device_id
|
||||||
from app.models.favorite_scene import FavoriteScene
|
from app.models.favorite_scene import FavoriteScene
|
||||||
from app.models.performer import Performer
|
from app.models.performer import Performer
|
||||||
from app.models.play_progress import ScenePlayProgress
|
from app.models.play_progress import ScenePlayProgress
|
||||||
|
|
@ -40,6 +41,85 @@ router = APIRouter(prefix="/scenes", tags=["scenes"], dependencies=[Depends(requ
|
||||||
|
|
||||||
_VALID_SORTS = {"created_at", "release_date", "title", "studio"}
|
_VALID_SORTS = {"created_at", "release_date", "title", "studio"}
|
||||||
|
|
||||||
|
# TTL-cache dla count'u scen-z-żywym-playback (default lista bez filtra). Full-scan
|
||||||
|
# 1.69M scen + EXISTS ~950ms; liczba zmienia się wolno i jest przybliżona (header
|
||||||
|
# paginacji), więc 10-min cache w pamięci procesu API jest akceptowalny trade-off.
|
||||||
|
_DEFAULT_COUNT_CACHE: dict = {"ts": 0.0, "val": 0}
|
||||||
|
_DEFAULT_COUNT_TTL = 600.0
|
||||||
|
|
||||||
|
def _default_scene_count(session: Session) -> int:
|
||||||
|
import time as _time
|
||||||
|
now = _time.monotonic()
|
||||||
|
if _DEFAULT_COUNT_CACHE["val"] and (now - _DEFAULT_COUNT_CACHE["ts"]) < _DEFAULT_COUNT_TTL:
|
||||||
|
return _DEFAULT_COUNT_CACHE["val"]
|
||||||
|
count_query = select(func.count()).select_from(
|
||||||
|
select(Scene.id).where(
|
||||||
|
exists(
|
||||||
|
select(1).where(
|
||||||
|
PlaybackSource.scene_id == Scene.id,
|
||||||
|
PlaybackSource.dead_at.is_(None),
|
||||||
|
)
|
||||||
|
),
|
||||||
|
# Domyślny feed wyklucza JAV (osobna sekcja) → licznik też.
|
||||||
|
~_jav_source_exists(),
|
||||||
|
).subquery()
|
||||||
|
)
|
||||||
|
total = session.execute(count_query).scalar_one()
|
||||||
|
_DEFAULT_COUNT_CACHE["ts"] = now
|
||||||
|
_DEFAULT_COUNT_CACHE["val"] = total
|
||||||
|
return total
|
||||||
|
|
||||||
|
|
||||||
|
# JAV vertical — osobna sekcja. Sceny z tych originów NIE wchodzą do głównego feedu
|
||||||
|
# (domyślnie wykluczone), tylko do zakładki JAV (?jav=true). JAV to osobny namespace
|
||||||
|
# (kody typu BKD-368, azjatyckie tytuły), nie deduplikuje się z zachodnim katalogiem,
|
||||||
|
# więc scena JAV ma WYŁĄCZNIE origin JAV → wykluczenie/inkluzja po tym originie jest pewna.
|
||||||
|
JAV_ORIGINS = ("tube:javflix", "tube:javguru", "tube:vjav", "tube:supjav")
|
||||||
|
|
||||||
|
|
||||||
|
def _jav_source_exists():
|
||||||
|
return exists(
|
||||||
|
select(1).where(
|
||||||
|
PlaybackSource.scene_id == Scene.id,
|
||||||
|
PlaybackSource.dead_at.is_(None),
|
||||||
|
PlaybackSource.origin.in_(JAV_ORIGINS),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# Blacklisty (performer/studio/tag) są zwykle PUSTE (self-hosted, single-user). Mimo to
|
||||||
|
# 3 NOT EXISTS klauzule doklejały się do KAŻDEJ filtrowanej listy scen i były ewaluowane
|
||||||
|
# per-row — przy filtrze typu duży-tag/has_playback planer chodzi po ~176k scen, więc te
|
||||||
|
# puste-zawsze klauzule kosztowały ~3.4s (mega-tag „anal": 6.7s→3.3s po pominięciu).
|
||||||
|
# Cache'ujemy emptiness (TTL 5 min); gdy ktoś doda blacklist-wpis, w ciągu 5 min klauzule
|
||||||
|
# wracają. Patrz reference_scenes_list_perf / task #22.
|
||||||
|
# Cache per device_id (blacklisty są teraz device-scoped — bug 2026-06-08).
|
||||||
|
_BLACKLIST_EMPTY_CACHE: dict[str, tuple[float, bool]] = {}
|
||||||
|
_BLACKLIST_EMPTY_TTL = 300.0
|
||||||
|
|
||||||
|
|
||||||
|
def _blacklists_empty(session: Session, device_id: str) -> bool:
|
||||||
|
"""True gdy WSZYSTKIE 3 blacklisty TEGO device puste → pomiń NOT EXISTS klauzule."""
|
||||||
|
import time as _time
|
||||||
|
from app.models.blacklist import (
|
||||||
|
BlacklistedPerformer,
|
||||||
|
BlacklistedStudio,
|
||||||
|
BlacklistedTag,
|
||||||
|
)
|
||||||
|
now = _time.monotonic()
|
||||||
|
cached = _BLACKLIST_EMPTY_CACHE.get(device_id)
|
||||||
|
if cached and (now - cached[0]) < _BLACKLIST_EMPTY_TTL:
|
||||||
|
return cached[1]
|
||||||
|
has_any = session.execute(
|
||||||
|
select(
|
||||||
|
exists(select(1).select_from(BlacklistedPerformer).where(BlacklistedPerformer.device_id == device_id))
|
||||||
|
| exists(select(1).select_from(BlacklistedStudio).where(BlacklistedStudio.device_id == device_id))
|
||||||
|
| exists(select(1).select_from(BlacklistedTag).where(BlacklistedTag.device_id == device_id))
|
||||||
|
)
|
||||||
|
).scalar_one()
|
||||||
|
_BLACKLIST_EMPTY_CACHE[device_id] = (now, not has_any)
|
||||||
|
return not has_any
|
||||||
|
|
||||||
|
|
||||||
def _split_csv(raw: str | None) -> list[str]:
|
def _split_csv(raw: str | None) -> list[str]:
|
||||||
if not raw:
|
if not raw:
|
||||||
|
|
@ -47,9 +127,77 @@ def _split_csv(raw: str | None) -> list[str]:
|
||||||
return [s.strip() for s in raw.split(",") if s.strip()]
|
return [s.strip() for s in raw.split(",") if s.strip()]
|
||||||
|
|
||||||
|
|
||||||
|
# ---- Współdzielone klauzule widoczności sceny ----------------------------------
|
||||||
|
# Definicja "co user widzi na liście" żyje TU i jest reużywana przez list_scenes ORAZ
|
||||||
|
# licznik "+N nowych" w app/api/favorites.py. Wcześniej favorites miał ręczną kopię tych
|
||||||
|
# klauzul (dryf: zmiana filtra listy nie trafiała do licznika → "+6 a nic nowego").
|
||||||
|
|
||||||
|
|
||||||
|
def live_playback_exists():
|
||||||
|
"""EXISTS: scena ma choć jeden żywy playback_source."""
|
||||||
|
return exists(
|
||||||
|
select(1).where(
|
||||||
|
PlaybackSource.scene_id == Scene.id,
|
||||||
|
PlaybackSource.dead_at.is_(None),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def blacklist_clauses(session: Session, device_id: str) -> list:
|
||||||
|
"""NOT-EXISTS klauzule blacklist device (performer/studio/tag). [] gdy wszystkie puste."""
|
||||||
|
if _blacklists_empty(session, device_id):
|
||||||
|
return []
|
||||||
|
from app.models.blacklist import (
|
||||||
|
BlacklistedPerformer,
|
||||||
|
BlacklistedStudio,
|
||||||
|
BlacklistedTag,
|
||||||
|
)
|
||||||
|
|
||||||
|
return [
|
||||||
|
~exists(
|
||||||
|
select(1)
|
||||||
|
.select_from(ScenePerformer)
|
||||||
|
.join(
|
||||||
|
BlacklistedPerformer,
|
||||||
|
(BlacklistedPerformer.performer_id == ScenePerformer.performer_id)
|
||||||
|
& (BlacklistedPerformer.device_id == device_id),
|
||||||
|
)
|
||||||
|
.where(ScenePerformer.scene_id == Scene.id)
|
||||||
|
),
|
||||||
|
~Scene.studio_id.in_(
|
||||||
|
select(BlacklistedStudio.studio_id).where(BlacklistedStudio.device_id == device_id)
|
||||||
|
),
|
||||||
|
~exists(
|
||||||
|
select(1)
|
||||||
|
.select_from(SceneTag)
|
||||||
|
.join(
|
||||||
|
BlacklistedTag,
|
||||||
|
(BlacklistedTag.tag_id == SceneTag.tag_id)
|
||||||
|
& (BlacklistedTag.device_id == device_id),
|
||||||
|
)
|
||||||
|
.where(SceneTag.scene_id == Scene.id)
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def stub_exclusion_clause():
|
||||||
|
"""Odsiew stub-scen: tube-only bez release_date AND bez canonical (TPDB/StashDB) AND
|
||||||
|
bez performera. NOT stub gdy ma release_date OR canonical OR performera."""
|
||||||
|
canonical_exists = exists(
|
||||||
|
select(1)
|
||||||
|
.select_from(SceneExternalRef)
|
||||||
|
.join(Source, Source.id == SceneExternalRef.source_id)
|
||||||
|
.where(SceneExternalRef.scene_id == Scene.id)
|
||||||
|
.where(Source.kind.in_([SourceKind.tpdb, SourceKind.stashdb]))
|
||||||
|
)
|
||||||
|
has_performer = exists(select(1).where(ScenePerformer.scene_id == Scene.id))
|
||||||
|
return Scene.release_date.is_not(None) | canonical_exists | has_performer
|
||||||
|
|
||||||
|
|
||||||
@router.get("", response_model=SceneListOut)
|
@router.get("", response_model=SceneListOut)
|
||||||
def list_scenes(
|
def list_scenes(
|
||||||
session: Annotated[Session, Depends(get_session)],
|
session: Annotated[Session, Depends(get_session)],
|
||||||
|
device_id: Annotated[str, Depends(get_device_id)],
|
||||||
q: str | None = Query(default=None, description="Wyszukiwanie po title_normalized (trgm)"),
|
q: str | None = Query(default=None, description="Wyszukiwanie po title_normalized (trgm)"),
|
||||||
studio_slug: str | None = Query(default=None, description="DEPRECATED — użyj studio_slugs"),
|
studio_slug: str | None = Query(default=None, description="DEPRECATED — użyj studio_slugs"),
|
||||||
studio_slugs: str | None = Query(
|
studio_slugs: str | None = Query(
|
||||||
|
|
@ -66,10 +214,6 @@ def list_scenes(
|
||||||
has_playback: bool | None = Query(
|
has_playback: bool | None = Query(
|
||||||
default=None, description="True: tylko sceny z ≥1 playback_source"
|
default=None, description="True: tylko sceny z ≥1 playback_source"
|
||||||
),
|
),
|
||||||
has_animated_thumbnail: bool | None = Query(
|
|
||||||
default=None,
|
|
||||||
description="True: tylko sceny z ≥1 playback_source z animated_thumbnail_url (hold-to-preview)",
|
|
||||||
),
|
|
||||||
min_duration_sec: int | None = Query(default=None, ge=0),
|
min_duration_sec: int | None = Query(default=None, ge=0),
|
||||||
max_duration_sec: int | None = Query(default=None, ge=0),
|
max_duration_sec: int | None = Query(default=None, ge=0),
|
||||||
released_within_days: int | None = Query(
|
released_within_days: int | None = Query(
|
||||||
|
|
@ -83,6 +227,13 @@ def list_scenes(
|
||||||
"po PlaybackSource.quality (string typu '720p' / '1080p Full HD')."
|
"po PlaybackSource.quality (string typu '720p' / '1080p Full HD')."
|
||||||
),
|
),
|
||||||
),
|
),
|
||||||
|
origin: str | None = Query(
|
||||||
|
default=None,
|
||||||
|
description=(
|
||||||
|
"Filtruj po playback origin (np. 'tube:hqpornercom'). Substring match — "
|
||||||
|
"'hqporner' złapie tube:hqpornercom. Diagnostyka per-hoster."
|
||||||
|
),
|
||||||
|
),
|
||||||
include_stubs: bool = Query(
|
include_stubs: bool = Query(
|
||||||
default=False,
|
default=False,
|
||||||
description=(
|
description=(
|
||||||
|
|
@ -90,6 +241,13 @@ def list_scenes(
|
||||||
"z jedynym playback z hqporner (~7-min Brazzers trailer clipy zalewają katalog)."
|
"z jedynym playback z hqporner (~7-min Brazzers trailer clipy zalewają katalog)."
|
||||||
),
|
),
|
||||||
),
|
),
|
||||||
|
jav: bool = Query(
|
||||||
|
default=False,
|
||||||
|
description=(
|
||||||
|
"False (default): ukrywa sceny JAV (osobny vertical). True: TYLKO sceny JAV. "
|
||||||
|
"JAV to osobna sekcja w apce, nie zalewa głównego feedu (origin javflix/javguru/vjav/supjav)."
|
||||||
|
),
|
||||||
|
),
|
||||||
sort: str = Query(default="created_at", description="created_at|release_date|title|studio"),
|
sort: str = Query(default="created_at", description="created_at|release_date|title|studio"),
|
||||||
page: int = Query(default=1, ge=1),
|
page: int = Query(default=1, ge=1),
|
||||||
per_page: int = Query(default=50, ge=1, le=200),
|
per_page: int = Query(default=50, ge=1, le=200),
|
||||||
|
|
@ -102,9 +260,18 @@ def list_scenes(
|
||||||
if q:
|
if q:
|
||||||
base = base.where(Scene.title_normalized.ilike(f"%{q.lower()}%"))
|
base = base.where(Scene.title_normalized.ilike(f"%{q.lower()}%"))
|
||||||
|
|
||||||
|
# Cap rozmiarów filtrów. Bez tego pojedynczy request z setkami studio_slugs +
|
||||||
|
# dziesiątkami tagów (każdy tag = osobny correlated EXISTS) + ILIKE budował zapytanie,
|
||||||
|
# które OOM-killer ubijał → PG crash-recovery = ~1s globalnej przerwy (GOON-1M,
|
||||||
|
# 2026-06-26: 194 studios + 23 tagi). Realny UI nigdy nie wysyła tylu. 422 zamiast
|
||||||
|
# wywalania bazy. Limity hojne (>> normalne użycie), ale ograniczają złożoność query.
|
||||||
|
_MAX_STUDIOS, _MAX_TAGS, _MAX_PERFORMERS = 50, 15, 15
|
||||||
|
|
||||||
studio_slug_list = _split_csv(studio_slugs)
|
studio_slug_list = _split_csv(studio_slugs)
|
||||||
if studio_slug:
|
if studio_slug:
|
||||||
studio_slug_list.append(studio_slug)
|
studio_slug_list.append(studio_slug)
|
||||||
|
if len(studio_slug_list) > _MAX_STUDIOS:
|
||||||
|
raise HTTPException(status_code=422, detail=f"too many studio filters (max {_MAX_STUDIOS})")
|
||||||
if studio_slug_list:
|
if studio_slug_list:
|
||||||
base = base.where(
|
base = base.where(
|
||||||
Scene.studio_id.in_(
|
Scene.studio_id.in_(
|
||||||
|
|
@ -113,19 +280,39 @@ def list_scenes(
|
||||||
)
|
)
|
||||||
|
|
||||||
tag_slug_list = _split_csv(tags)
|
tag_slug_list = _split_csv(tags)
|
||||||
|
if len(tag_slug_list) > _MAX_TAGS:
|
||||||
|
raise HTTPException(status_code=422, detail=f"too many tag filters (max {_MAX_TAGS})")
|
||||||
# AND między tagami: scena musi mieć WSZYSTKIE zaznaczone tagi. Każdy slug → osobny
|
# AND między tagami: scena musi mieć WSZYSTKIE zaznaczone tagi. Każdy slug → osobny
|
||||||
# exists() — zaznaczanie kolejnych filtrów zawęża wyniki, jak intuicja użytkownika.
|
# exists() — zaznaczanie kolejnych filtrów zawęża wyniki, jak intuicja użytkownika.
|
||||||
|
#
|
||||||
|
# PERF (2026-06-07): resolvujemy slug→tag_id w aplikacji i filtrujemy po LITERALNYM
|
||||||
|
# tag_id (NIE JOIN po Tag.slug). Z literałem planner zna kardynalność tagu ze
|
||||||
|
# statystyk (MCV) → dla popularnych tagów (blowjob ~273k scen) wybiera index-walk po
|
||||||
|
# ix_scenes_created_at_desc zamiast materializować wszystkie scene_tags. Slug-JOIN
|
||||||
|
# ukrywał tag_id przed plannerem → używał średniej (8.4M/11541≈726) → zły plan
|
||||||
|
# (4-12s). Z literałem: ~20ms. Zob. też _build... light mode.
|
||||||
|
if tag_slug_list:
|
||||||
|
id_by_slug = dict(
|
||||||
|
session.execute(
|
||||||
|
select(Tag.slug, Tag.id).where(Tag.slug.in_(tag_slug_list))
|
||||||
|
).all()
|
||||||
|
)
|
||||||
for slug in tag_slug_list:
|
for slug in tag_slug_list:
|
||||||
|
tag_id = id_by_slug.get(slug)
|
||||||
|
if tag_id is None:
|
||||||
|
base = base.where(false()) # nieznany slug → brak wyników
|
||||||
|
break
|
||||||
base = base.where(
|
base = base.where(
|
||||||
exists(
|
exists(
|
||||||
select(1)
|
select(1)
|
||||||
.select_from(SceneTag)
|
.select_from(SceneTag)
|
||||||
.join(Tag, Tag.id == SceneTag.tag_id)
|
.where(SceneTag.scene_id == Scene.id, SceneTag.tag_id == tag_id)
|
||||||
.where(SceneTag.scene_id == Scene.id, Tag.slug == slug)
|
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
perf_id_strings = _split_csv(performer_ids)
|
perf_id_strings = _split_csv(performer_ids)
|
||||||
|
if len(perf_id_strings) > _MAX_PERFORMERS:
|
||||||
|
raise HTTPException(status_code=422, detail=f"too many performer filters (max {_MAX_PERFORMERS})")
|
||||||
if perf_id_strings:
|
if perf_id_strings:
|
||||||
try:
|
try:
|
||||||
perf_ids = [uuid.UUID(s) for s in perf_id_strings]
|
perf_ids = [uuid.UUID(s) for s in perf_id_strings]
|
||||||
|
|
@ -145,63 +332,34 @@ def list_scenes(
|
||||||
)
|
)
|
||||||
|
|
||||||
if has_playback is True:
|
if has_playback is True:
|
||||||
# Tylko sceny z choć jednym ŻYWYM playback_source.
|
base = base.where(live_playback_exists())
|
||||||
base = base.where(
|
|
||||||
exists(
|
|
||||||
select(1).where(
|
|
||||||
PlaybackSource.scene_id == Scene.id,
|
|
||||||
PlaybackSource.dead_at.is_(None),
|
|
||||||
)
|
|
||||||
)
|
|
||||||
)
|
|
||||||
elif has_playback is False:
|
elif has_playback is False:
|
||||||
base = base.where(
|
base = base.where(~live_playback_exists())
|
||||||
~exists(
|
|
||||||
select(1).where(
|
|
||||||
PlaybackSource.scene_id == Scene.id,
|
|
||||||
PlaybackSource.dead_at.is_(None),
|
|
||||||
)
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
# Blacklisty — globalne wykluczenia. Jeśli scena ma JAKIEGOKOLWIEK blacklisted
|
if origin:
|
||||||
# performera, jest na blacklisted studio, lub ma JAKIKOLWIEK blacklisted tag → out.
|
# Substring match na origin — 'hqporner' złapie 'tube:hqpornercom'.
|
||||||
from app.models.blacklist import (
|
|
||||||
BlacklistedPerformer,
|
|
||||||
BlacklistedStudio,
|
|
||||||
BlacklistedTag,
|
|
||||||
)
|
|
||||||
base = base.where(
|
|
||||||
~exists(
|
|
||||||
select(1)
|
|
||||||
.select_from(ScenePerformer)
|
|
||||||
.join(BlacklistedPerformer, BlacklistedPerformer.performer_id == ScenePerformer.performer_id)
|
|
||||||
.where(ScenePerformer.scene_id == Scene.id)
|
|
||||||
)
|
|
||||||
)
|
|
||||||
base = base.where(
|
|
||||||
~Scene.studio_id.in_(select(BlacklistedStudio.studio_id))
|
|
||||||
)
|
|
||||||
base = base.where(
|
|
||||||
~exists(
|
|
||||||
select(1)
|
|
||||||
.select_from(SceneTag)
|
|
||||||
.join(BlacklistedTag, BlacklistedTag.tag_id == SceneTag.tag_id)
|
|
||||||
.where(SceneTag.scene_id == Scene.id)
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
if has_animated_thumbnail:
|
|
||||||
base = base.where(
|
base = base.where(
|
||||||
exists(
|
exists(
|
||||||
select(1).where(
|
select(1).where(
|
||||||
PlaybackSource.scene_id == Scene.id,
|
PlaybackSource.scene_id == Scene.id,
|
||||||
PlaybackSource.dead_at.is_(None),
|
PlaybackSource.dead_at.is_(None),
|
||||||
PlaybackSource.animated_thumbnail_url.isnot(None),
|
PlaybackSource.origin.ilike(f"%{origin}%"),
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# JAV vertical gate: domyślnie JAV wykluczone z głównego feedu; ?jav=true → TYLKO JAV.
|
||||||
|
# (origin=... diagnostyka nadal podlega bramie — do JAV użyj jav=true.)
|
||||||
|
if jav:
|
||||||
|
base = base.where(_jav_source_exists())
|
||||||
|
else:
|
||||||
|
base = base.where(~_jav_source_exists())
|
||||||
|
|
||||||
|
# Blacklisty device (performer/studio/tag) — globalne wykluczenia, współdzielone z
|
||||||
|
# licznikiem +N ulubionych. Puste blacklisty → [] (typowy single-user, zero kosztu).
|
||||||
|
for _bl_clause in blacklist_clauses(session, device_id):
|
||||||
|
base = base.where(_bl_clause)
|
||||||
|
|
||||||
if min_duration_sec is not None:
|
if min_duration_sec is not None:
|
||||||
base = base.where(Scene.duration_sec >= min_duration_sec)
|
base = base.where(Scene.duration_sec >= min_duration_sec)
|
||||||
if max_duration_sec is not None:
|
if max_duration_sec is not None:
|
||||||
|
|
@ -237,55 +395,44 @@ def list_scenes(
|
||||||
)
|
)
|
||||||
|
|
||||||
if not include_stubs:
|
if not include_stubs:
|
||||||
# Stub scene heuristic: tube-only scena BEZ release_date AND BEZ canonical
|
# Stub scene heuristic (współdzielona z licznikiem +N ulubionych): tube-only scena
|
||||||
# (TPDB/StashDB) ref AND BEZ żadnego ScenePerformer linka. ScenePerformer
|
# bez release_date AND bez canonical (TPDB/StashDB) AND bez performera. Continuous
|
||||||
# dodaje continuous worker (search-by-name → wymusza link), więc per-performer
|
# worker dodaje ScenePerformer (search-by-name), więc per-performer wynik nie jest
|
||||||
# search-result NIGDY nie jest stub. To filtruje tylko anonymous tube-only
|
# stubem. Filtruje anonymous tube-only sceny z newUrl/categories ingestu.
|
||||||
# sceny z newUrl/categories ingestu które nie zostały zsyntowane z performerem.
|
base = base.where(stub_exclusion_clause())
|
||||||
canonical_exists = exists(
|
|
||||||
select(1)
|
|
||||||
.select_from(SceneExternalRef)
|
|
||||||
.join(Source, Source.id == SceneExternalRef.source_id)
|
|
||||||
.where(SceneExternalRef.scene_id == Scene.id)
|
|
||||||
.where(Source.kind.in_([SourceKind.tpdb, SourceKind.stashdb]))
|
|
||||||
)
|
|
||||||
has_performer = exists(
|
|
||||||
select(1).where(ScenePerformer.scene_id == Scene.id)
|
|
||||||
)
|
|
||||||
# NOT stub gdy: ma canonical_ref OR ma release_date OR ma performera
|
|
||||||
base = base.where(
|
|
||||||
Scene.release_date.is_not(None) | canonical_exists | has_performer
|
|
||||||
)
|
|
||||||
|
|
||||||
# Count: dla dużych baz (~400k scen) pełny count z 3 nested EXISTS bierze ~5s.
|
_is_pure_default = (
|
||||||
# Liczymy total na uproszczonym query (bez stub-filter w count) — daje ~5% off
|
not include_stubs and not q and not studio_slug_list and not tag_slug_list
|
||||||
# ale jest akceptowalne dla user-facing pagination header. Items query NADAL
|
and not perf_id_strings and origin is None and has_playback is None
|
||||||
# ma stub-filter, więc lista pokazuje poprawne sceny. Liczba w header jest
|
and min_duration_sec is None
|
||||||
# przybliżoną górną granicą — co dla 400k scen i tak nie ma sensu reading dokładnie.
|
and max_duration_sec is None and released_within_days is None
|
||||||
if not include_stubs and not q and not studio_slug_list and not tags and not perf_id_strings:
|
and min_quality_p is None and not jav
|
||||||
# Fast path: typowy default request (lista bez filtra) — count tylko po
|
|
||||||
# has_playback (single EXISTS, dobrze zindeksowany).
|
|
||||||
count_query = select(func.count()).select_from(
|
|
||||||
select(Scene.id).where(
|
|
||||||
exists(
|
|
||||||
select(1).where(
|
|
||||||
PlaybackSource.scene_id == Scene.id,
|
|
||||||
PlaybackSource.dead_at.is_(None),
|
|
||||||
)
|
)
|
||||||
)
|
# Count strategy:
|
||||||
).subquery()
|
# - PURE default: cached pełny licznik katalogu (TTL 10 min).
|
||||||
)
|
# - FILTROWANE: NIE liczymy dokładnie. Bounded-count nad EXISTS-filtrami był
|
||||||
total = session.execute(count_query).scalar_one()
|
# dominującym kosztem (~4s na has_playback / min_duration / duży tag) i plan
|
||||||
else:
|
# był NIESTABILNY (literal LIMIT + count-nad-PK pomogły w części przypadków,
|
||||||
total = session.execute(select(func.count()).select_from(base.subquery())).scalar_one()
|
# ale planer i tak czasem skanuje cały zbiór zamiast urwać). Mobile paginuje
|
||||||
|
# po `has_more` (per_page+1 fetch), NIE po `total` — `total` to tylko licznik
|
||||||
|
# "N+" w UI. Wyprowadzamy go z has_more PO fetchu (patrz niżej): dolna granica
|
||||||
|
# + flaga "jest więcej". Eliminuje cały koszt count z każdej filtrowanej listy.
|
||||||
|
total_capped = False
|
||||||
|
total: int | None = _default_scene_count(session) if _is_pure_default else None
|
||||||
|
|
||||||
# Sort: zawsze tie-break po created_at desc dla determinizmu paginacji.
|
# Sort: KAŻDY tryb kończy się `Scene.id.desc()` — unikalny PK jako ostateczny
|
||||||
|
# tie-break. Bez tego przy OFFSET-paginacji sceny z IDENTYCZNYM created_at (masowy
|
||||||
|
# ingest = ten sam timestamp co do sekundy) i release_date mają NIEustalony porządek,
|
||||||
|
# więc kolejne strony nakładają się na siebie → feed "powtarza się" przy scrollu
|
||||||
|
# (report 304c9258). created_at desc dalej pierwszy, więc index-walk zostaje.
|
||||||
if sort == "release_date":
|
if sort == "release_date":
|
||||||
ordered = base.order_by(
|
ordered = base.order_by(
|
||||||
Scene.release_date.desc().nullslast(), Scene.created_at.desc()
|
Scene.release_date.desc().nullslast(), Scene.created_at.desc(), Scene.id.desc()
|
||||||
)
|
)
|
||||||
elif sort == "title":
|
elif sort == "title":
|
||||||
ordered = base.order_by(Scene.title_normalized.asc(), Scene.created_at.desc())
|
ordered = base.order_by(
|
||||||
|
Scene.title_normalized.asc(), Scene.created_at.desc(), Scene.id.desc()
|
||||||
|
)
|
||||||
elif sort == "studio":
|
elif sort == "studio":
|
||||||
# Sceny bez studio na końcu; w obrębie studio — najświeższe pierwsze.
|
# Sceny bez studio na końcu; w obrębie studio — najświeższe pierwsze.
|
||||||
ordered = (
|
ordered = (
|
||||||
|
|
@ -294,33 +441,80 @@ def list_scenes(
|
||||||
Studio.name_normalized.asc().nullslast(),
|
Studio.name_normalized.asc().nullslast(),
|
||||||
Scene.release_date.desc().nullslast(),
|
Scene.release_date.desc().nullslast(),
|
||||||
Scene.created_at.desc(),
|
Scene.created_at.desc(),
|
||||||
|
Scene.id.desc(),
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
else: # created_at
|
else: # created_at
|
||||||
ordered = base.order_by(
|
ordered = base.order_by(
|
||||||
Scene.created_at.desc(), Scene.release_date.desc().nullslast()
|
Scene.created_at.desc(), Scene.release_date.desc().nullslast(), Scene.id.desc()
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Fetch per_page+1 — obecność (per_page+1)-szego wiersza = jest kolejna strona.
|
||||||
|
# To źródło prawdy dla paginacji (mobile getNextPageParam), niezależne od bounded
|
||||||
|
# `total`. Nadmiarowy wiersz odcinamy przed serializacją.
|
||||||
|
# LIMIT/OFFSET literalne (NIE bound-param) — patrz wyżej: sparametryzowany LIMIT
|
||||||
|
# psuje early-termination i przy filtrach EXISTS planer robi gather-all+sort (sekundy)
|
||||||
|
# zamiast limit-aware index-walk po `ix_scenes_created_at_desc`. page/per_page to
|
||||||
|
# walidowane inty (Query ge=1, le=200), więc literal_column jest bezpieczne.
|
||||||
|
_off = (page - 1) * per_page
|
||||||
rows = (
|
rows = (
|
||||||
session.execute(ordered.offset((page - 1) * per_page).limit(per_page))
|
session.execute(
|
||||||
|
ordered.offset(literal_column(str(_off))).limit(literal_column(str(per_page + 1)))
|
||||||
|
)
|
||||||
.scalars()
|
.scalars()
|
||||||
.all()
|
.all()
|
||||||
)
|
)
|
||||||
|
has_more = len(rows) > per_page
|
||||||
|
rows = rows[:per_page]
|
||||||
|
|
||||||
items = _build_scenes_out_batch(session, list(rows))
|
# Filtrowane listy: total = dolna granica z dotychczas-widzianych wierszy, a
|
||||||
|
# total_capped=has_more daje UI "N+" (jest kolejna strona). Bez osobnego count query.
|
||||||
|
if total is None:
|
||||||
|
total = (page - 1) * per_page + len(rows)
|
||||||
|
total_capped = has_more
|
||||||
|
|
||||||
return SceneListOut(items=items, total=total, page=page, per_page=per_page)
|
items = _build_scenes_out_batch(session, list(rows), light=True, device_id=device_id)
|
||||||
|
|
||||||
|
return SceneListOut(
|
||||||
|
items=items,
|
||||||
|
total=total,
|
||||||
|
page=page,
|
||||||
|
per_page=per_page,
|
||||||
|
has_more=has_more,
|
||||||
|
total_capped=total_capped,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@router.get("/{scene_id}", response_model=SceneOut)
|
@router.get("/{scene_id}", response_model=SceneOut)
|
||||||
def get_scene(
|
def get_scene(
|
||||||
scene_id: uuid.UUID,
|
scene_id: uuid.UUID,
|
||||||
session: Annotated[Session, Depends(get_session)],
|
session: Annotated[Session, Depends(get_session)],
|
||||||
|
device_id: Annotated[str, Depends(get_device_id)],
|
||||||
) -> SceneOut:
|
) -> SceneOut:
|
||||||
scene = session.get(Scene, scene_id)
|
scene = session.get(Scene, scene_id)
|
||||||
if scene is None:
|
if scene is None:
|
||||||
raise HTTPException(status_code=404, detail="scene not found")
|
raise HTTPException(status_code=404, detail="scene not found")
|
||||||
return _build_scene_out(session, scene)
|
return _build_scene_out(session, scene, device_id=device_id)
|
||||||
|
|
||||||
|
|
||||||
|
_SXYPRN_POST_RE = re.compile(r"sxyprn\.com/post/([0-9a-f]{6,40})", re.IGNORECASE)
|
||||||
|
|
||||||
|
|
||||||
|
def _sxyprn_thumb_url(page_url: str | None) -> str | None:
|
||||||
|
"""Dla źródła sxyprn zwraca STABILNY endpoint on-demand resolvera
|
||||||
|
(`/proxy/sxyprn-thumb/<id>`) zamiast martwego trafficdeposit URL — token żyje ~1h,
|
||||||
|
więc poster resolvujemy przy serwowaniu (bug 2026-06-10)."""
|
||||||
|
if not page_url:
|
||||||
|
return None
|
||||||
|
m = _SXYPRN_POST_RE.search(page_url)
|
||||||
|
return f"/proxy/sxyprn-thumb/{m.group(1)}" if m else None
|
||||||
|
|
||||||
|
|
||||||
|
def _is_rotting_thumb(url: str) -> bool:
|
||||||
|
"""sxyprn/trafficdeposit miniaturki są czasowo podpisane i rotują (asset 404 po
|
||||||
|
~tygodniach, nie odświeżalne server-side; bug 2026-06-10). De-prioritize je w wyborze
|
||||||
|
slim-thumbnaila — używamy tylko gdy scena nie ma żadnej innej miniaturki."""
|
||||||
|
return "trafficdeposit.com" in url
|
||||||
|
|
||||||
|
|
||||||
def _needs_proxy(url: str) -> bool:
|
def _needs_proxy(url: str) -> bool:
|
||||||
|
|
@ -336,19 +530,28 @@ def _wrap_image_proxy(url: str, referer: str) -> str:
|
||||||
znać sekretu Referer — backend wstawi sam. Long TTL (30d) bo thumby
|
znać sekretu Referer — backend wstawi sam. Long TTL (30d) bo thumby
|
||||||
są stabilne, krótkie ttl by tylko niepotrzebnie zaśmiecało cache."""
|
są stabilne, krótkie ttl by tylko niepotrzebnie zaśmiecało cache."""
|
||||||
from app.api.stream_proxy import make_token
|
from app.api.stream_proxy import make_token
|
||||||
token = make_token(url, referer, ttl_sec=30 * 24 * 3600)
|
# stable_bucket_sec=7d → proxied URL identyczny przez tydzień → expo-image
|
||||||
|
# disk-cache hit zamiast re-download miniatur przy każdym fetchu listy / starcie apki.
|
||||||
|
token = make_token(url, referer, ttl_sec=30 * 24 * 3600, stable_bucket_sec=7 * 24 * 3600)
|
||||||
# Path zachowuje rozszerzenie żeby HTTP Content-Type był rozpoznany.
|
# Path zachowuje rozszerzenie żeby HTTP Content-Type był rozpoznany.
|
||||||
import os as _os
|
import os as _os
|
||||||
ext = _os.path.splitext(url.split("?")[0])[1].lstrip(".") or "jpg"
|
ext = _os.path.splitext(url.split("?")[0])[1].lstrip(".") or "jpg"
|
||||||
return f"/proxy/img/{token}/img.{ext}"
|
return f"/proxy/img/{token}/img.{ext}"
|
||||||
|
|
||||||
|
|
||||||
def _build_scenes_out_batch(session: Session, scenes: list[Scene]) -> list[SceneOut]:
|
def _build_scenes_out_batch(
|
||||||
|
session: Session, scenes: list[Scene], *, light: bool = False, device_id: str = LEGACY_DEVICE
|
||||||
|
) -> list[SceneOut]:
|
||||||
"""Batch-fetch wszystkich relacji dla N scen w 7 zapytaniach (zamiast 7×N).
|
"""Batch-fetch wszystkich relacji dla N scen w 7 zapytaniach (zamiast 7×N).
|
||||||
|
|
||||||
Eliminuje N+1 z `_build_scene_out` w listach scen — `/scenes?per_page=24` szło
|
Eliminuje N+1 z `_build_scene_out` w listach scen — `/scenes?per_page=24` szło
|
||||||
z ~9.6s do <500ms. Pojedyncza scena (`/scenes/{id}`) nadal używa `_build_scene_out`
|
z ~9.6s do <500ms. Pojedyncza scena (`/scenes/{id}`) nadal używa `_build_scene_out`
|
||||||
bo overhead na batch nie ma sensu dla N=1.
|
bo overhead na batch nie ma sensu dla N=1.
|
||||||
|
|
||||||
|
`light=True` (listy/grid): pomija `tags` i `external_refs` (kafelek SceneTile ich
|
||||||
|
NIE używa, a SceneDetail re-fetchuje pełną scenę osobno) i ślimaczy `playback_sources`
|
||||||
|
do 1 wpisu z samą miniaturką (kafelek czyta tylko thumbnail_url/animated_thumbnail_url).
|
||||||
|
Mniej DB + mniej payloadu + szybszy parse na kliencie (perf 2026-06-07).
|
||||||
"""
|
"""
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
if not scenes:
|
if not scenes:
|
||||||
|
|
@ -384,23 +587,24 @@ def _build_scenes_out_batch(session: Session, scenes: list[Scene]) -> list[Scene
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
# 3) Tags
|
# 3) Tags + 4) External refs — kafelek listy ich nie używa; w light mode pomijamy
|
||||||
|
# (SceneDetail re-fetchuje pełną scenę przez /scenes/{id}).
|
||||||
|
tags_by_scene: dict = defaultdict(list)
|
||||||
|
refs_by_scene: dict = defaultdict(list)
|
||||||
|
if not light:
|
||||||
tag_rows = session.execute(
|
tag_rows = session.execute(
|
||||||
select(SceneTag.scene_id, Tag)
|
select(SceneTag.scene_id, Tag)
|
||||||
.join(Tag, Tag.id == SceneTag.tag_id)
|
.join(Tag, Tag.id == SceneTag.tag_id)
|
||||||
.where(SceneTag.scene_id.in_(scene_ids))
|
.where(SceneTag.scene_id.in_(scene_ids))
|
||||||
).all()
|
).all()
|
||||||
tags_by_scene: dict = defaultdict(list)
|
|
||||||
for sid, t in tag_rows:
|
for sid, t in tag_rows:
|
||||||
tags_by_scene[sid].append(TagOut.model_validate(t))
|
tags_by_scene[sid].append(TagOut.model_validate(t))
|
||||||
|
|
||||||
# 4) External refs + sources
|
|
||||||
ref_rows = session.execute(
|
ref_rows = session.execute(
|
||||||
select(SceneExternalRef, Source)
|
select(SceneExternalRef, Source)
|
||||||
.join(Source, Source.id == SceneExternalRef.source_id)
|
.join(Source, Source.id == SceneExternalRef.source_id)
|
||||||
.where(SceneExternalRef.scene_id.in_(scene_ids))
|
.where(SceneExternalRef.scene_id.in_(scene_ids))
|
||||||
).all()
|
).all()
|
||||||
refs_by_scene: dict = defaultdict(list)
|
|
||||||
for ref, src in ref_rows:
|
for ref, src in ref_rows:
|
||||||
refs_by_scene[ref.scene_id].append(
|
refs_by_scene[ref.scene_id].append(
|
||||||
ExternalRefOut(
|
ExternalRefOut(
|
||||||
|
|
@ -411,7 +615,68 @@ def _build_scenes_out_batch(session: Session, scenes: list[Scene]) -> list[Scene
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
# 5) Playback sources
|
# 5) Playback sources. Light mode: tylko miniaturka (jedna na scenę) — kafelek
|
||||||
|
# czyta wyłącznie playback_sources[].thumbnail_url / animated_thumbnail_url.
|
||||||
|
pb_by_scene: dict = defaultdict(list)
|
||||||
|
if light:
|
||||||
|
pb_light = session.execute(
|
||||||
|
select(
|
||||||
|
PlaybackSource.scene_id,
|
||||||
|
PlaybackSource.thumbnail_url,
|
||||||
|
PlaybackSource.animated_thumbnail_url,
|
||||||
|
PlaybackSource.page_url,
|
||||||
|
)
|
||||||
|
.where(
|
||||||
|
PlaybackSource.scene_id.in_(scene_ids),
|
||||||
|
PlaybackSource.dead_at.is_(None),
|
||||||
|
)
|
||||||
|
.order_by(PlaybackSource.origin.asc())
|
||||||
|
).all()
|
||||||
|
# Pierwsza miniaturka + pierwszy animated per scena (1 slim wpis). De-prioritize
|
||||||
|
# sxyprn/trafficdeposit thumbnaile — są podpisane czasowo i ROTUJĄ (asset 404 po
|
||||||
|
# ~tygodniach, nie da się re-signować; bug 2026-06-10). Wolimy miniaturkę z innego
|
||||||
|
# źródła gdy istnieje; sxyprn bierzemy tylko gdy nic innego nie ma (świeże jeszcze
|
||||||
|
# działają, martwe → mobile pokazuje placeholder zamiast broken-image).
|
||||||
|
thumb_by_scene: dict = {}
|
||||||
|
thumb_fallback: dict = {}
|
||||||
|
anim_by_scene: dict = {}
|
||||||
|
for sid, thumb, anim, page_url in pb_light:
|
||||||
|
sxy = _sxyprn_thumb_url(page_url)
|
||||||
|
if sxy:
|
||||||
|
# sxyprn → żywy on-demand resolver (martwy stored URL ignorujemy),
|
||||||
|
# tier fallback: użyty tylko gdy scena nie ma stabilniejszej miniatury.
|
||||||
|
thumb_fallback.setdefault(sid, (sxy, page_url))
|
||||||
|
elif thumb:
|
||||||
|
if _is_rotting_thumb(thumb):
|
||||||
|
thumb_fallback.setdefault(sid, (thumb, page_url))
|
||||||
|
elif sid not in thumb_by_scene:
|
||||||
|
thumb_by_scene[sid] = (thumb, page_url)
|
||||||
|
if sid not in anim_by_scene and anim:
|
||||||
|
anim_by_scene[sid] = (anim, page_url)
|
||||||
|
# Uzupełnij scenami które mają TYLKO rotting thumbnail (sxyprn-only).
|
||||||
|
for sid, val in thumb_fallback.items():
|
||||||
|
thumb_by_scene.setdefault(sid, val)
|
||||||
|
for sid in scene_ids:
|
||||||
|
t = thumb_by_scene.get(sid)
|
||||||
|
a = anim_by_scene.get(sid)
|
||||||
|
if not t and not a:
|
||||||
|
continue
|
||||||
|
t_url = t[0] if t else None
|
||||||
|
a_url = a[0] if a else None
|
||||||
|
ref = (t or a)[1]
|
||||||
|
if t_url and _needs_proxy(t_url):
|
||||||
|
t_url = _wrap_image_proxy(t_url, ref)
|
||||||
|
if a_url and _needs_proxy(a_url):
|
||||||
|
a_url = _wrap_image_proxy(a_url, ref)
|
||||||
|
# id/origin/page_url wymagane przez schemat ale nieużywane przez kafelek
|
||||||
|
# (SceneDetail re-fetchuje pełne źródła) — dummy sentinel.
|
||||||
|
pb_by_scene[sid].append(
|
||||||
|
PlaybackSourceOut(
|
||||||
|
id=uuid.UUID(int=0), origin="", page_url="",
|
||||||
|
thumbnail_url=t_url, animated_thumbnail_url=a_url,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
else:
|
||||||
pb_rows = session.execute(
|
pb_rows = session.execute(
|
||||||
select(PlaybackSource)
|
select(PlaybackSource)
|
||||||
.where(
|
.where(
|
||||||
|
|
@ -420,27 +685,40 @@ def _build_scenes_out_batch(session: Session, scenes: list[Scene]) -> list[Scene
|
||||||
)
|
)
|
||||||
.order_by(PlaybackSource.origin.asc())
|
.order_by(PlaybackSource.origin.asc())
|
||||||
).scalars().all()
|
).scalars().all()
|
||||||
pb_by_scene: dict = defaultdict(list)
|
|
||||||
for p in pb_rows:
|
for p in pb_rows:
|
||||||
out = PlaybackSourceOut.model_validate(p)
|
out = PlaybackSourceOut.model_validate(p)
|
||||||
|
# sxyprn/trafficdeposit stored thumb rotuje (404 po ~tygodniach). SceneDetail
|
||||||
|
# bierze PIERWSZE źródło z thumbem (origin ASC → sxyprncom często pierwsze), więc
|
||||||
|
# bez tego pokazywał martwy obraz mimo że LISTA używała żywej wersji (report:
|
||||||
|
# "na liście jest miniaturka, w scenie nie"). Podmieniamy na żywy resolver;
|
||||||
|
# inne martwe rotting-thumby zerujemy → mobile bierze kolejne źródło / placeholder.
|
||||||
|
sxy = _sxyprn_thumb_url(p.page_url)
|
||||||
|
if sxy:
|
||||||
|
out.thumbnail_url = sxy
|
||||||
|
elif out.thumbnail_url and _is_rotting_thumb(out.thumbnail_url):
|
||||||
|
out.thumbnail_url = None
|
||||||
if out.thumbnail_url and _needs_proxy(out.thumbnail_url):
|
if out.thumbnail_url and _needs_proxy(out.thumbnail_url):
|
||||||
out.thumbnail_url = _wrap_image_proxy(out.thumbnail_url, p.page_url)
|
out.thumbnail_url = _wrap_image_proxy(out.thumbnail_url, p.page_url)
|
||||||
if out.animated_thumbnail_url and _needs_proxy(out.animated_thumbnail_url):
|
if out.animated_thumbnail_url and _needs_proxy(out.animated_thumbnail_url):
|
||||||
out.animated_thumbnail_url = _wrap_image_proxy(out.animated_thumbnail_url, p.page_url)
|
out.animated_thumbnail_url = _wrap_image_proxy(out.animated_thumbnail_url, p.page_url)
|
||||||
pb_by_scene[p.scene_id].append(out)
|
pb_by_scene[p.scene_id].append(out)
|
||||||
|
|
||||||
# 6) Progress
|
# 6) Progress (device-scoped)
|
||||||
progress_by_scene: dict = {}
|
progress_by_scene: dict = {}
|
||||||
for prog in session.execute(
|
for prog in session.execute(
|
||||||
select(ScenePlayProgress).where(ScenePlayProgress.scene_id.in_(scene_ids))
|
select(ScenePlayProgress).where(
|
||||||
|
ScenePlayProgress.scene_id.in_(scene_ids),
|
||||||
|
ScenePlayProgress.device_id == device_id,
|
||||||
|
)
|
||||||
).scalars():
|
).scalars():
|
||||||
progress_by_scene[prog.scene_id] = prog
|
progress_by_scene[prog.scene_id] = prog
|
||||||
|
|
||||||
# 7) Favorites
|
# 7) Favorites (device-scoped)
|
||||||
fav_scene_ids: set = set(
|
fav_scene_ids: set = set(
|
||||||
session.execute(
|
session.execute(
|
||||||
select(FavoriteScene.scene_id).where(
|
select(FavoriteScene.scene_id).where(
|
||||||
FavoriteScene.scene_id.in_(scene_ids)
|
FavoriteScene.scene_id.in_(scene_ids),
|
||||||
|
FavoriteScene.device_id == device_id,
|
||||||
)
|
)
|
||||||
).scalars()
|
).scalars()
|
||||||
)
|
)
|
||||||
|
|
@ -467,6 +745,7 @@ def _build_scenes_out_batch(session: Session, scenes: list[Scene]) -> list[Scene
|
||||||
external_refs=refs_by_scene.get(scene.id, []),
|
external_refs=refs_by_scene.get(scene.id, []),
|
||||||
playback_sources=pb_by_scene.get(scene.id, []),
|
playback_sources=pb_by_scene.get(scene.id, []),
|
||||||
created_at=scene.created_at,
|
created_at=scene.created_at,
|
||||||
|
backfill=scene.backfill,
|
||||||
last_played_at=progress.last_played_at if progress else None,
|
last_played_at=progress.last_played_at if progress else None,
|
||||||
finished=progress.finished if progress else False,
|
finished=progress.finished if progress else False,
|
||||||
position_sec=progress.position_sec if progress else 0,
|
position_sec=progress.position_sec if progress else 0,
|
||||||
|
|
@ -476,7 +755,7 @@ def _build_scenes_out_batch(session: Session, scenes: list[Scene]) -> list[Scene
|
||||||
return out
|
return out
|
||||||
|
|
||||||
|
|
||||||
def _build_scene_out(session: Session, scene: Scene) -> SceneOut:
|
def _build_scene_out(session: Session, scene: Scene, *, device_id: str = LEGACY_DEVICE) -> SceneOut:
|
||||||
studio_out: StudioOut | None = None
|
studio_out: StudioOut | None = None
|
||||||
if scene.studio_id is not None:
|
if scene.studio_id is not None:
|
||||||
st = session.get(Studio, scene.studio_id)
|
st = session.get(Studio, scene.studio_id)
|
||||||
|
|
@ -537,6 +816,27 @@ def _build_scene_out(session: Session, scene: Scene) -> SceneOut:
|
||||||
.scalars()
|
.scalars()
|
||||||
.all()
|
.all()
|
||||||
)
|
)
|
||||||
|
# Collapse źródła dzielące ten sam origin (hoster). Zmergowana scena często agreguje
|
||||||
|
# kilka uploadów z JEDNEGO tube'a (re-enkody / wersje 4K: bug-report aa79a995 "2 linki,
|
||||||
|
# oba do porntrex" = ta sama scena std+4K) — w UI to nierozróżnialne linki do tego
|
||||||
|
# samego hostera (resolvują tym samym extractorem). Zostawiamy jeden najlepszy per
|
||||||
|
# origin: preferuj długość zgodną ze sceną (realny match) → jakąkolwiek długość →
|
||||||
|
# pierwszy (stabilnie, query jest origin-asc). Martwe już odfiltrowane (dead_at).
|
||||||
|
def _origin_pick_key(p: PlaybackSource) -> tuple[int, int]:
|
||||||
|
dur_match = (
|
||||||
|
0 if (scene.duration_sec and p.duration_sec
|
||||||
|
and abs(p.duration_sec - scene.duration_sec) <= 5) else 1
|
||||||
|
)
|
||||||
|
return (dur_match, 0 if p.duration_sec else 1)
|
||||||
|
|
||||||
|
_best_by_origin: dict[str, PlaybackSource] = {}
|
||||||
|
for p in playback_rows:
|
||||||
|
key = p.origin or ""
|
||||||
|
cur = _best_by_origin.get(key)
|
||||||
|
if cur is None or _origin_pick_key(p) < _origin_pick_key(cur):
|
||||||
|
_best_by_origin[key] = p
|
||||||
|
playback_rows = list(_best_by_origin.values())
|
||||||
|
|
||||||
playback_out: list[PlaybackSourceOut] = []
|
playback_out: list[PlaybackSourceOut] = []
|
||||||
for p in playback_rows:
|
for p in playback_rows:
|
||||||
out = PlaybackSourceOut.model_validate(p)
|
out = PlaybackSourceOut.model_validate(p)
|
||||||
|
|
@ -549,8 +849,22 @@ def _build_scene_out(session: Session, scene: Scene) -> SceneOut:
|
||||||
out.animated_thumbnail_url = _wrap_image_proxy(out.animated_thumbnail_url, p.page_url)
|
out.animated_thumbnail_url = _wrap_image_proxy(out.animated_thumbnail_url, p.page_url)
|
||||||
playback_out.append(out)
|
playback_out.append(out)
|
||||||
|
|
||||||
progress = session.get(ScenePlayProgress, scene.id)
|
# Rank natywne-resolve źródła PRZED WebView-fallback (IP-bound/ad-heavy: fpoxxx,
|
||||||
is_fav = session.get(FavoriteScene, scene.id) is not None
|
# pornxpph, pornhub...). Query był alfabetyczny po origin, więc np. fpoxxx-WebView
|
||||||
|
# pokazywał się przed działającym freshporno (bug-report 2026-06-07). Stabilny sort:
|
||||||
|
# natywne (0) → fallback (1), tie-break po origin.
|
||||||
|
from app.extractors import is_vps_blocked_fallback
|
||||||
|
|
||||||
|
def _resolve_rank(origin: str | None) -> int:
|
||||||
|
if not origin:
|
||||||
|
return 1
|
||||||
|
sitetag = origin.split(":", 1)[1] if ":" in origin else origin
|
||||||
|
return 1 if is_vps_blocked_fallback(sitetag) else 0
|
||||||
|
|
||||||
|
playback_out.sort(key=lambda o: (_resolve_rank(o.origin), o.origin or ""))
|
||||||
|
|
||||||
|
progress = session.get(ScenePlayProgress, (device_id, scene.id))
|
||||||
|
is_fav = session.get(FavoriteScene, (device_id, scene.id)) is not None
|
||||||
|
|
||||||
return SceneOut(
|
return SceneOut(
|
||||||
id=scene.id,
|
id=scene.id,
|
||||||
|
|
@ -567,6 +881,7 @@ def _build_scene_out(session: Session, scene: Scene) -> SceneOut:
|
||||||
external_refs=refs_out,
|
external_refs=refs_out,
|
||||||
playback_sources=playback_out,
|
playback_sources=playback_out,
|
||||||
created_at=scene.created_at,
|
created_at=scene.created_at,
|
||||||
|
backfill=scene.backfill,
|
||||||
last_played_at=progress.last_played_at if progress else None,
|
last_played_at=progress.last_played_at if progress else None,
|
||||||
finished=progress.finished if progress else False,
|
finished=progress.finished if progress else False,
|
||||||
position_sec=progress.position_sec if progress else 0,
|
position_sec=progress.position_sec if progress else 0,
|
||||||
|
|
@ -622,6 +937,67 @@ def remove_performer_from_scene(
|
||||||
session.commit()
|
session.commit()
|
||||||
|
|
||||||
|
|
||||||
|
class SceneHideOut(BaseModel):
|
||||||
|
scene_id: uuid.UUID
|
||||||
|
playback_marked_dead: int
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/{scene_id}/hide", response_model=SceneHideOut)
|
||||||
|
def hide_scene(
|
||||||
|
scene_id: uuid.UUID,
|
||||||
|
session: Annotated[Session, Depends(get_session)],
|
||||||
|
) -> SceneHideOut:
|
||||||
|
"""Ukryj scenę (user long-press → „usuń"). Oznacza wszystkie playback_sources
|
||||||
|
jako dead → scena wypada z list (has_playback=false). Odwracalne w DB (dead_at).
|
||||||
|
Nie kasujemy wiersza sceny — zachowujemy refs/dedup, tylko znika z UI."""
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
|
||||||
|
from app.models.playback_source import PlaybackSource
|
||||||
|
|
||||||
|
if session.get(Scene, scene_id) is None:
|
||||||
|
raise HTTPException(status_code=404, detail="scene not found")
|
||||||
|
rows = session.execute(
|
||||||
|
select(PlaybackSource).where(
|
||||||
|
PlaybackSource.scene_id == scene_id,
|
||||||
|
PlaybackSource.dead_at.is_(None),
|
||||||
|
)
|
||||||
|
).scalars().all()
|
||||||
|
now = datetime.now(UTC)
|
||||||
|
for p in rows:
|
||||||
|
p.dead_at = now
|
||||||
|
p.dead_reason = "user hid scene (long-press)"
|
||||||
|
session.commit()
|
||||||
|
return SceneHideOut(scene_id=scene_id, playback_marked_dead=len(rows))
|
||||||
|
|
||||||
|
|
||||||
|
class SceneMergeOut(BaseModel):
|
||||||
|
keep_id: uuid.UUID
|
||||||
|
dropped_id: uuid.UUID
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/{keep_id}/merge/{drop_id}", response_model=SceneMergeOut)
|
||||||
|
def merge_duplicate_scene(
|
||||||
|
keep_id: uuid.UUID,
|
||||||
|
drop_id: uuid.UUID,
|
||||||
|
session: Annotated[Session, Depends(get_session)],
|
||||||
|
) -> SceneMergeOut:
|
||||||
|
"""Scal `drop_id` w `keep_id` (user long-press → „oznacz duplikat" → wybór drugiej
|
||||||
|
sceny). Przenosi refs/performers/tags/fingerprints/playback (scene_merge), kasuje
|
||||||
|
`drop`. keep = scena na której user trzyma (zostaje), drop = wskazany duplikat."""
|
||||||
|
from app.resolve.scene_merge import MergeError, merge_scenes
|
||||||
|
|
||||||
|
if keep_id == drop_id:
|
||||||
|
raise HTTPException(status_code=400, detail="keep_id == drop_id")
|
||||||
|
if session.get(Scene, keep_id) is None or session.get(Scene, drop_id) is None:
|
||||||
|
raise HTTPException(status_code=404, detail="scene not found")
|
||||||
|
try:
|
||||||
|
merge_scenes(session, keep_id=keep_id, drop_id=drop_id, resolved_by="user_long_press_duplicate")
|
||||||
|
except MergeError as e:
|
||||||
|
raise HTTPException(status_code=400, detail=str(e)) from e
|
||||||
|
session.commit()
|
||||||
|
return SceneMergeOut(keep_id=keep_id, dropped_id=drop_id)
|
||||||
|
|
||||||
|
|
||||||
class EnrichTagsOut(BaseModel):
|
class EnrichTagsOut(BaseModel):
|
||||||
scene_id: uuid.UUID
|
scene_id: uuid.UUID
|
||||||
added: int
|
added: int
|
||||||
|
|
@ -660,7 +1036,7 @@ def enrich_tags_from_tube(
|
||||||
|
|
||||||
# Priority: mainstream tubes (bogate metadane) > niche (mniej tagów albo garbage).
|
# Priority: mainstream tubes (bogate metadane) > niche (mniej tagów albo garbage).
|
||||||
PRIORITY = ["xhamstercom", "porntrexcom", "epornercom", "youporncom",
|
PRIORITY = ["xhamstercom", "porntrexcom", "epornercom", "youporncom",
|
||||||
"xvideoscom", "xnxxcom", "redtubecom", "pornhatcom"]
|
"xvideoscom", "xnxxcom"]
|
||||||
sources = session.execute(
|
sources = session.execute(
|
||||||
select(PlaybackSource).where(
|
select(PlaybackSource).where(
|
||||||
PlaybackSource.scene_id == scene_id,
|
PlaybackSource.scene_id == scene_id,
|
||||||
|
|
@ -896,6 +1272,7 @@ class EnrichThumbOut(BaseModel):
|
||||||
def enrich_thumbnail_from_tube(
|
def enrich_thumbnail_from_tube(
|
||||||
scene_id: uuid.UUID,
|
scene_id: uuid.UUID,
|
||||||
session: Annotated[Session, Depends(get_session)],
|
session: Annotated[Session, Depends(get_session)],
|
||||||
|
force: bool = False,
|
||||||
) -> EnrichThumbOut:
|
) -> EnrichThumbOut:
|
||||||
"""Pobiera detail page z dowolnego tube playback_source bez thumbnail_url
|
"""Pobiera detail page z dowolnego tube playback_source bez thumbnail_url
|
||||||
i wyciąga miniaturkę (og:image / twitter:image / LD-JSON thumbnailUrl /
|
i wyciąga miniaturkę (og:image / twitter:image / LD-JSON thumbnailUrl /
|
||||||
|
|
@ -904,7 +1281,10 @@ def enrich_thumbnail_from_tube(
|
||||||
Update'uje WSZYSTKIE PlaybackSource'y dla tej sceny które nie mają thumb,
|
Update'uje WSZYSTKIE PlaybackSource'y dla tej sceny które nie mają thumb,
|
||||||
żeby kolejne otwarcia listy widziały miniaturę niezależnie od source pick.
|
żeby kolejne otwarcia listy widziały miniaturę niezależnie od source pick.
|
||||||
Mobile auto-wywoła to przy otwarciu SceneDetail bez thumb (jak duration).
|
Mobile auto-wywoła to przy otwarciu SceneDetail bez thumb (jak duration).
|
||||||
"""
|
|
||||||
|
`force=true` (przycisk "Refresh thumbnail" na SceneDetail, zgłoszenie d3376a71):
|
||||||
|
NADPISUJE istniejącą miniaturę świeżą ze strony tube'a — dla zepsutych/stałych
|
||||||
|
(rotting sxyprn/trafficdeposit, błędna grafika)."""
|
||||||
from app.extractors._fetch import browser_get
|
from app.extractors._fetch import browser_get
|
||||||
from app.extractors._models import TubePageError
|
from app.extractors._models import TubePageError
|
||||||
from app.extractors.thumb_extract import extract_thumbnail_url
|
from app.extractors.thumb_extract import extract_thumbnail_url
|
||||||
|
|
@ -923,8 +1303,8 @@ def enrich_thumbnail_from_tube(
|
||||||
).scalars().all()
|
).scalars().all()
|
||||||
|
|
||||||
sources_with_thumb = [s for s in sources if s.thumbnail_url]
|
sources_with_thumb = [s for s in sources if s.thumbnail_url]
|
||||||
if sources_with_thumb:
|
if sources_with_thumb and not force:
|
||||||
# już mamy — idempotent return.
|
# już mamy — idempotent return (force=true pomija, żeby odświeżyć).
|
||||||
return EnrichThumbOut(
|
return EnrichThumbOut(
|
||||||
scene_id=scene_id,
|
scene_id=scene_id,
|
||||||
thumbnail_url=sources_with_thumb[0].thumbnail_url,
|
thumbnail_url=sources_with_thumb[0].thumbnail_url,
|
||||||
|
|
@ -941,10 +1321,10 @@ def enrich_thumbnail_from_tube(
|
||||||
continue
|
continue
|
||||||
thumb = extract_thumbnail_url(r.text)
|
thumb = extract_thumbnail_url(r.text)
|
||||||
if thumb:
|
if thumb:
|
||||||
# Zapisz na wszystkich źródłach bez thumb (oszczędza duplikat fetch)
|
# Zapisz na wszystkich źródłach bez thumb (force → też nadpisz istniejące).
|
||||||
updated = 0
|
updated = 0
|
||||||
for s in sources:
|
for s in sources:
|
||||||
if not s.thumbnail_url:
|
if force or not s.thumbnail_url:
|
||||||
s.thumbnail_url = thumb
|
s.thumbnail_url = thumb
|
||||||
updated += 1
|
updated += 1
|
||||||
session.commit()
|
session.commit()
|
||||||
|
|
|
||||||
|
|
@ -70,6 +70,9 @@ class SceneOut(BaseModel):
|
||||||
# "NEW" na karcie scen w PerformerScenesScreen / StudioScenesScreen — gdy
|
# "NEW" na karcie scen w PerformerScenesScreen / StudioScenesScreen — gdy
|
||||||
# `created_at > last_seen_at` (favorite) → badge.
|
# `created_at > last_seen_at` (favorite) → badge.
|
||||||
created_at: datetime | None = None
|
created_at: datetime | None = None
|
||||||
|
# True = scena z masowego backfillu katalogu (deep-crawl). Mobile NIE pokazuje na niej
|
||||||
|
# NEW badge nawet gdy created_at > last_seen, to stara treść, nie świeży release.
|
||||||
|
backfill: bool = False
|
||||||
# Watched indicator (z `scene_play_progress`): mobile dim'uje kafelek gdy
|
# Watched indicator (z `scene_play_progress`): mobile dim'uje kafelek gdy
|
||||||
# `finished=True`, pokazuje progress bar gdy `position_sec > 0`.
|
# `finished=True`, pokazuje progress bar gdy `position_sec > 0`.
|
||||||
last_played_at: datetime | None = None
|
last_played_at: datetime | None = None
|
||||||
|
|
@ -83,6 +86,12 @@ class SceneListOut(BaseModel):
|
||||||
total: int
|
total: int
|
||||||
page: int
|
page: int
|
||||||
per_page: int
|
per_page: int
|
||||||
|
# has_more: czy istnieje kolejna strona. Liczone z fetcha per_page+1 (≈darmowe),
|
||||||
|
# NIE z `total` — bo dla filtrowanych list `total` jest bounded ("1000+") żeby
|
||||||
|
# uniknąć ~5s exhaustive count. Mobile paginuje po has_more, nie po total.
|
||||||
|
has_more: bool = False
|
||||||
|
# total_capped: True gdy `total` to bounded cap (są >total wyników). UI: "{total}+".
|
||||||
|
total_capped: bool = False
|
||||||
|
|
||||||
|
|
||||||
class MovieChapterOut(BaseModel):
|
class MovieChapterOut(BaseModel):
|
||||||
|
|
@ -118,6 +127,11 @@ class MovieOut(BaseModel):
|
||||||
# and MovieDetail favorite star.
|
# and MovieDetail favorite star.
|
||||||
created_at: datetime | None = None
|
created_at: datetime | None = None
|
||||||
is_favorite: bool = False
|
is_favorite: bool = False
|
||||||
|
# Watched / continue-watching state (mirror SceneOut, bug-report b207ff17
|
||||||
|
# 2026-05-26 "przydałoby się oznaczenie filmów już obejrzanych").
|
||||||
|
last_played_at: datetime | None = None
|
||||||
|
finished: bool = False
|
||||||
|
position_sec: int = 0
|
||||||
|
|
||||||
|
|
||||||
class MovieListOut(BaseModel):
|
class MovieListOut(BaseModel):
|
||||||
|
|
|
||||||
513
app/api/seo.py
Normal file
513
app/api/seo.py
Normal file
|
|
@ -0,0 +1,513 @@
|
||||||
|
"""Publiczna, crawlowalna powierzchnia SEO — programmatic entity pages.
|
||||||
|
|
||||||
|
Jedyny publiczny router HTML poza /static i healthchecks — NIE wymaga api key,
|
||||||
|
bo Googlebot/użytkownik musi dotrzeć bez tokenu. Cel: łapać nawigacyjny long-tail
|
||||||
|
(nazwy performerów / studiów / tytuły scen), którego mainstream-SEO nie indeksuje,
|
||||||
|
a który realnie generuje organiczny ruch → pobranie apki.
|
||||||
|
|
||||||
|
Zasady (świadome, nie przypadkowe):
|
||||||
|
|
||||||
|
* **LINK-OUT only.** Strony renderują metadane i odsyłają do źródeł (`page_url`).
|
||||||
|
NIE eksponujemy `stream_url`/`embed_url` — to zostaje value-add apki i trzyma
|
||||||
|
legalny profil "agregatora/wyszukiwarki" (dowozimy ruch tubom, nie re-streamujemy).
|
||||||
|
* **Age-gate = client-side overlay** (cookie `age_ok`). Treść jest w HTML, więc
|
||||||
|
crawler ją indeksuje; overlay zasłania ją tylko ludziom do potwierdzenia 18+.
|
||||||
|
Dodatkowo `RTA` meta tag dla filtrów rodzicielskich.
|
||||||
|
* **Blacklist respektowany** — te same wykluczenia performer/studio/tag co w /scenes.
|
||||||
|
Nie publikujemy SEO-stron dla zblacklistowanej treści.
|
||||||
|
* **Anti-thin-page** — strona encji powstaje tylko gdy ma realną zawartość
|
||||||
|
(performer/studio z ≥1 żywą sceną; scena z ≥1 żywym source). Pusta encja → 404,
|
||||||
|
nie pusty doorway (Google karze masowe cienkie strony).
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
import uuid
|
||||||
|
from datetime import date
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Annotated
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Depends, HTTPException, Request
|
||||||
|
from fastapi.responses import HTMLResponse, PlainTextResponse, Response
|
||||||
|
from fastapi.templating import Jinja2Templates
|
||||||
|
from sqlalchemy import exists, func, select
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app.api.scenes import _needs_proxy, _wrap_image_proxy
|
||||||
|
from app.db import get_session
|
||||||
|
from app.models.blacklist import (
|
||||||
|
BlacklistedPerformer,
|
||||||
|
BlacklistedStudio,
|
||||||
|
BlacklistedTag,
|
||||||
|
)
|
||||||
|
from app.models.performer import Performer, PerformerAlias
|
||||||
|
from app.models.playback_source import PlaybackSource
|
||||||
|
from app.models.scene import Scene, ScenePerformer, SceneTag
|
||||||
|
from app.models.studio import Studio
|
||||||
|
from app.models.tag import Tag
|
||||||
|
|
||||||
|
_TEMPLATES_DIR = Path(__file__).resolve().parent.parent / "templates"
|
||||||
|
templates = Jinja2Templates(directory=str(_TEMPLATES_DIR))
|
||||||
|
|
||||||
|
# Analityka — wstrzykiwana do każdej strony SEO tylko gdy odpowiedni env jest ustawiony.
|
||||||
|
# Puste = tag się nie renderuje (zero third-party requestów, zachowanie bez zmian).
|
||||||
|
# Włączenie = ustaw zmienną w .env na VPS + restart, bez zmian w kodzie.
|
||||||
|
templates.env.globals["gtm_id"] = os.environ.get("GOON_GTM_ID", "")
|
||||||
|
templates.env.globals["ga4_id"] = os.environ.get("GOON_GA4_ID", "")
|
||||||
|
templates.env.globals["gsc_verify"] = os.environ.get("GOON_GSC_VERIFY", "")
|
||||||
|
|
||||||
|
# Limit URL-i na pojedynczy plik sitemap (spec: max 50k). Trzymamy z zapasem.
|
||||||
|
_SITEMAP_PAGE = 25_000
|
||||||
|
# Ile scen renderujemy na stronie encji (performer/studio) — pełna lista 1000+ scen
|
||||||
|
# to thin/slow; bierzemy najświeższe N, reszta i tak wpada przez sitemap scen.
|
||||||
|
_SCENES_PER_ENTITY = 120
|
||||||
|
|
||||||
|
router = APIRouter(tags=["seo"])
|
||||||
|
|
||||||
|
|
||||||
|
def base_url() -> str:
|
||||||
|
"""Publiczny origin pod którym serwowane są te strony (do canonical/sitemap/OG)."""
|
||||||
|
return os.environ.get("BACKEND_PUBLIC_URL", "https://goon-foss.org").rstrip("/")
|
||||||
|
|
||||||
|
|
||||||
|
# --- reużywalne fragmenty zapytań -------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _live_playback_exists():
|
||||||
|
"""EXISTS: scena ma ≥1 żywy (dead_at IS NULL) playback_source."""
|
||||||
|
return exists(
|
||||||
|
select(1).where(
|
||||||
|
PlaybackSource.scene_id == Scene.id,
|
||||||
|
PlaybackSource.dead_at.is_(None),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _not_blacklisted():
|
||||||
|
"""Lista warunków WHERE wykluczających zblacklistowaną treść (performer/studio/tag).
|
||||||
|
|
||||||
|
Te same reguły co w GET /scenes — żeby SEO nie publikowało tego, co katalog ukrywa.
|
||||||
|
"""
|
||||||
|
return [
|
||||||
|
~exists(
|
||||||
|
select(1)
|
||||||
|
.select_from(ScenePerformer)
|
||||||
|
.join(
|
||||||
|
BlacklistedPerformer,
|
||||||
|
BlacklistedPerformer.performer_id == ScenePerformer.performer_id,
|
||||||
|
)
|
||||||
|
.where(ScenePerformer.scene_id == Scene.id)
|
||||||
|
),
|
||||||
|
~Scene.studio_id.in_(select(BlacklistedStudio.studio_id)),
|
||||||
|
~exists(
|
||||||
|
select(1)
|
||||||
|
.select_from(SceneTag)
|
||||||
|
.join(BlacklistedTag, BlacklistedTag.tag_id == SceneTag.tag_id)
|
||||||
|
.where(SceneTag.scene_id == Scene.id)
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _indexable_scenes():
|
||||||
|
"""SELECT Scene żywych, nie-zblacklistowanych scen — baza pod listy i sitemap."""
|
||||||
|
stmt = select(Scene).where(_live_playback_exists())
|
||||||
|
for cond in _not_blacklisted():
|
||||||
|
stmt = stmt.where(cond)
|
||||||
|
return stmt
|
||||||
|
|
||||||
|
|
||||||
|
def _performer_indexable_exists():
|
||||||
|
"""EXISTS: performer ma ≥1 indeksowalną (żywą, nie-blacklisted) scenę.
|
||||||
|
|
||||||
|
Trzyma sitemap performerów w zgodzie z tym, co realnie renderuje
|
||||||
|
`performer_page` — inaczej sitemap zgłaszałby URL-e dające 404.
|
||||||
|
"""
|
||||||
|
sub = (
|
||||||
|
select(1)
|
||||||
|
.select_from(ScenePerformer)
|
||||||
|
.join(Scene, Scene.id == ScenePerformer.scene_id)
|
||||||
|
.where(ScenePerformer.performer_id == Performer.id)
|
||||||
|
.where(_live_playback_exists())
|
||||||
|
)
|
||||||
|
for cond in _not_blacklisted():
|
||||||
|
sub = sub.where(cond)
|
||||||
|
return exists(sub)
|
||||||
|
|
||||||
|
|
||||||
|
def _scene_card_rows(session: Session, scene_ids: list[uuid.UUID]) -> dict[uuid.UUID, dict]:
|
||||||
|
"""Batch: dla listy scen zbierz dane do karty (studio name/slug, #źródeł, thumb)."""
|
||||||
|
if not scene_ids:
|
||||||
|
return {}
|
||||||
|
out: dict[uuid.UUID, dict] = {sid: {"sources": 0, "thumb": None} for sid in scene_ids}
|
||||||
|
|
||||||
|
# liczba żywych źródeł + pierwszy thumbnail (z page_url do proxy referer)
|
||||||
|
pb_rows = session.execute(
|
||||||
|
select(PlaybackSource.scene_id, PlaybackSource.thumbnail_url, PlaybackSource.page_url)
|
||||||
|
.where(
|
||||||
|
PlaybackSource.scene_id.in_(scene_ids),
|
||||||
|
PlaybackSource.dead_at.is_(None),
|
||||||
|
)
|
||||||
|
).all()
|
||||||
|
for sid, thumb, page_url in pb_rows:
|
||||||
|
out[sid]["sources"] += 1
|
||||||
|
if out[sid]["thumb"] is None and thumb:
|
||||||
|
if _needs_proxy(thumb):
|
||||||
|
thumb = _wrap_image_proxy(thumb, page_url)
|
||||||
|
out[sid]["thumb"] = thumb
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _iso_duration(seconds: int | None) -> str | None:
|
||||||
|
"""sekundy → ISO-8601 (PT#M#S) dla schema.org VideoObject.duration."""
|
||||||
|
if not seconds or seconds <= 0:
|
||||||
|
return None
|
||||||
|
m, s = divmod(int(seconds), 60)
|
||||||
|
return f"PT{m}M{s}S"
|
||||||
|
|
||||||
|
|
||||||
|
# --- strony encji -----------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/p/{slug}", response_class=HTMLResponse)
|
||||||
|
def performer_page(
|
||||||
|
slug: str,
|
||||||
|
request: Request,
|
||||||
|
session: Annotated[Session, Depends(get_session)],
|
||||||
|
) -> HTMLResponse:
|
||||||
|
performer = session.execute(
|
||||||
|
select(Performer).where(Performer.slug == slug)
|
||||||
|
).scalar_one_or_none()
|
||||||
|
if performer is None:
|
||||||
|
raise HTTPException(status_code=404, detail="performer not found")
|
||||||
|
|
||||||
|
# Sceny tej osoby — żywe, nie-blacklisted, najświeższe pierwsze.
|
||||||
|
scenes = (
|
||||||
|
session.execute(
|
||||||
|
_indexable_scenes()
|
||||||
|
.where(
|
||||||
|
exists(
|
||||||
|
select(1).where(
|
||||||
|
ScenePerformer.scene_id == Scene.id,
|
||||||
|
ScenePerformer.performer_id == performer.id,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
.order_by(Scene.release_date.desc().nullslast(), Scene.created_at.desc())
|
||||||
|
.limit(_SCENES_PER_ENTITY)
|
||||||
|
)
|
||||||
|
.scalars()
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
if not scenes:
|
||||||
|
# Pusta encja → 404 zamiast thin doorway.
|
||||||
|
raise HTTPException(status_code=404, detail="no indexable scenes for performer")
|
||||||
|
|
||||||
|
cards = _scene_card_rows(session, [s.id for s in scenes])
|
||||||
|
studios = {
|
||||||
|
st.id: st
|
||||||
|
for st in session.execute(
|
||||||
|
select(Studio).where(
|
||||||
|
Studio.id.in_({s.studio_id for s in scenes if s.studio_id})
|
||||||
|
)
|
||||||
|
).scalars()
|
||||||
|
}
|
||||||
|
aliases = [
|
||||||
|
a.alias
|
||||||
|
for a in session.execute(
|
||||||
|
select(PerformerAlias).where(PerformerAlias.performer_id == performer.id)
|
||||||
|
).scalars()
|
||||||
|
]
|
||||||
|
|
||||||
|
return templates.TemplateResponse(
|
||||||
|
request,
|
||||||
|
"seo/performer.html",
|
||||||
|
{
|
||||||
|
"base_url": base_url(),
|
||||||
|
"performer": performer,
|
||||||
|
"aliases": sorted({a for a in aliases if a.lower() != performer.canonical_name.lower()}),
|
||||||
|
"scenes": scenes,
|
||||||
|
"cards": cards,
|
||||||
|
"studios": studios,
|
||||||
|
"canonical": f"{base_url()}/p/{performer.slug}",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/studio/{slug}", response_class=HTMLResponse)
|
||||||
|
def studio_page(
|
||||||
|
slug: str,
|
||||||
|
request: Request,
|
||||||
|
session: Annotated[Session, Depends(get_session)],
|
||||||
|
) -> HTMLResponse:
|
||||||
|
studio = session.execute(
|
||||||
|
select(Studio).where(Studio.slug == slug)
|
||||||
|
).scalar_one_or_none()
|
||||||
|
if studio is None:
|
||||||
|
raise HTTPException(status_code=404, detail="studio not found")
|
||||||
|
|
||||||
|
scenes = (
|
||||||
|
session.execute(
|
||||||
|
_indexable_scenes()
|
||||||
|
.where(Scene.studio_id == studio.id)
|
||||||
|
.order_by(Scene.release_date.desc().nullslast(), Scene.created_at.desc())
|
||||||
|
.limit(_SCENES_PER_ENTITY)
|
||||||
|
)
|
||||||
|
.scalars()
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
if not scenes:
|
||||||
|
raise HTTPException(status_code=404, detail="no indexable scenes for studio")
|
||||||
|
|
||||||
|
cards = _scene_card_rows(session, [s.id for s in scenes])
|
||||||
|
|
||||||
|
return templates.TemplateResponse(
|
||||||
|
request,
|
||||||
|
"seo/studio.html",
|
||||||
|
{
|
||||||
|
"base_url": base_url(),
|
||||||
|
"studio": studio,
|
||||||
|
"scenes": scenes,
|
||||||
|
"cards": cards,
|
||||||
|
"canonical": f"{base_url()}/studio/{studio.slug}",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/scene/{scene_id}", response_class=HTMLResponse)
|
||||||
|
def scene_page(
|
||||||
|
scene_id: uuid.UUID,
|
||||||
|
request: Request,
|
||||||
|
session: Annotated[Session, Depends(get_session)],
|
||||||
|
) -> HTMLResponse:
|
||||||
|
scene = session.get(Scene, scene_id)
|
||||||
|
if scene is None:
|
||||||
|
raise HTTPException(status_code=404, detail="scene not found")
|
||||||
|
|
||||||
|
# Źródła — żywe, deduplikowane po origin (pokazujemy 1 link per tube).
|
||||||
|
sources_raw = (
|
||||||
|
session.execute(
|
||||||
|
select(PlaybackSource)
|
||||||
|
.where(
|
||||||
|
PlaybackSource.scene_id == scene.id,
|
||||||
|
PlaybackSource.dead_at.is_(None),
|
||||||
|
)
|
||||||
|
.order_by(PlaybackSource.origin.asc())
|
||||||
|
)
|
||||||
|
.scalars()
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
if not sources_raw:
|
||||||
|
raise HTTPException(status_code=404, detail="scene has no live sources")
|
||||||
|
|
||||||
|
seen_origins: set[str] = set()
|
||||||
|
sources = []
|
||||||
|
thumb: str | None = None
|
||||||
|
for s in sources_raw:
|
||||||
|
if thumb is None and s.thumbnail_url:
|
||||||
|
thumb = _wrap_image_proxy(s.thumbnail_url, s.page_url) if _needs_proxy(s.thumbnail_url) else s.thumbnail_url
|
||||||
|
if s.origin in seen_origins:
|
||||||
|
continue
|
||||||
|
seen_origins.add(s.origin)
|
||||||
|
label = s.origin.split(":", 1)[1] if ":" in s.origin else s.origin
|
||||||
|
sources.append({"label": label, "page_url": s.page_url, "quality": s.quality})
|
||||||
|
|
||||||
|
studio = session.get(Studio, scene.studio_id) if scene.studio_id else None
|
||||||
|
performers = (
|
||||||
|
session.execute(
|
||||||
|
select(Performer)
|
||||||
|
.join(ScenePerformer, ScenePerformer.performer_id == Performer.id)
|
||||||
|
.where(ScenePerformer.scene_id == scene.id)
|
||||||
|
.order_by(ScenePerformer.position.asc().nullslast())
|
||||||
|
)
|
||||||
|
.scalars()
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
tags = (
|
||||||
|
session.execute(
|
||||||
|
select(Tag)
|
||||||
|
.join(SceneTag, SceneTag.tag_id == Tag.id)
|
||||||
|
.where(SceneTag.scene_id == scene.id)
|
||||||
|
)
|
||||||
|
.scalars()
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
|
||||||
|
return templates.TemplateResponse(
|
||||||
|
request,
|
||||||
|
"seo/scene.html",
|
||||||
|
{
|
||||||
|
"base_url": base_url(),
|
||||||
|
"scene": scene,
|
||||||
|
"studio": studio,
|
||||||
|
"performers": performers,
|
||||||
|
"tags": tags,
|
||||||
|
"sources": sources,
|
||||||
|
"thumb": thumb,
|
||||||
|
"iso_duration": _iso_duration(scene.duration_sec),
|
||||||
|
"canonical": f"{base_url()}/scene/{scene.id}",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/", response_class=HTMLResponse)
|
||||||
|
def landing(
|
||||||
|
request: Request,
|
||||||
|
session: Annotated[Session, Depends(get_session)],
|
||||||
|
) -> HTMLResponse:
|
||||||
|
"""Strona główna — crawl-entry. Najświeższe indeksowalne sceny + CTA."""
|
||||||
|
scenes = (
|
||||||
|
session.execute(
|
||||||
|
_indexable_scenes()
|
||||||
|
.order_by(Scene.created_at.desc())
|
||||||
|
.limit(48)
|
||||||
|
)
|
||||||
|
.scalars()
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
cards = _scene_card_rows(session, [s.id for s in scenes])
|
||||||
|
studios = {
|
||||||
|
st.id: st
|
||||||
|
for st in session.execute(
|
||||||
|
select(Studio).where(Studio.id.in_({s.studio_id for s in scenes if s.studio_id}))
|
||||||
|
).scalars()
|
||||||
|
}
|
||||||
|
return templates.TemplateResponse(
|
||||||
|
request,
|
||||||
|
"seo/landing.html",
|
||||||
|
{
|
||||||
|
"base_url": base_url(),
|
||||||
|
"scenes": scenes,
|
||||||
|
"cards": cards,
|
||||||
|
"studios": studios,
|
||||||
|
"canonical": f"{base_url()}/",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/get", response_class=HTMLResponse)
|
||||||
|
def get_app(request: Request) -> HTMLResponse:
|
||||||
|
"""Paid-traffic landing page — odchudzona pod konwersję instalacji APK.
|
||||||
|
noindex (patrz get.html) — nie ma konkurować w SERP z entity-stronami."""
|
||||||
|
return templates.TemplateResponse(
|
||||||
|
request,
|
||||||
|
"seo/get.html",
|
||||||
|
{"base_url": base_url(), "canonical": f"{base_url()}/get"},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/2257", response_class=HTMLResponse)
|
||||||
|
def page_2257(request: Request) -> HTMLResponse:
|
||||||
|
return templates.TemplateResponse(
|
||||||
|
request,
|
||||||
|
"seo/page_2257.html",
|
||||||
|
{"base_url": base_url(), "canonical": f"{base_url()}/2257"},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# --- robots + sitemap -------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/{fname}.html", response_class=PlainTextResponse, include_in_schema=False)
|
||||||
|
def gsc_site_verification(fname: str) -> PlainTextResponse:
|
||||||
|
"""Plik weryfikacyjny Google Search Console (metoda 'Plik HTML').
|
||||||
|
|
||||||
|
Token z env GOON_GSC_FILE (np. 'google67b49088b5416adc' — bez '.html'). Google
|
||||||
|
pobiera /<token>.html i oczekuje body 'google-site-verification: <token>.html'.
|
||||||
|
Każda inna nazwa .html → 404. Nie koliduje z innymi trasami (żadna inna nie jest
|
||||||
|
jednosegmentowym *.html w rootcie).
|
||||||
|
"""
|
||||||
|
expected = os.environ.get("GOON_GSC_FILE", "")
|
||||||
|
if expected and fname == expected:
|
||||||
|
return PlainTextResponse(f"google-site-verification: {fname}.html")
|
||||||
|
raise HTTPException(status_code=404, detail="not found")
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/robots.txt", response_class=PlainTextResponse)
|
||||||
|
def robots() -> PlainTextResponse:
|
||||||
|
body = (
|
||||||
|
"User-agent: *\n"
|
||||||
|
"Allow: /\n"
|
||||||
|
"Disallow: /proxy/\n"
|
||||||
|
"Disallow: /ui/\n"
|
||||||
|
f"Sitemap: {base_url()}/sitemap.xml\n"
|
||||||
|
)
|
||||||
|
return PlainTextResponse(body)
|
||||||
|
|
||||||
|
|
||||||
|
def _count(session: Session, stmt) -> int:
|
||||||
|
return session.execute(select(func.count()).select_from(stmt.subquery())).scalar_one()
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/sitemap.xml")
|
||||||
|
def sitemap_index(session: Annotated[Session, Depends(get_session)]) -> Response:
|
||||||
|
"""Sitemap index — listuje paginowane pod-sitemapy per typ encji."""
|
||||||
|
n_perf = _count(
|
||||||
|
session,
|
||||||
|
select(Performer.id).where(_performer_indexable_exists()),
|
||||||
|
)
|
||||||
|
n_studio = _count(
|
||||||
|
session,
|
||||||
|
select(Studio.id).where(
|
||||||
|
exists(select(1).where(Scene.studio_id == Studio.id).where(_live_playback_exists()))
|
||||||
|
),
|
||||||
|
)
|
||||||
|
n_scene = _count(session, _indexable_scenes().with_only_columns(Scene.id))
|
||||||
|
|
||||||
|
bu = base_url()
|
||||||
|
parts = ['<?xml version="1.0" encoding="UTF-8"?>',
|
||||||
|
'<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">']
|
||||||
|
for kind, total in (("performers", n_perf), ("studios", n_studio), ("scenes", n_scene)):
|
||||||
|
pages = max(1, -(-total // _SITEMAP_PAGE)) # ceil
|
||||||
|
for p in range(pages):
|
||||||
|
parts.append(f"<sitemap><loc>{bu}/sitemap/{kind}-{p}.xml</loc></sitemap>")
|
||||||
|
parts.append("</sitemapindex>")
|
||||||
|
return Response("\n".join(parts), media_type="application/xml")
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/sitemap/{kind}-{page}.xml")
|
||||||
|
def sitemap_page(
|
||||||
|
kind: str,
|
||||||
|
page: int,
|
||||||
|
session: Annotated[Session, Depends(get_session)],
|
||||||
|
) -> Response:
|
||||||
|
bu = base_url()
|
||||||
|
off = page * _SITEMAP_PAGE
|
||||||
|
urls: list[str] = []
|
||||||
|
|
||||||
|
if kind == "performers":
|
||||||
|
rows = session.execute(
|
||||||
|
select(Performer.slug)
|
||||||
|
.where(_performer_indexable_exists())
|
||||||
|
.order_by(Performer.created_at.asc())
|
||||||
|
.offset(off)
|
||||||
|
.limit(_SITEMAP_PAGE)
|
||||||
|
).scalars()
|
||||||
|
urls = [f"{bu}/p/{slug}" for slug in rows]
|
||||||
|
elif kind == "studios":
|
||||||
|
rows = session.execute(
|
||||||
|
select(Studio.slug)
|
||||||
|
.where(
|
||||||
|
exists(select(1).where(Scene.studio_id == Studio.id).where(_live_playback_exists()))
|
||||||
|
)
|
||||||
|
.order_by(Studio.created_at.asc())
|
||||||
|
.offset(off)
|
||||||
|
.limit(_SITEMAP_PAGE)
|
||||||
|
).scalars()
|
||||||
|
urls = [f"{bu}/studio/{slug}" for slug in rows]
|
||||||
|
elif kind == "scenes":
|
||||||
|
rows = session.execute(
|
||||||
|
_indexable_scenes()
|
||||||
|
.with_only_columns(Scene.id)
|
||||||
|
.order_by(Scene.created_at.asc())
|
||||||
|
.offset(off)
|
||||||
|
.limit(_SITEMAP_PAGE)
|
||||||
|
).scalars()
|
||||||
|
urls = [f"{bu}/scene/{sid}" for sid in rows]
|
||||||
|
else:
|
||||||
|
raise HTTPException(status_code=404, detail="unknown sitemap kind")
|
||||||
|
|
||||||
|
parts = ['<?xml version="1.0" encoding="UTF-8"?>',
|
||||||
|
'<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">']
|
||||||
|
parts += [f"<url><loc>{u}</loc></url>" for u in urls]
|
||||||
|
parts.append("</urlset>")
|
||||||
|
return Response("\n".join(parts), media_type="application/xml")
|
||||||
174
app/api/sources.py
Normal file
174
app/api/sources.py
Normal file
|
|
@ -0,0 +1,174 @@
|
||||||
|
"""GET /sources — lista tube źródeł dla feature "Sites" (mobile top-level tab).
|
||||||
|
|
||||||
|
Bug-report 2026-05-24 (ea6f05f9, Scenes screen): user chce wybrać "pages"
|
||||||
|
obok Scenes i Movies — widzieć liście tube'ów i wchodzić w nie żeby zobaczyć
|
||||||
|
najnowsze sceny z konkretnego źródła.
|
||||||
|
|
||||||
|
Endpoint enumeruje distinct `playback_sources.origin` z ŻYWYCH playback_sources
|
||||||
|
(`dead_at IS NULL`), tylko origins zaczynające się od 'tube:' (kanoniczne źródła
|
||||||
|
typu `canonical:tpdb_trailer` są pomijane — to nie są "scrapowane strony" w sensie
|
||||||
|
intencji feature'a).
|
||||||
|
|
||||||
|
Sortowanie: scene_count DESC (najbardziej "wypełnione" tubey na górze).
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import re
|
||||||
|
from datetime import datetime
|
||||||
|
from typing import Annotated
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Depends
|
||||||
|
from pydantic import BaseModel
|
||||||
|
from sqlalchemy import func, select
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app.auth import require_api_key
|
||||||
|
from app.db import get_session
|
||||||
|
from app.models.playback_source import PlaybackSource
|
||||||
|
from app.models.source_stats import SourceStats
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/sources", tags=["sources"], dependencies=[Depends(require_api_key)])
|
||||||
|
|
||||||
|
|
||||||
|
class SourceRating(BaseModel):
|
||||||
|
"""Ocena źródła do rankingu na Sites screen. Pola = osie z user-requesta."""
|
||||||
|
stars: int
|
||||||
|
"""Ogólna ocena 0-5 (0 = offline). Główny sygnał do sortowania/wyświetlenia."""
|
||||||
|
freshness: int
|
||||||
|
"""0-5: jak często wpada nowy content (wiek najnowszej + wolumen 7d)."""
|
||||||
|
richness: int
|
||||||
|
"""0-5: bogactwo metadanych (miniaturka/tagi/desc/aktorzy/studio/długość)."""
|
||||||
|
health: int | None
|
||||||
|
"""0-5: czy realnie gra (telemetria odtwarzania) — 0=offline. None gdy brak danych."""
|
||||||
|
health_basis: str | None = None
|
||||||
|
"""'telemetry' (realne pingi z apki) albo 'proxy' (oszacowanie z typu resolve)."""
|
||||||
|
components: dict | None = None
|
||||||
|
"""Surowe składowe do rozkładu w UI (% per pole, success-rate, ttff)."""
|
||||||
|
|
||||||
|
|
||||||
|
class SourceOut(BaseModel):
|
||||||
|
origin: str
|
||||||
|
"""Raw origin string z DB — np. 'tube:hqpornercom'. Używany jako parametr
|
||||||
|
`origin=` filtra w GET /scenes (substring match)."""
|
||||||
|
|
||||||
|
sitetag: str
|
||||||
|
"""Origin bez prefiksu 'tube:' — np. 'hqpornercom'. Stabilne ID tube'a (zgodne
|
||||||
|
z `BaseDirectTubeScraper.sitetag`)."""
|
||||||
|
|
||||||
|
display_name: str
|
||||||
|
"""Czytelna nazwa do UI — np. 'hqporner.com'. Wyprowadzona z sitetag przez
|
||||||
|
`_sitetag_to_display`. Tylko presentation; logikę trzymamy na sitetag/origin."""
|
||||||
|
|
||||||
|
scene_count: int
|
||||||
|
"""Liczba ŻYWYCH playback_sources (dead_at IS NULL) per origin. Approx scenes
|
||||||
|
coverage — scena może mieć wiele sources tego samego origin (różne page_url),
|
||||||
|
więc trochę zawyża rzeczywistą scene-distinct count, ale dla orientacji OK."""
|
||||||
|
|
||||||
|
last_scraped_at: datetime | None
|
||||||
|
"""MAX(last_seen_at) — najświeższy scrape dla tego origin. Pozwala mobile pokazać
|
||||||
|
'scrapowane Xh temu' i sortować świeżość."""
|
||||||
|
|
||||||
|
rating: SourceRating | None = None
|
||||||
|
"""Ocena 0-5★ (freshness/richness/health) z source_stats — None gdy jeszcze
|
||||||
|
nie policzona (job source-stats leci co kilka h)."""
|
||||||
|
|
||||||
|
|
||||||
|
class SourceListOut(BaseModel):
|
||||||
|
items: list[SourceOut]
|
||||||
|
total: int
|
||||||
|
|
||||||
|
|
||||||
|
# Hardcoded display-name overrides dla edge cases. Większość sitetags mapuje się
|
||||||
|
# czysto `_sitetag_to_display` regex'em (`hqpornercom` → `hqporner.com`), ale niektóre
|
||||||
|
# tubey mają nietypowe TLDs / brakujące kropki w sitetag.
|
||||||
|
_DISPLAY_OVERRIDES: dict[str, str] = {
|
||||||
|
"fpoxxx": "fpo.xxx",
|
||||||
|
"siskavideo": "siska.video",
|
||||||
|
"porn4dayspw": "porn4days.pw",
|
||||||
|
"porn00org": "porn00.org",
|
||||||
|
"freshpornoorg": "freshporno.org",
|
||||||
|
"pornxpph": "pornxp.ph",
|
||||||
|
"shyfapnet": "shyfap.net",
|
||||||
|
"hdporngg": "hdporn.gg",
|
||||||
|
"fullmoviesxxx": "fullmovies.xxx",
|
||||||
|
"latestleaksco": "latestleaks.co",
|
||||||
|
"xxxfreewatch": "xxxfreewatch.com",
|
||||||
|
"watchporn": "watchporn.to",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
_TLD_RE = re.compile(r"^(.+?)(com|org|net|info)$")
|
||||||
|
|
||||||
|
|
||||||
|
def _sitetag_to_display(sitetag: str) -> str:
|
||||||
|
"""`hqpornercom` → `hqporner.com`. Fallback dla mainstream tube'ów."""
|
||||||
|
if sitetag in _DISPLAY_OVERRIDES:
|
||||||
|
return _DISPLAY_OVERRIDES[sitetag]
|
||||||
|
m = _TLD_RE.match(sitetag)
|
||||||
|
if m:
|
||||||
|
return f"{m.group(1)}.{m.group(2)}"
|
||||||
|
return sitetag
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("", response_model=SourceListOut)
|
||||||
|
def list_sources(
|
||||||
|
session: Annotated[Session, Depends(get_session)],
|
||||||
|
) -> SourceListOut:
|
||||||
|
"""Zwraca listę tube źródeł z ŻYWYMI playback_sources.
|
||||||
|
|
||||||
|
Filter: `origin LIKE 'tube:%'` (drop canonical:* — TPDB trailery to inna semantyka).
|
||||||
|
"""
|
||||||
|
rows = session.execute(
|
||||||
|
select(
|
||||||
|
PlaybackSource.origin,
|
||||||
|
func.count(PlaybackSource.id).label("scene_count"),
|
||||||
|
func.max(PlaybackSource.last_seen_at).label("last_scraped_at"),
|
||||||
|
)
|
||||||
|
.where(PlaybackSource.dead_at.is_(None))
|
||||||
|
.where(PlaybackSource.origin.like("tube:%"))
|
||||||
|
.group_by(PlaybackSource.origin)
|
||||||
|
).all()
|
||||||
|
|
||||||
|
# Oceny z source_stats (policzone offline przez run_source_stats). Origin → row.
|
||||||
|
stats = {s.origin: s for s in session.execute(select(SourceStats)).scalars().all()}
|
||||||
|
|
||||||
|
items: list[SourceOut] = []
|
||||||
|
for origin, scene_count, last_scraped_at in rows:
|
||||||
|
sitetag = origin.split(":", 1)[1] if origin.startswith("tube:") else origin
|
||||||
|
st = stats.get(origin)
|
||||||
|
rating = (
|
||||||
|
SourceRating(
|
||||||
|
stars=st.stars,
|
||||||
|
freshness=st.freshness,
|
||||||
|
richness=st.richness,
|
||||||
|
health=st.health,
|
||||||
|
health_basis=(st.components or {}).get("health_basis"),
|
||||||
|
components=st.components,
|
||||||
|
)
|
||||||
|
if st is not None
|
||||||
|
else None
|
||||||
|
)
|
||||||
|
items.append(
|
||||||
|
SourceOut(
|
||||||
|
origin=origin,
|
||||||
|
sitetag=sitetag,
|
||||||
|
display_name=_sitetag_to_display(sitetag),
|
||||||
|
scene_count=scene_count,
|
||||||
|
last_scraped_at=last_scraped_at,
|
||||||
|
rating=rating,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
# Sort: najpierw ocena (stars desc, źródła bez oceny na końcu), potem rozmiar.
|
||||||
|
items.sort(
|
||||||
|
key=lambda it: (
|
||||||
|
it.rating.stars if it.rating else -1,
|
||||||
|
it.scene_count,
|
||||||
|
),
|
||||||
|
reverse=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
return SourceListOut(items=items, total=len(items))
|
||||||
|
|
@ -86,6 +86,53 @@ DEFAULT_UA = (
|
||||||
"(KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36"
|
"(KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36"
|
||||||
)
|
)
|
||||||
TOKEN_TTL_SEC = 4 * 60 * 60 # 4h
|
TOKEN_TTL_SEC = 4 * 60 * 60 # 4h
|
||||||
|
|
||||||
|
|
||||||
|
# URL-level redirect cache: target_url -> (final_resolved_url, expires_ts).
|
||||||
|
# Mobile ExoPlayer robi range-requesty per seek/preload — każdy hituje proxy z tym
|
||||||
|
# samym tokenem, proxy GET-uje target_url. Dla `porntrex.com/get_file/...` (a także
|
||||||
|
# fpoxxx, freshporno) URL jest **single-use**: pierwszy GET → 302 → CDN URL (time-bound),
|
||||||
|
# drugi GET → 410. Bez cache: drugi range = 410 → ExoPlayer fail → mobile fallback do
|
||||||
|
# `Linking.openURL(page_url)` → reklama (bug-reports `cee51c76`, `e2e365e3` 2026-05-22).
|
||||||
|
#
|
||||||
|
# Z cache: pierwszy GET follow-uje redirect, cache'uje final URL. Kolejne range hituje
|
||||||
|
# direct w CDN URL który jest time-bound (~1-2h), nie single-use. Mobile gra do końca
|
||||||
|
# bez fallbacku.
|
||||||
|
#
|
||||||
|
# TTL 1800s = 30 min: krócej niż typowy CDN signed-URL lifetime (~1h+), więc stale
|
||||||
|
# entries nie powodują 403 spam. Mobile po expiry retry-uje /resolve → fresh token.
|
||||||
|
_REDIRECT_CACHE: dict[str, tuple[str, float]] = {}
|
||||||
|
_REDIRECT_CACHE_TTL_SEC = 1800
|
||||||
|
_REDIRECT_CACHE_MAX = 1000
|
||||||
|
|
||||||
|
|
||||||
|
def _redirect_cache_get(target_url: str) -> str | None:
|
||||||
|
entry = _REDIRECT_CACHE.get(target_url)
|
||||||
|
if not entry:
|
||||||
|
return None
|
||||||
|
final, exp = entry
|
||||||
|
if exp < time.time():
|
||||||
|
_REDIRECT_CACHE.pop(target_url, None)
|
||||||
|
return None
|
||||||
|
return final
|
||||||
|
|
||||||
|
|
||||||
|
def _redirect_cache_put(target_url: str, final_url: str) -> None:
|
||||||
|
if not final_url or target_url == final_url:
|
||||||
|
return
|
||||||
|
_REDIRECT_CACHE[target_url] = (final_url, time.time() + _REDIRECT_CACHE_TTL_SEC)
|
||||||
|
if len(_REDIRECT_CACHE) > _REDIRECT_CACHE_MAX:
|
||||||
|
cutoff = time.time()
|
||||||
|
for k in list(_REDIRECT_CACHE.keys()):
|
||||||
|
v = _REDIRECT_CACHE.get(k)
|
||||||
|
if v is None or v[1] < cutoff:
|
||||||
|
_REDIRECT_CACHE.pop(k, None)
|
||||||
|
|
||||||
|
|
||||||
|
def _redirect_cache_invalidate(target_url: str) -> None:
|
||||||
|
_REDIRECT_CACHE.pop(target_url, None)
|
||||||
|
|
||||||
|
|
||||||
HOP_BY_HOP = {
|
HOP_BY_HOP = {
|
||||||
"connection",
|
"connection",
|
||||||
"keep-alive",
|
"keep-alive",
|
||||||
|
|
@ -115,6 +162,7 @@ def make_token(
|
||||||
refresh: str | None = None,
|
refresh: str | None = None,
|
||||||
refresh_hoster: str | None = None,
|
refresh_hoster: str | None = None,
|
||||||
impersonate: bool = False,
|
impersonate: bool = False,
|
||||||
|
stable_bucket_sec: int | None = None,
|
||||||
) -> str:
|
) -> str:
|
||||||
"""Build proxy token.
|
"""Build proxy token.
|
||||||
|
|
||||||
|
|
@ -124,8 +172,15 @@ def make_token(
|
||||||
dispatch do dedicated re-extract logic.
|
dispatch do dedicated re-extract logic.
|
||||||
`impersonate`: użyć curl_cffi chrome120 zamiast httpx (dla hosterów z JA3 bot
|
`impersonate`: użyć curl_cffi chrome120 zamiast httpx (dla hosterów z JA3 bot
|
||||||
detection — mxcontent, cloudflare-protected).
|
detection — mxcontent, cloudflare-protected).
|
||||||
|
`stable_bucket_sec`: kwantyzuje czas bazowy expiry do okna N sekund, więc token
|
||||||
|
(→ proxied URL) jest IDENTYCZNY dla wszystkich requestów w oknie. Bez tego `e`
|
||||||
|
zawiera bieżący timestamp → URL inny przy każdym fetchu listy → expo-image cache
|
||||||
|
miss → re-download miniatur za każdym razem. Dla thumbów (stabilne) dajemy 7d bucket
|
||||||
|
→ URL stały przez tydzień → disk-cache hit. exp = bucket_start + ttl (zawsze > now).
|
||||||
"""
|
"""
|
||||||
payload: dict = {"u": url, "r": referer or "", "e": int(time.time()) + ttl_sec}
|
now = int(time.time())
|
||||||
|
base = (now // stable_bucket_sec) * stable_bucket_sec if stable_bucket_sec else now
|
||||||
|
payload: dict = {"u": url, "r": referer or "", "e": base + ttl_sec}
|
||||||
if refresh:
|
if refresh:
|
||||||
payload["rf"] = refresh
|
payload["rf"] = refresh
|
||||||
if refresh_hoster:
|
if refresh_hoster:
|
||||||
|
|
@ -222,6 +277,26 @@ def _rewrite_m3u8(content: str, base_url: str, referer: str | None) -> str:
|
||||||
return "\n".join(out) + "\n"
|
return "\n".join(out) + "\n"
|
||||||
|
|
||||||
|
|
||||||
|
def _absolutize_m3u8(content: str, base_url: str) -> str:
|
||||||
|
"""Rewrite m3u8 tak, że child-URL-e (variant/segmenty/key) są ABSOLUTNE na CDN —
|
||||||
|
NIE przez proxy. Dla direct-HLS passthrough (`/proxy/hls/...`): telefon ciągnie je
|
||||||
|
bezpośrednio z CDN-a (phone IP, time-bound token). Relatywne → absolute (urljoin),
|
||||||
|
absolute → bez zmian. Odwrotność `_rewrite_m3u8` (który wszystko proxifikuje)."""
|
||||||
|
out: list[str] = []
|
||||||
|
for raw_line in content.splitlines():
|
||||||
|
line = raw_line.strip()
|
||||||
|
if not line:
|
||||||
|
out.append(raw_line)
|
||||||
|
continue
|
||||||
|
if line.startswith("#"):
|
||||||
|
def _sub(m: re.Match) -> str:
|
||||||
|
return f"{m.group(1)}{urljoin(base_url, m.group(2))}{m.group(3)}"
|
||||||
|
out.append(_M3U8_URI_RE.sub(_sub, raw_line))
|
||||||
|
continue
|
||||||
|
out.append(urljoin(base_url, line))
|
||||||
|
return "\n".join(out) + "\n"
|
||||||
|
|
||||||
|
|
||||||
@router.get("/sign")
|
@router.get("/sign")
|
||||||
def sign_url(
|
def sign_url(
|
||||||
_api: Annotated[None, Depends(require_api_key)],
|
_api: Annotated[None, Depends(require_api_key)],
|
||||||
|
|
@ -280,6 +355,67 @@ async def proxy_image(
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# sxyprn on-demand thumbnail resolve (bug 2026-06-10). trafficdeposit poster token
|
||||||
|
# żyje ~1h (bucket godzinowy), więc URL-i NIE da się przechować — resolvujemy bieżący
|
||||||
|
# og:image ze strony /post/<id> przy serwowaniu. Cache resolved poster URL ~40min
|
||||||
|
# (< 1h TTL). Klient dostaje STABILNY /proxy/sxyprn-thumb/<id> → cache'uje bajty na
|
||||||
|
# stałe (treść postera niezmienna), więc fetchujemy stronę post ~raz per post.
|
||||||
|
_SXYPRN_POSTER_CACHE: dict[str, tuple[str, float]] = {}
|
||||||
|
_SXYPRN_POSTER_TTL = 2400
|
||||||
|
_OG_IMG_RE = re.compile(r"og:image[\"'][^>]*content=[\"']([^\"']+)", re.IGNORECASE)
|
||||||
|
_OG_IMG_RE2 = re.compile(r"content=[\"']([^\"']+)[\"'][^>]*property=[\"']og:image", re.IGNORECASE)
|
||||||
|
_VID_POSTER_RE = re.compile(r"<video[^>]*poster=[\"']([^\"']+)", re.IGNORECASE)
|
||||||
|
_SXYPRN_PID_RE = re.compile(r"^[0-9a-f]{6,40}$")
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/sxyprn-thumb/{post_id}")
|
||||||
|
async def sxyprn_thumb(post_id: str) -> Response:
|
||||||
|
"""On-demand poster sxyprn. URL stabilny per post_id (klient cache'uje bajty);
|
||||||
|
backend resolvuje bieżący og:image (token ~1h) i streamuje z Refererem."""
|
||||||
|
pid = post_id.split(".")[0] # zdejmij ewentualne .jpg
|
||||||
|
if not _SXYPRN_PID_RE.match(pid):
|
||||||
|
raise HTTPException(status_code=400, detail="bad post_id")
|
||||||
|
now = time.time()
|
||||||
|
cached = _SXYPRN_POSTER_CACHE.get(pid)
|
||||||
|
poster = cached[0] if (cached and cached[1] > now) else None
|
||||||
|
timeout = httpx.Timeout(connect=10.0, read=20.0, write=10.0, pool=5.0)
|
||||||
|
async with httpx.AsyncClient(timeout=timeout, follow_redirects=True) as client:
|
||||||
|
if poster is None:
|
||||||
|
try:
|
||||||
|
r = await client.get(
|
||||||
|
f"https://sxyprn.com/post/{pid}.html", headers=_build_headers(None)
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
log.info("sxyprn-thumb page fetch failed %s: %s", pid, e)
|
||||||
|
return Response(content=b"", status_code=502, media_type="image/jpeg")
|
||||||
|
html = r.text
|
||||||
|
if "Post Not Found" in html:
|
||||||
|
return Response(content=b"", status_code=404, media_type="image/jpeg")
|
||||||
|
m = _OG_IMG_RE.search(html) or _OG_IMG_RE2.search(html) or _VID_POSTER_RE.search(html)
|
||||||
|
if not m:
|
||||||
|
return Response(content=b"", status_code=404, media_type="image/jpeg")
|
||||||
|
poster = m.group(1).strip()
|
||||||
|
if poster.startswith("//"):
|
||||||
|
poster = "https:" + poster
|
||||||
|
_SXYPRN_POSTER_CACHE[pid] = (poster, now + _SXYPRN_POSTER_TTL)
|
||||||
|
if len(_SXYPRN_POSTER_CACHE) > 8000:
|
||||||
|
for k in [k for k, v in list(_SXYPRN_POSTER_CACHE.items()) if v[1] < now]:
|
||||||
|
_SXYPRN_POSTER_CACHE.pop(k, None)
|
||||||
|
try:
|
||||||
|
pr = await client.get(poster, headers=_build_headers("https://sxyprn.com/"))
|
||||||
|
except Exception as e:
|
||||||
|
log.info("sxyprn-thumb poster fetch failed %s: %s", pid, e)
|
||||||
|
return Response(content=b"", status_code=502, media_type="image/jpeg")
|
||||||
|
if pr.status_code >= 400:
|
||||||
|
_SXYPRN_POSTER_CACHE.pop(pid, None) # stale token → re-resolve next time
|
||||||
|
return Response(content=b"", status_code=502, media_type="image/jpeg")
|
||||||
|
return Response(
|
||||||
|
content=pr.content,
|
||||||
|
media_type=pr.headers.get("content-type", "image/jpeg"),
|
||||||
|
headers={"Cache-Control": "public, max-age=604800"},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
async def _refetch_mixdrop_url(session: "AsyncSession", embed_url: str) -> str | None:
|
async def _refetch_mixdrop_url(session: "AsyncSession", embed_url: str) -> str | None:
|
||||||
"""Re-fetch mixdrop embed, decode P.A.C.K.E.R., extract fresh MDCore.wurl.
|
"""Re-fetch mixdrop embed, decode P.A.C.K.E.R., extract fresh MDCore.wurl.
|
||||||
Cookies persist w session, użytkowane potem do mp4 GET (same-session bind).
|
Cookies persist w session, użytkowane potem do mp4 GET (same-session bind).
|
||||||
|
|
@ -383,6 +519,42 @@ async def _curl_cffi_stream(
|
||||||
raise HTTPException(status_code=502, detail=f"proxy error: {e}") from e
|
raise HTTPException(status_code=502, detail=f"proxy error: {e}") from e
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/hls/{token}/{_basename:path}")
|
||||||
|
async def proxy_hls_manifest(token: str, _basename: str) -> Response:
|
||||||
|
"""Direct-HLS manifest passthrough dla time-bound (mobile_direct_ok) m3u8 hosterów.
|
||||||
|
|
||||||
|
Problem: expo-video 2.0.6 nie ma `contentType`, więc ExoPlayer zgaduje typ z URL.
|
||||||
|
Time-bound manifesty których ścieżka NIE kończy się `.m3u8` (pornhat:
|
||||||
|
`.../552351,_360p.mp4,.mp4,_720p.mp4,?...`) → ExoPlayer leci Mp4Extractor →
|
||||||
|
UnrecognizedInputFormat → fallback na pełny `/proxy/` → CAŁE wideo przez VPS
|
||||||
|
(privatehost 466 hitów/48h, audit 2026-06-11).
|
||||||
|
|
||||||
|
Tu serwujemy SAM manifest pod `play.m3u8` (ExoPlayer → HlsMediaSource), a child-URL-e
|
||||||
|
(variant/segmenty) zostają ABSOLUTNE na CDN → telefon ciągnie je DIRECT (phone IP,
|
||||||
|
token time-bound działa cross-IP; Referer propaguje Media3 na child-requesty —
|
||||||
|
zweryfikowane cross-IP 2026-06-11). Przez VPS leci tylko ~1KB manifestu, nie segmenty.
|
||||||
|
Tylko dla mobile_direct_ok (gdzie segmenty z definicji działają cross-IP); IP-bound
|
||||||
|
hostery dalej idą pełnym `/proxy/`."""
|
||||||
|
payload = parse_token(token)
|
||||||
|
target = payload["u"]
|
||||||
|
referer = payload.get("r") or None
|
||||||
|
headers = _build_headers(referer)
|
||||||
|
async with httpx.AsyncClient(follow_redirects=True, timeout=20.0) as client:
|
||||||
|
try:
|
||||||
|
r = await client.get(target, headers=headers)
|
||||||
|
except httpx.HTTPError as e:
|
||||||
|
log.info("proxy-hls fetch failed %s: %s", target, e)
|
||||||
|
raise HTTPException(status_code=502, detail="manifest fetch failed") from e
|
||||||
|
if r.status_code >= 400:
|
||||||
|
return _upstream_error_response(r.status_code, dict(r.headers), target)
|
||||||
|
rewritten = _absolutize_m3u8(r.text, base_url=str(r.url))
|
||||||
|
return Response(
|
||||||
|
content=rewritten,
|
||||||
|
media_type="application/vnd.apple.mpegurl",
|
||||||
|
headers={"Cache-Control": "no-store"},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@router.get("/{token}/{_basename:path}")
|
@router.get("/{token}/{_basename:path}")
|
||||||
async def proxy_stream(
|
async def proxy_stream(
|
||||||
token: str,
|
token: str,
|
||||||
|
|
@ -390,12 +562,17 @@ async def proxy_stream(
|
||||||
request: Request,
|
request: Request,
|
||||||
) -> Response:
|
) -> Response:
|
||||||
payload = parse_token(token)
|
payload = parse_token(token)
|
||||||
target = payload["u"]
|
original_target = payload["u"]
|
||||||
referer = payload["r"] or None
|
referer = payload["r"] or None
|
||||||
use_impersonate = bool(payload.get("i"))
|
use_impersonate = bool(payload.get("i"))
|
||||||
refetch_url = payload.get("rf")
|
refetch_url = payload.get("rf")
|
||||||
refetch_hoster = payload.get("rh")
|
refetch_hoster = payload.get("rh")
|
||||||
|
|
||||||
|
# Jeśli ten target był już wcześniej follow-redirect-ed, użyj cached final URL.
|
||||||
|
# Powód: porntrex `get_file/` 410 po reuse — patrz `_REDIRECT_CACHE` docstring.
|
||||||
|
cached_target = _redirect_cache_get(original_target)
|
||||||
|
target = cached_target or original_target
|
||||||
|
|
||||||
# Forwardujemy Range header (HLS/MP4 player robi byte-range fetches dla seek/preload)
|
# Forwardujemy Range header (HLS/MP4 player robi byte-range fetches dla seek/preload)
|
||||||
headers = _build_headers(referer)
|
headers = _build_headers(referer)
|
||||||
range_h = request.headers.get("range")
|
range_h = request.headers.get("range")
|
||||||
|
|
@ -437,8 +614,21 @@ async def proxy_stream(
|
||||||
ups_headers = dict(upstream.headers)
|
ups_headers = dict(upstream.headers)
|
||||||
await upstream.aclose()
|
await upstream.aclose()
|
||||||
await client.aclose()
|
await client.aclose()
|
||||||
|
# Cached final URL zwrócił error (np. CDN signed-URL expired, 403/410) —
|
||||||
|
# invaliduj cache i daj mobile retry przez fresh /resolve. Bez tego stale
|
||||||
|
# cache trzymałby martwy CDN URL przez 30 min (TTL).
|
||||||
|
if cached_target is not None and status in (401, 403, 404, 410):
|
||||||
|
_redirect_cache_invalidate(original_target)
|
||||||
return _upstream_error_response(status, ups_headers, target)
|
return _upstream_error_response(status, ups_headers, target)
|
||||||
|
|
||||||
|
# Pierwszy successful pass dla single-use targets (np. porntrex get_file):
|
||||||
|
# cache resolved final URL (po follow_redirects). Następne range-requesty
|
||||||
|
# pójdą direct w CDN URL — get_file nie dostaje drugiego hita.
|
||||||
|
if cached_target is None:
|
||||||
|
final_url = str(upstream.url)
|
||||||
|
if final_url != original_target:
|
||||||
|
_redirect_cache_put(original_target, final_url)
|
||||||
|
|
||||||
ct = (upstream.headers.get("content-type") or "").lower()
|
ct = (upstream.headers.get("content-type") or "").lower()
|
||||||
is_m3u8 = (
|
is_m3u8 = (
|
||||||
path_suggests_m3u8
|
path_suggests_m3u8
|
||||||
|
|
@ -472,6 +662,16 @@ async def proxy_stream(
|
||||||
async for chunk in upstream.aiter_raw():
|
async for chunk in upstream.aiter_raw():
|
||||||
bytes_out += len(chunk)
|
bytes_out += len(chunk)
|
||||||
yield chunk
|
yield chunk
|
||||||
|
except httpx.TransportError as e:
|
||||||
|
# Upstream CDN zerwał połączenie w trakcie streamu (peer closed before
|
||||||
|
# full body — RemoteProtocolError, ReadError, timeout). Klient dostaje
|
||||||
|
# uciętą odpowiedź i ponawia Range-requestem; to NIE jest bug aplikacji.
|
||||||
|
# Bez tego wyjątek leci z generatora PO zwróceniu StreamingResponse, omija
|
||||||
|
# outer try/except i ląduje w Sentry jako unhandled (GOON-Y 2026-06-11).
|
||||||
|
log.info(
|
||||||
|
"proxy upstream dropped %s after %d bytes: %s",
|
||||||
|
target, bytes_out, type(e).__name__,
|
||||||
|
)
|
||||||
finally:
|
finally:
|
||||||
await upstream.aclose()
|
await upstream.aclose()
|
||||||
await client.aclose()
|
await client.aclose()
|
||||||
|
|
|
||||||
|
|
@ -108,7 +108,8 @@ def list_tags(
|
||||||
|
|
||||||
if for_movies:
|
if for_movies:
|
||||||
# Movie tag count — zliczamy tylko Movies z ≥1 live MoviePlaybackSource.
|
# Movie tag count — zliczamy tylko Movies z ≥1 live MoviePlaybackSource.
|
||||||
# Tag-bez-żadnego-movie zwraca 0 (LEFT OUTER JOIN przez coalesce).
|
# Tag-bez-żadnego-movie zwraca 0 (LEFT OUTER JOIN przez coalesce). Movies są
|
||||||
|
# małe (~41k) więc live-aggregat OK; scenes path niżej używa denormalizacji.
|
||||||
_movie_live = exists().where(
|
_movie_live = exists().where(
|
||||||
and_(
|
and_(
|
||||||
MoviePlaybackSource.movie_id == MovieTag.movie_id,
|
MoviePlaybackSource.movie_id == MovieTag.movie_id,
|
||||||
|
|
@ -121,21 +122,6 @@ def list_tags(
|
||||||
.group_by(MovieTag.tag_id)
|
.group_by(MovieTag.tag_id)
|
||||||
.subquery()
|
.subquery()
|
||||||
)
|
)
|
||||||
else:
|
|
||||||
# has_live_playback filter — zliczamy tylko sceny które user faktycznie zobaczy
|
|
||||||
# (TPDB/StashDB metadata-only stubs są do mergowania, nie do oglądania).
|
|
||||||
_live_playback = exists().where(
|
|
||||||
and_(
|
|
||||||
PlaybackSource.scene_id == SceneTag.scene_id,
|
|
||||||
PlaybackSource.dead_at.is_(None),
|
|
||||||
)
|
|
||||||
)
|
|
||||||
count_sub = (
|
|
||||||
select(SceneTag.tag_id, func.count(SceneTag.scene_id).label("c"))
|
|
||||||
.where(_live_playback)
|
|
||||||
.group_by(SceneTag.tag_id)
|
|
||||||
.subquery()
|
|
||||||
)
|
|
||||||
base = (
|
base = (
|
||||||
select(Tag, func.coalesce(count_sub.c.c, 0).label("scene_count"))
|
select(Tag, func.coalesce(count_sub.c.c, 0).label("scene_count"))
|
||||||
.outerjoin(count_sub, count_sub.c.tag_id == Tag.id)
|
.outerjoin(count_sub, count_sub.c.tag_id == Tag.id)
|
||||||
|
|
@ -143,26 +129,48 @@ def list_tags(
|
||||||
if q:
|
if q:
|
||||||
base = base.where(Tag.name.ilike(f"%{q}%"))
|
base = base.where(Tag.name.ilike(f"%{q}%"))
|
||||||
if only_with_content:
|
if only_with_content:
|
||||||
# exists() w outerjoin nie inner-joinowałby pustych tagów. Dlatego osobny
|
|
||||||
# exists check: pasują tylko tagi z ≥1 w subquery.
|
|
||||||
base = base.where(count_sub.c.tag_id.is_not(None))
|
base = base.where(count_sub.c.tag_id.is_not(None))
|
||||||
|
total = session.execute(
|
||||||
|
select(func.count()).select_from(base.subquery())
|
||||||
|
).scalar_one()
|
||||||
|
if order in ("popular", "scene_count"):
|
||||||
|
ordered = base.order_by(func.coalesce(count_sub.c.c, 0).desc(), Tag.name.asc())
|
||||||
|
else:
|
||||||
|
ordered = base.order_by(Tag.name.asc())
|
||||||
|
rows = session.execute(
|
||||||
|
ordered.offset((page - 1) * per_page).limit(per_page)
|
||||||
|
).all()
|
||||||
|
items = [
|
||||||
|
TagCount(id=t.id, name=t.name, slug=t.slug, scene_count=int(c))
|
||||||
|
for t, c in rows
|
||||||
|
]
|
||||||
|
return TagListOut(items=items, total=total, page=page, per_page=per_page)
|
||||||
|
|
||||||
|
# Scenes path — czyta zdenormalizowany Tag.scene_count (refresh w tle przez
|
||||||
|
# _job_refresh_taxonomy_counts). Wcześniej liczył agregat 6.3M scene_tags +
|
||||||
|
# EXISTS playback per-request (~4.3s ×2). Patrz migracja 0019.
|
||||||
|
base = select(Tag)
|
||||||
|
if q:
|
||||||
|
base = base.where(Tag.name.ilike(f"%{q}%"))
|
||||||
|
if only_with_content:
|
||||||
|
base = base.where(Tag.scene_count > 0)
|
||||||
|
|
||||||
total = session.execute(
|
total = session.execute(
|
||||||
select(func.count()).select_from(base.subquery())
|
select(func.count()).select_from(base.subquery())
|
||||||
).scalar_one()
|
).scalar_one()
|
||||||
|
|
||||||
if order in ("popular", "scene_count"):
|
if order in ("popular", "scene_count"):
|
||||||
ordered = base.order_by(func.coalesce(count_sub.c.c, 0).desc(), Tag.name.asc())
|
ordered = base.order_by(Tag.scene_count.desc(), Tag.name.asc())
|
||||||
else:
|
else:
|
||||||
ordered = base.order_by(Tag.name.asc())
|
ordered = base.order_by(Tag.name.asc())
|
||||||
|
|
||||||
rows = session.execute(
|
tags_page = session.execute(
|
||||||
ordered.offset((page - 1) * per_page).limit(per_page)
|
ordered.offset((page - 1) * per_page).limit(per_page)
|
||||||
).all()
|
).scalars().all()
|
||||||
|
|
||||||
items = [
|
items = [
|
||||||
TagCount(id=t.id, name=t.name, slug=t.slug, scene_count=int(c))
|
TagCount(id=t.id, name=t.name, slug=t.slug, scene_count=t.scene_count)
|
||||||
for t, c in rows
|
for t in tags_page
|
||||||
]
|
]
|
||||||
return TagListOut(items=items, total=total, page=page, per_page=per_page)
|
return TagListOut(items=items, total=total, page=page, per_page=per_page)
|
||||||
|
|
||||||
|
|
@ -178,23 +186,9 @@ def list_performers(
|
||||||
if order not in ("scene_count", "popular", "name"):
|
if order not in ("scene_count", "popular", "name"):
|
||||||
raise HTTPException(status_code=400, detail="order must be 'scene_count' or 'name'")
|
raise HTTPException(status_code=400, detail="order must be 'scene_count' or 'name'")
|
||||||
|
|
||||||
# has_live_playback filter — patrz list_tags wyżej.
|
# Czyta zdenormalizowany Performer.scene_count (refresh w tle). Wcześniej agregat
|
||||||
_perf_live_playback = exists().where(
|
# 3M scene_performers + EXISTS playback per-request. Patrz migracja 0019.
|
||||||
and_(
|
base = select(Performer)
|
||||||
PlaybackSource.scene_id == ScenePerformer.scene_id,
|
|
||||||
PlaybackSource.dead_at.is_(None),
|
|
||||||
)
|
|
||||||
)
|
|
||||||
count_sub = (
|
|
||||||
select(ScenePerformer.performer_id, func.count(ScenePerformer.scene_id).label("c"))
|
|
||||||
.where(_perf_live_playback)
|
|
||||||
.group_by(ScenePerformer.performer_id)
|
|
||||||
.subquery()
|
|
||||||
)
|
|
||||||
base = (
|
|
||||||
select(Performer, func.coalesce(count_sub.c.c, 0).label("scene_count"))
|
|
||||||
.outerjoin(count_sub, count_sub.c.performer_id == Performer.id)
|
|
||||||
)
|
|
||||||
if q:
|
if q:
|
||||||
base = base.where(Performer.name_normalized.ilike(f"%{q.lower()}%"))
|
base = base.where(Performer.name_normalized.ilike(f"%{q.lower()}%"))
|
||||||
|
|
||||||
|
|
@ -204,14 +198,14 @@ def list_performers(
|
||||||
|
|
||||||
if order in ("scene_count", "popular"):
|
if order in ("scene_count", "popular"):
|
||||||
ordered = base.order_by(
|
ordered = base.order_by(
|
||||||
func.coalesce(count_sub.c.c, 0).desc(), Performer.canonical_name.asc()
|
Performer.scene_count.desc(), Performer.canonical_name.asc()
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
ordered = base.order_by(Performer.canonical_name.asc())
|
ordered = base.order_by(Performer.canonical_name.asc())
|
||||||
|
|
||||||
rows = session.execute(
|
perfs_page = session.execute(
|
||||||
ordered.offset((page - 1) * per_page).limit(per_page)
|
ordered.offset((page - 1) * per_page).limit(per_page)
|
||||||
).all()
|
).scalars().all()
|
||||||
|
|
||||||
items = [
|
items = [
|
||||||
PerformerCount(
|
PerformerCount(
|
||||||
|
|
@ -219,9 +213,9 @@ def list_performers(
|
||||||
canonical_name=p.canonical_name,
|
canonical_name=p.canonical_name,
|
||||||
slug=p.slug,
|
slug=p.slug,
|
||||||
gender=p.gender.value if p.gender else None,
|
gender=p.gender.value if p.gender else None,
|
||||||
scene_count=int(c),
|
scene_count=p.scene_count,
|
||||||
)
|
)
|
||||||
for p, c in rows
|
for p in perfs_page
|
||||||
]
|
]
|
||||||
return PerformerListOut(items=items, total=total, page=page, per_page=per_page)
|
return PerformerListOut(items=items, total=total, page=page, per_page=per_page)
|
||||||
|
|
||||||
|
|
@ -248,6 +242,7 @@ def list_studios(
|
||||||
raise HTTPException(status_code=400, detail="order must be 'name' or 'scene_count'")
|
raise HTTPException(status_code=400, detail="order must be 'name' or 'scene_count'")
|
||||||
|
|
||||||
if for_movies:
|
if for_movies:
|
||||||
|
# Movies małe (~41k) — live aggregat OK. Scenes path niżej = denormalizacja.
|
||||||
_movie_live = exists().where(
|
_movie_live = exists().where(
|
||||||
and_(
|
and_(
|
||||||
MoviePlaybackSource.movie_id == Movie.id,
|
MoviePlaybackSource.movie_id == Movie.id,
|
||||||
|
|
@ -261,21 +256,6 @@ def list_studios(
|
||||||
.group_by(Movie.studio_id)
|
.group_by(Movie.studio_id)
|
||||||
.subquery()
|
.subquery()
|
||||||
)
|
)
|
||||||
else:
|
|
||||||
# has_live_playback filter — patrz list_tags wyżej.
|
|
||||||
_studio_live_playback = exists().where(
|
|
||||||
and_(
|
|
||||||
PlaybackSource.scene_id == Scene.id,
|
|
||||||
PlaybackSource.dead_at.is_(None),
|
|
||||||
)
|
|
||||||
)
|
|
||||||
count_sub = (
|
|
||||||
select(Scene.studio_id, func.count(Scene.id).label("c"))
|
|
||||||
.where(Scene.studio_id.is_not(None))
|
|
||||||
.where(_studio_live_playback)
|
|
||||||
.group_by(Scene.studio_id)
|
|
||||||
.subquery()
|
|
||||||
)
|
|
||||||
base = (
|
base = (
|
||||||
select(Studio, func.coalesce(count_sub.c.c, 0).label("scene_count"))
|
select(Studio, func.coalesce(count_sub.c.c, 0).label("scene_count"))
|
||||||
.outerjoin(count_sub, count_sub.c.studio_id == Studio.id)
|
.outerjoin(count_sub, count_sub.c.studio_id == Studio.id)
|
||||||
|
|
@ -284,19 +264,42 @@ def list_studios(
|
||||||
base = base.where(Studio.name.ilike(f"%{q}%"))
|
base = base.where(Studio.name.ilike(f"%{q}%"))
|
||||||
if only_with_content:
|
if only_with_content:
|
||||||
base = base.where(count_sub.c.studio_id.is_not(None))
|
base = base.where(count_sub.c.studio_id.is_not(None))
|
||||||
|
total = session.execute(
|
||||||
|
select(func.count()).select_from(base.subquery())
|
||||||
|
).scalar_one()
|
||||||
|
if order in ("scene_count", "popular"):
|
||||||
|
ordered = base.order_by(func.coalesce(count_sub.c.c, 0).desc(), Studio.name.asc())
|
||||||
|
else:
|
||||||
|
ordered = base.order_by(Studio.name_normalized.asc())
|
||||||
|
rows = session.execute(
|
||||||
|
ordered.offset((page - 1) * per_page).limit(per_page)
|
||||||
|
).all()
|
||||||
|
items = [
|
||||||
|
StudioCount(id=s.id, name=s.name, slug=s.slug, network=s.network, scene_count=int(c))
|
||||||
|
for s, c in rows
|
||||||
|
]
|
||||||
|
return StudioListOut(items=items, total=total, page=page, per_page=per_page)
|
||||||
|
|
||||||
|
# Scenes path — czyta zdenormalizowany Studio.scene_count (refresh w tle).
|
||||||
|
# Wcześniej agregat 1.69M scenes + EXISTS playback per-request. Patrz migracja 0019.
|
||||||
|
base = select(Studio)
|
||||||
|
if q:
|
||||||
|
base = base.where(Studio.name.ilike(f"%{q}%"))
|
||||||
|
if only_with_content:
|
||||||
|
base = base.where(Studio.scene_count > 0)
|
||||||
|
|
||||||
total = session.execute(
|
total = session.execute(
|
||||||
select(func.count()).select_from(base.subquery())
|
select(func.count()).select_from(base.subquery())
|
||||||
).scalar_one()
|
).scalar_one()
|
||||||
|
|
||||||
if order in ("scene_count", "popular"):
|
if order in ("scene_count", "popular"):
|
||||||
ordered = base.order_by(func.coalesce(count_sub.c.c, 0).desc(), Studio.name.asc())
|
ordered = base.order_by(Studio.scene_count.desc(), Studio.name.asc())
|
||||||
else:
|
else:
|
||||||
ordered = base.order_by(Studio.name_normalized.asc())
|
ordered = base.order_by(Studio.name_normalized.asc())
|
||||||
|
|
||||||
rows = session.execute(
|
studios_page = session.execute(
|
||||||
ordered.offset((page - 1) * per_page).limit(per_page)
|
ordered.offset((page - 1) * per_page).limit(per_page)
|
||||||
).all()
|
).scalars().all()
|
||||||
|
|
||||||
items = [
|
items = [
|
||||||
StudioCount(
|
StudioCount(
|
||||||
|
|
@ -304,9 +307,9 @@ def list_studios(
|
||||||
name=s.name,
|
name=s.name,
|
||||||
slug=s.slug,
|
slug=s.slug,
|
||||||
network=s.network,
|
network=s.network,
|
||||||
scene_count=int(c),
|
scene_count=s.scene_count,
|
||||||
)
|
)
|
||||||
for s, c in rows
|
for s in studios_page
|
||||||
]
|
]
|
||||||
return StudioListOut(items=items, total=total, page=page, per_page=per_page)
|
return StudioListOut(items=items, total=total, page=page, per_page=per_page)
|
||||||
|
|
||||||
|
|
@ -346,10 +349,42 @@ _TAG_RESCRAPE_THRESHOLD = 3
|
||||||
# Mainstream tubes priority dla tagów — bogate metadane.
|
# Mainstream tubes priority dla tagów — bogate metadane.
|
||||||
_TAG_PRIORITY = [
|
_TAG_PRIORITY = [
|
||||||
"xhamstercom", "porntrexcom", "epornercom", "youporncom",
|
"xhamstercom", "porntrexcom", "epornercom", "youporncom",
|
||||||
"xvideoscom", "xnxxcom", "redtubecom", "pornhatcom",
|
"xvideoscom", "xnxxcom",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/performers/{performer_id}/tags", response_model=TagListOut)
|
||||||
|
def performer_top_tags(
|
||||||
|
performer_id: uuid.UUID,
|
||||||
|
session: Annotated[Session, Depends(get_session)],
|
||||||
|
limit: int = Query(default=20, ge=1, le=50),
|
||||||
|
) -> TagListOut:
|
||||||
|
"""Top tagi/kategorie agregowane ze scen performera (PerformerScenes UI chips,
|
||||||
|
bug-report 1a4bf258 — zastępują dev-only przycisk Re-scrape). Liczymy tylko po
|
||||||
|
scenach z żywym playbackiem (to co user realnie widzi na liście). scene_id index
|
||||||
|
pokrywa oba joiny; performer ma rzędu setek scen → tani agregat per-request."""
|
||||||
|
live = exists().where(
|
||||||
|
and_(
|
||||||
|
PlaybackSource.scene_id == ScenePerformer.scene_id,
|
||||||
|
PlaybackSource.dead_at.is_(None),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
rows = session.execute(
|
||||||
|
select(Tag.id, Tag.name, Tag.slug, func.count().label("c"))
|
||||||
|
.select_from(ScenePerformer)
|
||||||
|
.join(SceneTag, SceneTag.scene_id == ScenePerformer.scene_id)
|
||||||
|
.join(Tag, Tag.id == SceneTag.tag_id)
|
||||||
|
.where(ScenePerformer.performer_id == performer_id, live)
|
||||||
|
.group_by(Tag.id, Tag.name, Tag.slug)
|
||||||
|
.order_by(func.count().desc(), Tag.name.asc())
|
||||||
|
.limit(limit)
|
||||||
|
).all()
|
||||||
|
items = [
|
||||||
|
TagCount(id=r.id, name=r.name, slug=r.slug, scene_count=int(r.c)) for r in rows
|
||||||
|
]
|
||||||
|
return TagListOut(items=items, total=len(items), page=1, per_page=limit)
|
||||||
|
|
||||||
|
|
||||||
@router.post("/performers/{performer_id}/rescrape", response_model=PerformerRescrapeOut)
|
@router.post("/performers/{performer_id}/rescrape", response_model=PerformerRescrapeOut)
|
||||||
def rescrape_performer_scenes(
|
def rescrape_performer_scenes(
|
||||||
performer_id: uuid.UUID,
|
performer_id: uuid.UUID,
|
||||||
|
|
|
||||||
102
app/api/watch.py
102
app/api/watch.py
|
|
@ -19,11 +19,13 @@ from pydantic import BaseModel
|
||||||
from sqlalchemy import desc, select
|
from sqlalchemy import desc, select
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app.api.device import get_device_id
|
||||||
from app.api.scenes import _build_scene_out
|
from app.api.scenes import _build_scene_out
|
||||||
from app.api.schemas import SceneOut
|
from app.api.schemas import SceneOut
|
||||||
from app.auth import require_api_key
|
from app.auth import require_api_key
|
||||||
from app.db import get_session
|
from app.db import get_session
|
||||||
from app.models.play_progress import ScenePlayProgress
|
from app.models.movie import Movie
|
||||||
|
from app.models.play_progress import MoviePlayProgress, ScenePlayProgress
|
||||||
from app.models.scene import Scene
|
from app.models.scene import Scene
|
||||||
|
|
||||||
router = APIRouter(tags=["watch"], dependencies=[Depends(require_api_key)])
|
router = APIRouter(tags=["watch"], dependencies=[Depends(require_api_key)])
|
||||||
|
|
@ -48,6 +50,7 @@ def upsert_progress(
|
||||||
scene_id: uuid.UUID,
|
scene_id: uuid.UUID,
|
||||||
body: ProgressIn,
|
body: ProgressIn,
|
||||||
session: Annotated[Session, Depends(get_session)],
|
session: Annotated[Session, Depends(get_session)],
|
||||||
|
device_id: Annotated[str, Depends(get_device_id)],
|
||||||
) -> ProgressOut:
|
) -> ProgressOut:
|
||||||
if session.get(Scene, scene_id) is None:
|
if session.get(Scene, scene_id) is None:
|
||||||
raise HTTPException(status_code=404, detail="scene not found")
|
raise HTTPException(status_code=404, detail="scene not found")
|
||||||
|
|
@ -67,6 +70,7 @@ def upsert_progress(
|
||||||
stmt = (
|
stmt = (
|
||||||
pg_insert(ScenePlayProgress)
|
pg_insert(ScenePlayProgress)
|
||||||
.values(
|
.values(
|
||||||
|
device_id=device_id,
|
||||||
scene_id=scene_id,
|
scene_id=scene_id,
|
||||||
position_sec=position_sec,
|
position_sec=position_sec,
|
||||||
duration_sec=body.duration_sec,
|
duration_sec=body.duration_sec,
|
||||||
|
|
@ -74,7 +78,7 @@ def upsert_progress(
|
||||||
last_played_at=now,
|
last_played_at=now,
|
||||||
)
|
)
|
||||||
.on_conflict_do_update(
|
.on_conflict_do_update(
|
||||||
index_elements=["scene_id"],
|
index_elements=["device_id", "scene_id"],
|
||||||
set_={
|
set_={
|
||||||
"position_sec": position_sec,
|
"position_sec": position_sec,
|
||||||
# duration_sec: zachowaj istniejący gdy body nie podaje
|
# duration_sec: zachowaj istniejący gdy body nie podaje
|
||||||
|
|
@ -90,7 +94,7 @@ def upsert_progress(
|
||||||
)
|
)
|
||||||
session.execute(stmt)
|
session.execute(stmt)
|
||||||
session.commit()
|
session.commit()
|
||||||
row = session.get(ScenePlayProgress, scene_id)
|
row = session.get(ScenePlayProgress, (device_id, scene_id))
|
||||||
assert row is not None
|
assert row is not None
|
||||||
return ProgressOut(
|
return ProgressOut(
|
||||||
scene_id=scene_id,
|
scene_id=scene_id,
|
||||||
|
|
@ -108,8 +112,94 @@ def upsert_progress(
|
||||||
def remove_progress(
|
def remove_progress(
|
||||||
scene_id: uuid.UUID,
|
scene_id: uuid.UUID,
|
||||||
session: Annotated[Session, Depends(get_session)],
|
session: Annotated[Session, Depends(get_session)],
|
||||||
|
device_id: Annotated[str, Depends(get_device_id)],
|
||||||
) -> None:
|
) -> None:
|
||||||
row = session.get(ScenePlayProgress, scene_id)
|
row = session.get(ScenePlayProgress, (device_id, scene_id))
|
||||||
|
if row is None:
|
||||||
|
return
|
||||||
|
session.delete(row)
|
||||||
|
session.commit()
|
||||||
|
|
||||||
|
|
||||||
|
# ---- Movie progress (mirror scen) ------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class MovieProgressOut(BaseModel):
|
||||||
|
movie_id: uuid.UUID
|
||||||
|
position_sec: int
|
||||||
|
duration_sec: int | None
|
||||||
|
finished: bool
|
||||||
|
last_played_at: datetime
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/movies/{movie_id}/progress", response_model=MovieProgressOut)
|
||||||
|
def upsert_movie_progress(
|
||||||
|
movie_id: uuid.UUID,
|
||||||
|
body: ProgressIn,
|
||||||
|
session: Annotated[Session, Depends(get_session)],
|
||||||
|
device_id: Annotated[str, Depends(get_device_id)],
|
||||||
|
) -> MovieProgressOut:
|
||||||
|
"""Mirror upsert_progress dla filmów (bug-report b207ff17 2026-05-26 —
|
||||||
|
"przydałoby się oznaczenie filmów już obejrzanych")."""
|
||||||
|
if session.get(Movie, movie_id) is None:
|
||||||
|
raise HTTPException(status_code=404, detail="movie not found")
|
||||||
|
|
||||||
|
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||||
|
|
||||||
|
now = datetime.now(UTC)
|
||||||
|
position_sec = max(0, body.position_sec)
|
||||||
|
finished = body.finished or (
|
||||||
|
bool(body.duration_sec)
|
||||||
|
and body.duration_sec > 0
|
||||||
|
and position_sec >= int(body.duration_sec * 0.95)
|
||||||
|
)
|
||||||
|
stmt = (
|
||||||
|
pg_insert(MoviePlayProgress)
|
||||||
|
.values(
|
||||||
|
device_id=device_id,
|
||||||
|
movie_id=movie_id,
|
||||||
|
position_sec=position_sec,
|
||||||
|
duration_sec=body.duration_sec,
|
||||||
|
finished=finished,
|
||||||
|
last_played_at=now,
|
||||||
|
)
|
||||||
|
.on_conflict_do_update(
|
||||||
|
index_elements=["device_id", "movie_id"],
|
||||||
|
set_={
|
||||||
|
"position_sec": position_sec,
|
||||||
|
"duration_sec": (
|
||||||
|
body.duration_sec
|
||||||
|
if body.duration_sec is not None
|
||||||
|
else MoviePlayProgress.duration_sec
|
||||||
|
),
|
||||||
|
"finished": finished,
|
||||||
|
"last_played_at": now,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
session.execute(stmt)
|
||||||
|
session.commit()
|
||||||
|
row = session.get(MoviePlayProgress, (device_id, movie_id))
|
||||||
|
assert row is not None
|
||||||
|
return MovieProgressOut(
|
||||||
|
movie_id=movie_id,
|
||||||
|
position_sec=row.position_sec,
|
||||||
|
duration_sec=row.duration_sec,
|
||||||
|
finished=row.finished,
|
||||||
|
last_played_at=row.last_played_at,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.delete(
|
||||||
|
"/movies/{movie_id}/progress",
|
||||||
|
status_code=status.HTTP_204_NO_CONTENT,
|
||||||
|
)
|
||||||
|
def remove_movie_progress(
|
||||||
|
movie_id: uuid.UUID,
|
||||||
|
session: Annotated[Session, Depends(get_session)],
|
||||||
|
device_id: Annotated[str, Depends(get_device_id)],
|
||||||
|
) -> None:
|
||||||
|
row = session.get(MoviePlayProgress, (device_id, movie_id))
|
||||||
if row is None:
|
if row is None:
|
||||||
return
|
return
|
||||||
session.delete(row)
|
session.delete(row)
|
||||||
|
|
@ -131,6 +221,7 @@ class WatchListOut(BaseModel):
|
||||||
@router.get("/watch/recent", response_model=WatchListOut)
|
@router.get("/watch/recent", response_model=WatchListOut)
|
||||||
def list_recent(
|
def list_recent(
|
||||||
session: Annotated[Session, Depends(get_session)],
|
session: Annotated[Session, Depends(get_session)],
|
||||||
|
device_id: Annotated[str, Depends(get_device_id)],
|
||||||
limit: int = Query(default=10, ge=1, le=50),
|
limit: int = Query(default=10, ge=1, le=50),
|
||||||
include_finished: bool = Query(default=False),
|
include_finished: bool = Query(default=False),
|
||||||
) -> WatchListOut:
|
) -> WatchListOut:
|
||||||
|
|
@ -139,6 +230,7 @@ def list_recent(
|
||||||
stmt = (
|
stmt = (
|
||||||
select(ScenePlayProgress, Scene)
|
select(ScenePlayProgress, Scene)
|
||||||
.join(Scene, Scene.id == ScenePlayProgress.scene_id)
|
.join(Scene, Scene.id == ScenePlayProgress.scene_id)
|
||||||
|
.where(ScenePlayProgress.device_id == device_id)
|
||||||
.order_by(desc(ScenePlayProgress.last_played_at))
|
.order_by(desc(ScenePlayProgress.last_played_at))
|
||||||
.limit(limit)
|
.limit(limit)
|
||||||
)
|
)
|
||||||
|
|
@ -149,7 +241,7 @@ def list_recent(
|
||||||
for prog, scene in session.execute(stmt).all():
|
for prog, scene in session.execute(stmt).all():
|
||||||
items.append(
|
items.append(
|
||||||
WatchEntry(
|
WatchEntry(
|
||||||
scene=_build_scene_out(session, scene),
|
scene=_build_scene_out(session, scene, device_id=device_id),
|
||||||
position_sec=prog.position_sec,
|
position_sec=prog.position_sec,
|
||||||
duration_sec=prog.duration_sec,
|
duration_sec=prog.duration_sec,
|
||||||
finished=prog.finished,
|
finished=prog.finished,
|
||||||
|
|
|
||||||
106
app/config.py
106
app/config.py
|
|
@ -48,6 +48,15 @@ class Settings(BaseSettings):
|
||||||
title_token_set_min: int = 88
|
title_token_set_min: int = 88
|
||||||
date_window_days: int = 7
|
date_window_days: int = 7
|
||||||
|
|
||||||
|
# Skip ingestu clip-store (ManyVids/IWantClips/Clips4Sale/...) z canonical source —
|
||||||
|
# to permanentne orphany (free tubes nie hostują), ~56% ingestu TPDB/StashDB.
|
||||||
|
# False = wciągaj jak dawniej. Tube'y z clip-store studiem NIE są skipowane (mają playback).
|
||||||
|
skip_clip_store: bool = Field(default=True, validation_alias="GOON_SKIP_CLIP_STORE")
|
||||||
|
|
||||||
|
# Minimalny duration sceny z tube/scraper przy ingescie — <N s = trailer/teaser/preview.
|
||||||
|
# 0 = wyłączony. Nieznany duration nie jest wycinany. NIE dotyczy canonical (TPDB/StashDB).
|
||||||
|
min_ingest_duration_sec: int = Field(default=180, validation_alias="GOON_MIN_INGEST_DURATION_SEC")
|
||||||
|
|
||||||
# APScheduler (M5). Każdy 0/None = job wyłączony.
|
# APScheduler (M5). Każdy 0/None = job wyłączony.
|
||||||
sched_tpdb_hours: int = Field(default=6, validation_alias="GOON_SCHED_TPDB_HOURS")
|
sched_tpdb_hours: int = Field(default=6, validation_alias="GOON_SCHED_TPDB_HOURS")
|
||||||
sched_stashdb_hours: int = Field(default=6, validation_alias="GOON_SCHED_STASHDB_HOURS")
|
sched_stashdb_hours: int = Field(default=6, validation_alias="GOON_SCHED_STASHDB_HOURS")
|
||||||
|
|
@ -72,13 +81,78 @@ class Settings(BaseSettings):
|
||||||
sched_movie_ingest_hours: int = Field(
|
sched_movie_ingest_hours: int = Field(
|
||||||
default=24, validation_alias="GOON_SCHED_MOVIE_INGEST_HOURS"
|
default=24, validation_alias="GOON_SCHED_MOVIE_INGEST_HOURS"
|
||||||
)
|
)
|
||||||
# Browse-latest scheduler: freshporno/porn00/pornxp newest scenes raz dziennie.
|
# Browse-latest scheduler: freshporno/porn00/pornxp newest scenes.
|
||||||
|
# 6h cadence (zmiana z 24h 2026-05-20): user reportował brak Brazzers Exxtra po
|
||||||
|
# 15-05. Root cause był 2-fold: (1) freshporno publikuje sceny w ciągu dnia, 24h
|
||||||
|
# cadence łapie tylko te do 05:30 UTC; (2) meta_content/release_date bug osobno.
|
||||||
|
# 6h = 4 runs/dzień = każda freshporno scena zaingestowana w ciągu ~6h od publik.
|
||||||
sched_browse_latest_hours: int = Field(
|
sched_browse_latest_hours: int = Field(
|
||||||
default=24, validation_alias="GOON_SCHED_BROWSE_LATEST_HOURS"
|
default=6, validation_alias="GOON_SCHED_BROWSE_LATEST_HOURS"
|
||||||
)
|
)
|
||||||
sched_browse_latest_max_pages: int = Field(
|
sched_browse_latest_max_pages: int = Field(
|
||||||
default=5, validation_alias="GOON_SCHED_BROWSE_LATEST_MAX_PAGES"
|
default=5, validation_alias="GOON_SCHED_BROWSE_LATEST_MAX_PAGES"
|
||||||
)
|
)
|
||||||
|
# Deep-crawl (Faza 2a) — pełne katalogi browse-tube'ów (porndoe ~62k itd.), nie tylko
|
||||||
|
# top-N. Round-robin po tube'ach, wznawialny kursor (app/_state/deepcrawl_state.json).
|
||||||
|
# 0 = wyłączony. 60 stron/run × ~31 scen ≈ 1860 scen/run (~22 min, hard-timeout 1h).
|
||||||
|
sched_deep_crawl_hours: int = Field(default=1, validation_alias="GOON_SCHED_DEEP_CRAWL_HOURS")
|
||||||
|
deep_crawl_pages_per_run: int = Field(default=60, validation_alias="GOON_DEEP_CRAWL_PAGES_PER_RUN")
|
||||||
|
deepcrawl_state_path: str = Field(default="", validation_alias="GOON_DEEPCRAWL_STATE_PATH")
|
||||||
|
# Bulk-dedup performers safety net — auto-merge duplikatów które resolver-time
|
||||||
|
# scoring pominął. 12h cadence: leci 2x dziennie (po porannym browse-latest run).
|
||||||
|
sched_bulk_dedup_hours: int = Field(
|
||||||
|
default=12, validation_alias="GOON_SCHED_BULK_DEDUP_HOURS"
|
||||||
|
)
|
||||||
|
# Thumb-asset dedup — scala dupy hdporn.gg/fullmovies.xxx (ten sam film, różne tytuły,
|
||||||
|
# ten sam asset-id miniatury + długość). bulk_dedup tego nie łapie (brak phash/tytuł).
|
||||||
|
# Re-ingesty pod nowymi tytułami → dupy odrastają, stąd cykliczny job. 12h. 0 = off.
|
||||||
|
sched_thumb_dedup_hours: int = Field(
|
||||||
|
default=12, validation_alias="GOON_SCHED_THUMB_DEDUP_HOURS"
|
||||||
|
)
|
||||||
|
# Title+duration dedup — scala missing-merge dupy (ten sam performer + identyczny
|
||||||
|
# znormalizowany tytuł + długość co do sekundy), których bulk_dedup nie łapie (tube
|
||||||
|
# re-scrape / cross-tube np. porn00 vs xnxx, reports 28fe8181/32df33b1). Odrastają
|
||||||
|
# przy re-ingeście, stąd cyklicznie. 12h, playback-only (to co user widzi). 0 = off.
|
||||||
|
sched_title_dedup_hours: int = Field(
|
||||||
|
default=12, validation_alias="GOON_SCHED_TITLE_DEDUP_HOURS"
|
||||||
|
)
|
||||||
|
# Ingest freshness watchdog — alert do Sentry gdy aktywny tube (origin
|
||||||
|
# tube:<sitetag>) przestał dawać nowe sceny > próg. Łapie zamrożenie
|
||||||
|
# pojedynczego origin, którego globalny monitor (jeden Source "tube-scraper") nie
|
||||||
|
# widzi (np. freshporno browse z rotującego roota, report 14f3a655). 6h cadence
|
||||||
|
# (po browse-latest). Każdy 0/None = wyłączony.
|
||||||
|
sched_ingest_watchdog_hours: int = Field(
|
||||||
|
default=6, validation_alias="GOON_SCHED_INGEST_WATCHDOG_HOURS"
|
||||||
|
)
|
||||||
|
# Próg dla browse-scraperów (ALL_BROWSE_SCRAPERS) — crawlowane raz dziennie z
|
||||||
|
# listingu, więc 48h ciszy = anomalia.
|
||||||
|
ingest_watchdog_max_age_hours: int = Field(
|
||||||
|
default=48, validation_alias="GOON_INGEST_WATCHDOG_MAX_AGE_HOURS"
|
||||||
|
)
|
||||||
|
# Próg dla performer-driven search-scraperów (ALL_DIRECT_SCRAPERS) — kadencja jest
|
||||||
|
# nierówna (continuous queue ~30d refresh per performer, ingest orphan-heavy), więc
|
||||||
|
# 48h dawałoby false-positivy. 7d (168h): healthy search-tuby obserwowane <6h świeżości
|
||||||
|
# (continuous tick hituje wszystkie tuby per performer), zamrożone ≥73h → ~28× margines.
|
||||||
|
ingest_watchdog_search_max_age_hours: int = Field(
|
||||||
|
default=168, validation_alias="GOON_INGEST_WATCHDOG_SEARCH_MAX_AGE_HOURS"
|
||||||
|
)
|
||||||
|
# TPDB movie enrichment — wzbogaca filmy obsadą + kategoriami (tagi) + studiem +
|
||||||
|
# reżyserem, plus kanoniczny TPDB UUID do dedupu mirrorów. paradisehill (primary)
|
||||||
|
# prawie nie ma obsady, więc to domyka największą lukę zakładki movies. Batch
|
||||||
|
# najświeższych GRYWALNYCH filmów bez obsady/studia per run; 6h cadence. 0 = off.
|
||||||
|
sched_tpdb_movie_enrich_hours: int = Field(
|
||||||
|
default=6, validation_alias="GOON_SCHED_TPDB_MOVIE_ENRICH_HOURS"
|
||||||
|
)
|
||||||
|
tpdb_movie_enrich_batch: int = Field(
|
||||||
|
default=200, validation_alias="GOON_TPDB_MOVIE_ENRICH_BATCH"
|
||||||
|
)
|
||||||
|
# Taxonomy scene_count refresh — przelicza denormalizowane liczniki scen na
|
||||||
|
# tags/performers/studios (hot-path /tags|/performers|/studios|/favorites czyta
|
||||||
|
# gotową kolumnę zamiast agregować 6.3M scene_tags per-request). 3h cadence —
|
||||||
|
# counts do tego stale, dla sortu "popular" + badge "(N)" bez znaczenia. 0 = off.
|
||||||
|
sched_taxonomy_counts_hours: int = Field(
|
||||||
|
default=3, validation_alias="GOON_SCHED_TAXONOMY_COUNTS_HOURS"
|
||||||
|
)
|
||||||
|
|
||||||
# Hetzner Cloud bandwidth monitor — read-only API token (Security → API Tokens
|
# Hetzner Cloud bandwidth monitor — read-only API token (Security → API Tokens
|
||||||
# w panelu Hetzner Cloud). Bez tokenu monitor wyłączony (warning w log).
|
# w panelu Hetzner Cloud). Bez tokenu monitor wyłączony (warning w log).
|
||||||
|
|
@ -89,6 +163,34 @@ class Settings(BaseSettings):
|
||||||
hetzner_alert_info_pct: int = Field(default=50, validation_alias="HETZNER_ALERT_INFO_PCT")
|
hetzner_alert_info_pct: int = Field(default=50, validation_alias="HETZNER_ALERT_INFO_PCT")
|
||||||
hetzner_alert_warning_pct: int = Field(default=80, validation_alias="HETZNER_ALERT_WARNING_PCT")
|
hetzner_alert_warning_pct: int = Field(default=80, validation_alias="HETZNER_ALERT_WARNING_PCT")
|
||||||
hetzner_alert_error_pct: int = Field(default=95, validation_alias="HETZNER_ALERT_ERROR_PCT")
|
hetzner_alert_error_pct: int = Field(default=95, validation_alias="HETZNER_ALERT_ERROR_PCT")
|
||||||
|
# Cadence sprawdzania transferu (godziny). 0/None = monitor wyłączony. Domyślnie 6h
|
||||||
|
# (transfer rośnie wolno; częściej bez sensu). Działa tylko gdy ustawiony token+id.
|
||||||
|
sched_hetzner_monitor_hours: int = Field(
|
||||||
|
default=6, validation_alias="GOON_SCHED_HETZNER_MONITOR_HOURS"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Source ranking (Sites screen) — przelicz source_stats (freshness/richness/health
|
||||||
|
# per origin). 0/None = wyłączone. Domyślnie 6h (richness to ciężki agregat po
|
||||||
|
# ~2M live playback_sources; częściej bez sensu, dane zmieniają się powoli).
|
||||||
|
sched_source_stats_hours: int = Field(
|
||||||
|
default=6, validation_alias="GOON_SCHED_SOURCE_STATS_HOURS"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Bright Data ISP proxy (stałe IP od ISP, rozliczane ryczałtem NIE per-GB) —
|
||||||
|
# używany do ingestu HTML (scrape) tubów które blokują VPS IP twardym Cloudflare
|
||||||
|
# 403 nawet z browser-TLS (superporn). Streamu i tak nie ruszamy proxy (tokeny CDN
|
||||||
|
# IP-bound). Format env: `host:port:user:pass` (panel Bright Data). Pusty = brak.
|
||||||
|
brightdata_proxy_raw: str = Field(default="", validation_alias="BRIGHTDATA_PROXY_URL")
|
||||||
|
|
||||||
|
@property
|
||||||
|
def brightdata_proxy_url(self) -> str | None:
|
||||||
|
"""`host:port:user:pass` → `http://user:pass@host:port` dla curl_cffi/httpx.
|
||||||
|
None gdy nieustawiony lub w złym formacie."""
|
||||||
|
parts = self.brightdata_proxy_raw.split(":")
|
||||||
|
if len(parts) != 4 or not all(parts):
|
||||||
|
return None
|
||||||
|
host, port, user, pwd = parts
|
||||||
|
return f"http://{user}:{pwd}@{host}:{port}"
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def api_keys(self) -> set[str]:
|
def api_keys(self) -> set[str]:
|
||||||
|
|
|
||||||
|
|
@ -36,13 +36,26 @@ def get_movie_connectors() -> list[tuple[str, type]]:
|
||||||
from app.connectors.dooplay import (
|
from app.connectors.dooplay import (
|
||||||
MangopornConnector,
|
MangopornConnector,
|
||||||
PandamoviesConnector,
|
PandamoviesConnector,
|
||||||
StreampornConnector,
|
StreampornVipConnector,
|
||||||
)
|
)
|
||||||
|
from app.connectors.freeomovie import FreeoMovieConnector
|
||||||
from app.connectors.paradisehill import ParadisehillConnector
|
from app.connectors.paradisehill import ParadisehillConnector
|
||||||
|
|
||||||
|
# Kolejność ingestu: paradisehill FIRST (canonical primary, mirrory się do
|
||||||
|
# niego przyklejają), potem mangoporn (jedyny mirror z realnym new-content —
|
||||||
|
# 72 nowych 2026-05-28; streamporn/pandamovies zwracają stale 0 new), na końcu
|
||||||
|
# streamporn + pandamovies. Powód reorderu (2026-05-30): gdy streamporn wiesza
|
||||||
|
# się intermittentnie, mangoporn musi zdążyć przed nim — patrz per-connector
|
||||||
|
# timeout w _job_movie_ingest.
|
||||||
|
# streamporn.nl padł (0 new od 2026-05-23) → zastąpiony żywym streamporn.vip
|
||||||
|
# (StreampornVipConnector, ocena 2026-07-02). Stara klasa .nl zostaje w dooplay.py
|
||||||
|
# dla backref istniejących origins `streamporn:*`, ale nie ingestujemy jej dalej.
|
||||||
return [
|
return [
|
||||||
("paradisehill", ParadisehillConnector),
|
("paradisehill", ParadisehillConnector),
|
||||||
("streamporn", StreampornConnector),
|
|
||||||
("pandamovies", PandamoviesConnector),
|
|
||||||
("mangoporn", MangopornConnector),
|
("mangoporn", MangopornConnector),
|
||||||
|
("streampornvip", StreampornVipConnector),
|
||||||
|
("pandamovies", PandamoviesConnector),
|
||||||
|
# freeomovie.to (bestia theme, nie dooplay) — mirror: świeże filmy DVD,
|
||||||
|
# title-trigram attach do canonical + playback z TABS. Ocena 2026-07-02.
|
||||||
|
("freeomovie", FreeoMovieConnector),
|
||||||
]
|
]
|
||||||
|
|
|
||||||
|
|
@ -2,11 +2,15 @@
|
||||||
|
|
||||||
Każdy scraper hit'uje tube bezpośrednio HTTPm — różne tube'y to różne rate limit
|
Każdy scraper hit'uje tube bezpośrednio HTTPm — różne tube'y to różne rate limit
|
||||||
budgets, więc mogą iść równolegle. Wszystkie feedują sceny do tej samej
|
budgets, więc mogą iść równolegle. Wszystkie feedują sceny do tej samej
|
||||||
`Source(name='pornapp')` (legacy nazwa — kept for DB compat) z external_id
|
`Source(name=SCRAPER_SOURCE_NAME)` z external_id `f"{sitetag}:{url}"`. Resolver
|
||||||
`f"{sitetag}:{url}"`. Resolver mergeuje idempotentnie po tym kluczu.
|
mergeuje idempotentnie po tym kluczu.
|
||||||
|
|
||||||
Search-based ścieżka (per performer name); category browse'ng przez `categoriesUrl`
|
Nazwa źródła: do 2026-06-07 brzmiała `"pornapp"` — myląca pozostałość po usuniętym
|
||||||
overrides w pornapp connector był specyficzny dla porn-app API i zostanie usunięty.
|
zewnętrznym porn-app API (sugerowała zależność od obcego serwisu, której NIE MA —
|
||||||
|
to nasze własne direct-scrapery tubów). Przemianowana na `"tube-scraper"`; wiersz
|
||||||
|
`sources` zaktualizowany w DB (UPDATE name) więc cała historia ingest_runs została.
|
||||||
|
|
||||||
|
Search-based ścieżka (per performer name); category browse'ng przez `categoriesUrl`.
|
||||||
|
|
||||||
UWAGA — speculative scrapers: większość aggregator + special tubes (xmoviesforyou,
|
UWAGA — speculative scrapers: większość aggregator + special tubes (xmoviesforyou,
|
||||||
watchporn, siska, porn4days, porndish, xxxfreewatch, latestleaks, mypornerleak,
|
watchporn, siska, porn4days, porndish, xxxfreewatch, latestleaks, mypornerleak,
|
||||||
|
|
@ -14,6 +18,11 @@ porndittcom, perverzija, fpoxxx, ...) ma URL templates + regex'y oparte na typow
|
||||||
WordPress conventions. Wymagają post-deploy verification — gdy któryś nie zwraca
|
WordPress conventions. Wymagają post-deploy verification — gdy któryś nie zwraca
|
||||||
wyników, sprawdź real search HTML + popraw template/regex w odpowiednim pliku.
|
wyników, sprawdź real search HTML + popraw template/regex w odpowiednim pliku.
|
||||||
"""
|
"""
|
||||||
|
# Umbrella Source.name dla wszystkich direct-scraperów (deep-crawl, browse-latest,
|
||||||
|
# performer-driven). Rename z legacy "pornapp" 2026-06-07 (mylące — nie ma zależności
|
||||||
|
# od zewnętrznego porn-app API).
|
||||||
|
SCRAPER_SOURCE_NAME = "tube-scraper"
|
||||||
|
|
||||||
from app.connectors.direct_scrapers._browse_base import BaseBrowseScraper
|
from app.connectors.direct_scrapers._browse_base import BaseBrowseScraper
|
||||||
from app.connectors.direct_scrapers.base import BaseDirectTubeScraper
|
from app.connectors.direct_scrapers.base import BaseDirectTubeScraper
|
||||||
from app.connectors.direct_scrapers.eporner import EpornerScraper
|
from app.connectors.direct_scrapers.eporner import EpornerScraper
|
||||||
|
|
@ -23,14 +32,14 @@ from app.connectors.direct_scrapers.hqporner import HQPornerScraper
|
||||||
from app.connectors.direct_scrapers.latestleaks import LatestLeaksScraper
|
from app.connectors.direct_scrapers.latestleaks import LatestLeaksScraper
|
||||||
from app.connectors.direct_scrapers.latestpornvideo import LatestPornVideoScraper
|
from app.connectors.direct_scrapers.latestpornvideo import LatestPornVideoScraper
|
||||||
from app.connectors.direct_scrapers.mypornerleak import MyPornerLeakScraper
|
from app.connectors.direct_scrapers.mypornerleak import MyPornerLeakScraper
|
||||||
|
from app.connectors.direct_scrapers.mypornerleak_browse import MyPornerLeakBrowseScraper
|
||||||
from app.connectors.direct_scrapers.perverzija import PerverzijaScraper
|
from app.connectors.direct_scrapers.perverzija import PerverzijaScraper
|
||||||
from app.connectors.direct_scrapers.porn4days import Porn4DaysScraper
|
from app.connectors.direct_scrapers.porn4days import Porn4DaysScraper
|
||||||
from app.connectors.direct_scrapers.pornditt import PornDittScraper
|
|
||||||
from app.connectors.direct_scrapers.porndish import PornDishScraper
|
from app.connectors.direct_scrapers.porndish import PornDishScraper
|
||||||
from app.connectors.direct_scrapers.pornhat import PornHatScraper # noqa: F401 — kept for backref; ingest disabled
|
|
||||||
from app.connectors.direct_scrapers.pornhub import PornHubScraper
|
|
||||||
from app.connectors.direct_scrapers.porntrex import PornTrexScraper
|
from app.connectors.direct_scrapers.porntrex import PornTrexScraper
|
||||||
from app.connectors.direct_scrapers.redtube import RedTubeScraper
|
from app.connectors.direct_scrapers.porntrex_browse import PornTrexBrowseScraper
|
||||||
|
from app.connectors.direct_scrapers.xnxx_browse import XnxxBrowseScraper
|
||||||
|
from app.connectors.direct_scrapers.youporn_browse import YouPornBrowseScraper
|
||||||
from app.connectors.direct_scrapers.siska import SiskaScraper
|
from app.connectors.direct_scrapers.siska import SiskaScraper
|
||||||
from app.connectors.direct_scrapers.sxyland import SxyLandScraper
|
from app.connectors.direct_scrapers.sxyland import SxyLandScraper
|
||||||
from app.connectors.direct_scrapers.sxyprn import SxyPrnScraper
|
from app.connectors.direct_scrapers.sxyprn import SxyPrnScraper
|
||||||
|
|
@ -41,7 +50,6 @@ from app.connectors.direct_scrapers.xnxx import XnxxScraper
|
||||||
from app.connectors.direct_scrapers.xvideos import XVideosScraper
|
from app.connectors.direct_scrapers.xvideos import XVideosScraper
|
||||||
from app.connectors.direct_scrapers.xxxfreewatch import XxxFreeWatchScraper # noqa: F401 — kept for backref; delisted
|
from app.connectors.direct_scrapers.xxxfreewatch import XxxFreeWatchScraper # noqa: F401 — kept for backref; delisted
|
||||||
from app.connectors.direct_scrapers.youporn import YouPornScraper
|
from app.connectors.direct_scrapers.youporn import YouPornScraper
|
||||||
from app.connectors.direct_scrapers.zerodayxx import ZeroDayXXScraper
|
|
||||||
|
|
||||||
ALL_DIRECT_SCRAPERS: list[type[BaseDirectTubeScraper]] = [
|
ALL_DIRECT_SCRAPERS: list[type[BaseDirectTubeScraper]] = [
|
||||||
# Existing 4 (verified, in production)
|
# Existing 4 (verified, in production)
|
||||||
|
|
@ -51,20 +59,12 @@ ALL_DIRECT_SCRAPERS: list[type[BaseDirectTubeScraper]] = [
|
||||||
# popunder redirect. Mobile WebView page-as-hoster pokazuje ad redirect zamiast video.
|
# popunder redirect. Mobile WebView page-as-hoster pokazuje ad redirect zamiast video.
|
||||||
# 33,598 playback_sources mass-marked dead, 27,374 solo-orphan scenes deleted.
|
# 33,598 playback_sources mass-marked dead, 27,374 solo-orphan scenes deleted.
|
||||||
SxyLandScraper,
|
SxyLandScraper,
|
||||||
# ZeroDayXXScraper — wyłączony 2026-05-12 (source quality report): 25,596 scen, 0.1% canonical
|
# ZeroDayXXScraper (0dayxx) — USUNIĘTY CAŁKOWICIE 2026-06-22 (user request). Orphan
|
||||||
# match. Slug-concat tytuły (`bella reese big butt ready to be filled with cum analized`) bez
|
# factory (0.1% canonical), zastępujemy lepszymi źródłami. Dane/pliki/extractor skasowane.
|
||||||
# `[Studio]` lub `Studio - Perf - Title` prefixu (parse rate 3%) → resolver nie ma żadnego
|
|
||||||
# signalu do matchu. Wraps watchporn ale dziedziczy stripped metadata. Solo orphany usunięte
|
|
||||||
# (~21k scen) — plik scrapera + extractor zostają (istniejące playback_sources nadal się
|
|
||||||
# resolvują).
|
|
||||||
# Mainstream (URL templates well-known)
|
# Mainstream (URL templates well-known)
|
||||||
# PornHubScraper — wyłączony 2026-05-12 (analiza źródeł): 23,750 scen scrapnietych,
|
# PornHub + RedTube — USUNIĘTE CAŁKOWICIE 2026-06-22 (user request). Disabled od
|
||||||
# tylko 105 (0.4%) match z TPDB/StashDB. PH hostuje głównie własne shortened
|
# 2026-05-12 (0.4% canonical match), zamrożone dane skasowane z DB, pliki scraperów
|
||||||
# clipy + amateur upload — nigdy nie zmatchują studio canonical content. Plik
|
# i ekstraktory usunięte. Powód: skrócone amatorskie clipy, nigdy nie matchują studio.
|
||||||
# zostaje (extractor `pornhubcom` używa go w playback resolve dla istniejących
|
|
||||||
# playback_sources).
|
|
||||||
# RedTubeScraper — wyłączony 2026-05-12 (analiza źródeł): 20,127 scen, 82 match
|
|
||||||
# (0.4%). Same powody co PH (skrócone clipy + amateur upload).
|
|
||||||
XVideosScraper,
|
XVideosScraper,
|
||||||
XnxxScraper,
|
XnxxScraper,
|
||||||
XHamsterScraper,
|
XHamsterScraper,
|
||||||
|
|
@ -77,26 +77,23 @@ ALL_DIRECT_SCRAPERS: list[type[BaseDirectTubeScraper]] = [
|
||||||
# Mixdrop zrebrandował na m1xdrop.bz, yt-dlp out-of-date, packer/JS extract = fail.
|
# Mixdrop zrebrandował na m1xdrop.bz, yt-dlp out-of-date, packer/JS extract = fail.
|
||||||
# Playmogo = DoodStream CAPTCHA. Porn-app sam olewa xmoviesforyou (brak handlera w
|
# Playmogo = DoodStream CAPTCHA. Porn-app sam olewa xmoviesforyou (brak handlera w
|
||||||
# jadx). 1,321 solo-orphan scen.
|
# jadx). 1,321 solo-orphan scen.
|
||||||
# WatchPornScraper — wyłączony 2026-05-12 (user bug-report). Wszystkie iframes to
|
# WatchPornScraper — RE-ENABLE 2026-07-02, przeniesiony do ALL_BROWSE_SCRAPERS.
|
||||||
# DoodStream variants (playmogo/d0000d/dooood/mivalyo) z CAPTCHA gate. WebView na
|
# Site przebudowany na KVS (DoodStream-CAPTCHA z 2026-05-12 zniknął); browse
|
||||||
# mobile = black screen (player JS nie inicjalizuje się przez Turnstile). 16%
|
# /latest-updates/ + flashvars get_file direct mp4 (extractor watchporn). Patrz watchporn.py.
|
||||||
# scen solo (no backup tube), 84% multi-source — user może użyć innego tube. yt-dlp
|
# SiskaScraper — przeniesiony do ALL_BROWSE_SCRAPERS (browse-konwersja 2026-06-20,
|
||||||
# nie wspiera DoodStream ("Piracy"), własny resolver TBD jeśli warto.
|
# bo search siski zepsuty site-side — `?s=` ignoruje query). Patrz siska.py.
|
||||||
# SiskaScraper — wyłączony 2026-05-16 (filemoon shutdown). Każda siska scena
|
|
||||||
# embeduje filemoon iframe; filemoon.to/sx/nl serwują od ~2026-05 placeholder
|
|
||||||
# "Byse Frontend" SPA bez player JS. 14,839 playback_sources mass-marked dead.
|
|
||||||
# Plik scrapera + extractor zostają (mobile spróbuje resolve → DEAD_HOSTER_RE
|
|
||||||
# filemoon blacklist → None → 503 — fine, te scenes są też dead_at-filtered).
|
|
||||||
# SiskaScraper,
|
|
||||||
# Porn4DaysScraper — wyłączony 2026-05-12 (post audit fix). 100% scen na streamtape
|
# Porn4DaysScraper — wyłączony 2026-05-12 (post audit fix). 100% scen na streamtape
|
||||||
# only (DEAD_HOSTER_RE blacklist - malware drive-by .reg downloads). SERVER1_URL =
|
# only (DEAD_HOSTER_RE blacklist - malware drive-by .reg downloads). SERVER1_URL =
|
||||||
# streamtape, brak SERVER2/SERVER3 backup. Porn-app sam olewa porn4days. 10,346
|
# streamtape, brak SERVER2/SERVER3 backup. Porn-app sam olewa porn4days. 10,346
|
||||||
# solo-orphan scen.
|
# solo-orphan scen.
|
||||||
PornDishScraper,
|
# PornDishScraper — przeniesiony do ALL_BROWSE_SCRAPERS (browse-konwersja 2026-06-24,
|
||||||
|
# watchdog GOON-16: search `?s=` zamarzł 2026-05-07). WordPress → browse przez WP REST
|
||||||
|
# API (/wp-json/wp/v2/posts) jak perverzija: tytuł/data/thumb/studio(category)/tagi.
|
||||||
# XxxFreeWatchScraper — wyłączony 2026-05-18. 790 scen, 0% canonical match, 100% solo-orphan.
|
# XxxFreeWatchScraper — wyłączony 2026-05-18. 790 scen, 0% canonical match, 100% solo-orphan.
|
||||||
# Cloudflare 403 z VPS IP, mobile WebView teoretycznie działa ale 0/790 scen miało jakikolwiek
|
# Cloudflare 403 z VPS IP, mobile WebView teoretycznie działa ale 0/790 scen miało jakikolwiek
|
||||||
# match do TPDB/StashDB. Pure orphan factory. Solo scenes deleted, scraper disabled.
|
# match do TPDB/StashDB. Pure orphan factory. Solo scenes deleted, scraper disabled.
|
||||||
LatestPornVideoScraper,
|
# LatestPornVideoScraper — przeniesiony do ALL_BROWSE_SCRAPERS (browse-konwersja 2026-06-22,
|
||||||
|
# user 1da0375e: search-driven nie brał feedu "latest" → stary zestaw w apce).
|
||||||
# LatestLeaksScraper — wyłączony 2026-05-12 (source quality report): 16,438 scen, 0.0%
|
# LatestLeaksScraper — wyłączony 2026-05-12 (source quality report): 16,438 scen, 0.0%
|
||||||
# canonical match. Slug-concat tytuły, brak studio/duration/date signali. Solo orphany
|
# canonical match. Slug-concat tytuły, brak studio/duration/date signali. Solo orphany
|
||||||
# usunięte (~15k scen).
|
# usunięte (~15k scen).
|
||||||
|
|
@ -105,23 +102,17 @@ ALL_DIRECT_SCRAPERS: list[type[BaseDirectTubeScraper]] = [
|
||||||
# zwraca consistent search results. KVS engine, slug-aware scene URLs. Mostly
|
# zwraca consistent search results. KVS engine, slug-aware scene URLs. Mostly
|
||||||
# orphan ingest (auto-screenshots, no canonical phash match — sprawdzone), ale
|
# orphan ingest (auto-screenshots, no canonical phash match — sprawdzone), ale
|
||||||
# może łapać sceny popularnych performerów których jeszcze nie mamy w TPDB.
|
# może łapać sceny popularnych performerów których jeszcze nie mamy w TPDB.
|
||||||
# PornHatScraper — wyłączony 2026-05-18. 9,799 scen, 0.2% canonical match, 100% solo-orphan.
|
# PornHat (pornhatcom) + PornDitt (porndittcom) — USUNIĘTE CAŁKOWICIE 2026-06-22
|
||||||
# Pure orphan factory — auto-screenshot thumbs nie matchują phash do canonical, slug tytuły
|
# (user request). Orphan factories (0.2% / weak-signal canonical match), zastępujemy
|
||||||
# nie matchują rapidfuzz, brak duration/date signals. KEEP `pornhatcom` extractor i istniejące
|
# lepszymi źródłami. Dane/pliki scraperów/extractory skasowane.
|
||||||
# playback_sources żywe — mobile może je odtwarzać; disable tylko future ingest.
|
|
||||||
# PornDittScraper — wyłączony 2026-05-12 (bug-report 64356e9b). Każdy link
|
|
||||||
# produkował nową Scene row zamiast matchować do istniejącej kanonicznej
|
|
||||||
# (TPDB/StashDB) bo pornditt ma weak signal: title + cz. performera, brak
|
|
||||||
# fingerprintu/duration/date → composite_score zawsze poniżej auto_merge
|
|
||||||
# threshold (0.92). Plik scrapera + extractor zostają (istniejące playback_sources
|
|
||||||
# nadal się resolvują, _REGISTRY w app/extractors/__init__.py odpala
|
|
||||||
# `porndittcom` → _embed_iframe.extract). Re-enable wymaga albo
|
|
||||||
# "alternative-source mode" w resolverze (match-only, never create new),
|
|
||||||
# albo bogatszej extracji metadanych (duration + fingerprint).
|
|
||||||
# Special
|
# Special
|
||||||
SxyPrnScraper,
|
SxyPrnScraper,
|
||||||
PerverzijaScraper,
|
# PerverzijaScraper — przeniesiony do ALL_BROWSE_SCRAPERS (browse-konwersja 2026-06-22,
|
||||||
FpoxxxScraper,
|
# user request). Search `?s=` → 429, homepage JS-renderowane; browse przez WP REST API
|
||||||
|
# (/wp-json/wp/v2/posts) daje tytuł/datę/thumb/studio(category)/tagi. Playback embed-iframe.
|
||||||
|
# FpoxxxScraper — przeniesiony do ALL_BROWSE_SCRAPERS (browse-konwersja 2026-06-22,
|
||||||
|
# user request). fpo.xxx to KVS, nie WordPress → search `?s=` zwracał 0; browse z
|
||||||
|
# `/new-<n>/` daje listing tile (tytuł/thumb/duration). Playback i tak phone-side (KVS).
|
||||||
]
|
]
|
||||||
|
|
||||||
# Browse-mode scrapers — iterują `latest-vids` listing zamiast search-by-performer.
|
# Browse-mode scrapers — iterują `latest-vids` listing zamiast search-by-performer.
|
||||||
|
|
@ -137,11 +128,49 @@ ALL_DIRECT_SCRAPERS: list[type[BaseDirectTubeScraper]] = [
|
||||||
# (phash Hamming 0). Oryginalne tytuły + channels=studio 1:1. **Aktywny.**
|
# (phash Hamming 0). Oryginalne tytuły + channels=studio 1:1. **Aktywny.**
|
||||||
from app.connectors.direct_scrapers.freshporno import FreshpornoScraper # noqa: E402
|
from app.connectors.direct_scrapers.freshporno import FreshpornoScraper # noqa: E402
|
||||||
from app.connectors.direct_scrapers.porn00 import Porn00Scraper # noqa: E402
|
from app.connectors.direct_scrapers.porn00 import Porn00Scraper # noqa: E402
|
||||||
|
from app.connectors.direct_scrapers.porndoe import PornDoeScraper # noqa: E402
|
||||||
from app.connectors.direct_scrapers.pornxp import PornXPScraper # noqa: E402
|
from app.connectors.direct_scrapers.pornxp import PornXPScraper # noqa: E402
|
||||||
from app.connectors.direct_scrapers.shyfap import ShyfapScraper # noqa: E402, F401
|
from app.connectors.direct_scrapers.shyfap import ShyfapScraper # noqa: E402, F401
|
||||||
|
from app.connectors.direct_scrapers.yesporn import YesPornVipScraper # noqa: E402
|
||||||
|
from app.connectors.direct_scrapers.fullmovies import FullmoviesScraper # noqa: E402
|
||||||
|
from app.connectors.direct_scrapers.hdporngg import HDPornGGScraper # noqa: E402
|
||||||
|
from app.connectors.direct_scrapers.hqfap import HQFapScraper # noqa: E402
|
||||||
|
from app.connectors.direct_scrapers.javflix import JavflixScraper # noqa: E402
|
||||||
|
from app.connectors.direct_scrapers.vjav import VjavScraper # noqa: E402
|
||||||
|
from app.connectors.direct_scrapers.supjav import SupjavScraper # noqa: E402
|
||||||
|
from app.connectors.direct_scrapers.neporn import NepornScraper # noqa: E402
|
||||||
|
from app.connectors.direct_scrapers.superporn import SuperpornScraper # noqa: E402
|
||||||
|
from app.connectors.direct_scrapers.eporner_api import EpornerApiScraper # noqa: E402
|
||||||
|
from app.connectors.direct_scrapers.xvideos_browse import XVideosBrowseScraper # noqa: E402
|
||||||
|
|
||||||
ALL_BROWSE_SCRAPERS: list[type[BaseBrowseScraper]] = [
|
ALL_BROWSE_SCRAPERS: list[type[BaseBrowseScraper]] = [
|
||||||
|
PerverzijaScraper,
|
||||||
|
PornDishScraper,
|
||||||
|
# WatchPornScraper — re-enabled 2026-07-02 jako browse (site przebudowany na KVS,
|
||||||
|
# DoodStream-CAPTCHA zniknął). Browse /latest-updates/: og:title, JSON-LD duration/
|
||||||
|
# uploadDate, /models/ performerzy, /tags/ tagi, /categories/ studio. Playback KVS
|
||||||
|
# get_file direct mp4 (extractor watchporn, VPS-side, token nie IP-bound).
|
||||||
|
WatchPornScraper,
|
||||||
|
# Browse równolegle do istniejącego search scrapera (wzorzec xvideos/eporner):
|
||||||
|
# search zostaje (pokrycie back-catalogu performerów), browse gwarantuje świeżość
|
||||||
|
# wprost z feedu (watchdog 48h zamiast 168h). Konwersja 2026-06-24 (user request).
|
||||||
|
PornTrexBrowseScraper,
|
||||||
|
MyPornerLeakBrowseScraper,
|
||||||
|
YouPornBrowseScraper,
|
||||||
|
XnxxBrowseScraper,
|
||||||
FreshpornoScraper,
|
FreshpornoScraper,
|
||||||
|
FpoxxxScraper,
|
||||||
|
# LatestPornVideoScraper — browse od 2026-06-22 (user 1da0375e: search-driven
|
||||||
|
# nie brał feedu "latest"). Listing card: tytuł (z embedded "<Studio> YY MM DD"),
|
||||||
|
# thumb (studio+date w nazwie), category-* jako tag. Performerów listing nie ma
|
||||||
|
# czysto (brak `actors-*`) → puste, dorabia canonical-merge. Playback: luluvid
|
||||||
|
# iframe → extractor latestpornvideocom (_embed_iframe) → telefon resolwuje.
|
||||||
|
LatestPornVideoScraper,
|
||||||
|
# SiskaScraper — re-enabled 2026-06-20 jako browse (user fa4083a2). Search siski
|
||||||
|
# zepsuty site-side (`?s=` ignoruje query), więc latest-browse z `/page/<n>/`.
|
||||||
|
# Komplet metadanych z kafelka listingu (tytuł/duration/thumb/performer/studio/
|
||||||
|
# kategoria). Playback: playmogo + luluvid → telefon resolwuje phone-side.
|
||||||
|
SiskaScraper,
|
||||||
# PornXPScraper — pilot 2026-05-17 (20 scen): studio 100%, performer 95%,
|
# PornXPScraper — pilot 2026-05-17 (20 scen): studio 100%, performer 95%,
|
||||||
# release_date 100%, duration 100%, stream_url 100%, phash 100%. Najlepsze
|
# release_date 100%, duration 100%, stream_url 100%, phash 100%. Najlepsze
|
||||||
# sygnały spośród browse-mode scraperów. Stream direct mp4 (sv.porn-xp.com)
|
# sygnały spośród browse-mode scraperów. Stream direct mp4 (sv.porn-xp.com)
|
||||||
|
|
@ -152,10 +181,72 @@ ALL_BROWSE_SCRAPERS: list[type[BaseBrowseScraper]] = [
|
||||||
# 720p). Tytuł zachowuje studio prefix ("Studio Title - Scene Name") → title
|
# 720p). Tytuł zachowuje studio prefix ("Studio Title - Scene Name") → title
|
||||||
# fuzzy match (rapidfuzz token_set_ratio) może załapać canonical. Monitorować.
|
# fuzzy match (rapidfuzz token_set_ratio) może załapać canonical. Monitorować.
|
||||||
Porn00Scraper,
|
Porn00Scraper,
|
||||||
|
# PornDoeScraper — dołączony 2026-05-21 (theporndude audit). Każda scena ma
|
||||||
|
# kompletny JSON-LD VideoObject: title + uploadDate + duration + named studio
|
||||||
|
# (producer/publisher) + named performers (actor[]) + thumbnail. Najbogatsze
|
||||||
|
# strukturalne metadane spośród browse scraperów — composite fuzzy match ma
|
||||||
|
# komplet sygnałów. Phash hit-rate niski (własne crop-thumbnaile), studio +
|
||||||
|
# performer + date + duration nadrabiają.
|
||||||
|
PornDoeScraper,
|
||||||
|
# YesPornVipScraper — dołączony 2026-05-27 (user audit). JSON-LD VideoObject
|
||||||
|
# + `<meta property="video:duration|release_date|tag">` per scena (Goon ma
|
||||||
|
# duration w sekundach gotowe + ISO 8601 release_date z timezone). Studio +
|
||||||
|
# performerzy z `btn gold` linków (`/channels/<slug>/` + `/models/<slug>/`).
|
||||||
|
# 941k organic monthly (SE Ranking, comparable z porndoe 731k / porntrex 790k).
|
||||||
|
# Scraper-of-paysites (DogFart / HardX / TeamSkeet / Vixen) — wysokie expected
|
||||||
|
# canonical match dla studio scenes. Korekta: theporndude scorecard rank 26
|
||||||
|
# ('yespornvip.com', score -0.5, auth wall) dotyczył **innej domeny** — pdude.link
|
||||||
|
# redirect do porndudecams affiliate. Prawdziwa kanoniczna domena to TLD `.vip`.
|
||||||
|
YesPornVipScraper,
|
||||||
|
# FullmoviesScraper + HDPornGGScraper — dołączone 2026-06-01. KVS engine (sponsor_groups
|
||||||
|
# stack, `/videos/<slug>/` + `/latest-updates/`). Studio teraz z PREFIKSU tytułu
|
||||||
|
# ("Studio - Scene") — sidebar `/networks/` listował WSZYSTKIE sieci, więc pierwszy match
|
||||||
|
# zawsze Brazzers (mis-attribution, dlatego nigdy nie były włączone). Niosą paysite studio
|
||||||
|
# content (TeamSkeet/Dad Crush/Brazzers/...) z title+performer+duration → composite fuzzy.
|
||||||
|
# Nawet bez canonical match: grywalny content z inferred tagami (mission: daily tagged ingest).
|
||||||
|
FullmoviesScraper,
|
||||||
|
HDPornGGScraper,
|
||||||
|
# EpornerApiScraper — dołączony 2026-06-03 (Faza 2b alternatywa). eporner detail to
|
||||||
|
# JS-heavy KVS bez SSR metadanych (jak porntrex/hqporner — odrzucone), ALE eporner ma
|
||||||
|
# publiczne JSON API (api/v2/video/search): 1 call = 100 filmów z title+length_sec+
|
||||||
|
# keywords+added+thumb. ~100k filmów, deep-crawl przez crawl_page() (API, bez detail-fetch).
|
||||||
|
EpornerApiScraper,
|
||||||
|
# XVideosBrowseScraper — dołączony 2026-06-03. SSR JSON-LD (duration/title/uploadDate)
|
||||||
|
# + page-parse /models/ (performerzy) + /tags/. Sample: median ~10.5min, 93% ≥3min.
|
||||||
|
# Mega-katalog ~13M → deep_crawl._PAGE_CAP["xvideoscom"]=1800 (~50k najnowszych), nie
|
||||||
|
# full-crawl. (youporn pominięty — JSON-LD bez actor/keywords, scene-perf/tagi = nav A-Z.)
|
||||||
|
XVideosBrowseScraper,
|
||||||
|
# HQFapScraper — PlayTube CMS (re-uploader pornhd.pet). Usunięty 2026-06-25 gdy CAŁA
|
||||||
|
# biblioteka CDN serwowała `/upload/videos/video_down.mp4` stub; PRZYWRÓCONY 2026-07-07
|
||||||
|
# bo strona wróciła na CDN vstor.top z realnymi plikami (portable cross-IP), user request.
|
||||||
|
HQFapScraper,
|
||||||
|
# FourK69Scraper — USUNIĘTY 2026-06-25 (ten sam stub), NIE sprawdzany ponownie.
|
||||||
|
# JavflixScraper — JAV vertical (osobna sekcja). origin tube:javflix jest w
|
||||||
|
# JAV_ORIGINS → list_scenes wyklucza je z głównego feedu (tylko zakładka JAV).
|
||||||
|
JavflixScraper,
|
||||||
|
# VjavScraper — JAV vertical (osobna sekcja, origin tube:vjav w JAV_ORIGINS).
|
||||||
|
# TXXX network: sitemap id-walk (newest=max id) + JSON metadata API. Stream
|
||||||
|
# videofile.php → get_file HLS (portable, /proxy/hls passthrough). RE 2026-07-10.
|
||||||
|
VjavScraper,
|
||||||
|
# SupjavScraper — JAV vertical (osobna sekcja, origin tube:supjav w JAV_ORIGINS).
|
||||||
|
# CF-blokuje VPS → browse homepage przez Bright Data proxy. Embed-aggregator:
|
||||||
|
# data-link reverse-hex → lk1.supremejav.com → hoster (extractor supjav). RE 2026-07-10.
|
||||||
|
SupjavScraper,
|
||||||
|
# NepornScraper — dołączony 2026-06-10 (user request). KVS engine (jak freshporno/
|
||||||
|
# porn00), /latest-updates/N/. JSON-LD (title+desc+uploadDate+thumb) + video:duration
|
||||||
|
# meta + /models/ performerzy + /categories/ tagi. Brak studio (tytuł bywa
|
||||||
|
# "- HardX Update - ..." — fuzzy match po tytule). Resolve server-side _kvs,
|
||||||
|
# finalny remote_control.php portable cross-IP.
|
||||||
|
NepornScraper,
|
||||||
|
# SuperpornScraper — dołączony 2026-06-10 (user request). superporn blokuje VPS IP
|
||||||
|
# twardym CF 403 (każda impersonacja TLS), więc ingest HTML idzie przez Bright Data
|
||||||
|
# ISP proxy (BRIGHTDATA_PROXY_URL, ryczałt nie per-GB). Pierwszy scraper z proxy — `_proxy` w
|
||||||
|
# _browse_base. JSON-LD (title+desc+uploadDate+thumb+duration) + chipy pornstar/
|
||||||
|
# kategorie. Playback IP-bound → WebView (extractor superporncom → _vps_blocked_fallback).
|
||||||
|
# Bez proxy: scraper no-op (pusty iterator).
|
||||||
|
SuperpornScraper,
|
||||||
|
# porntrex/hqporner/youporn — NIE: KVS/JS bez SSR duration → niewidoczne orphany (2026-06-03).
|
||||||
# ShyfapScraper — wyłączony 2026-05-12 (pilot fail, 0% match — orphan factory).
|
# ShyfapScraper — wyłączony 2026-05-12 (pilot fail, 0% match — orphan factory).
|
||||||
# Follow-up: dorobić te tubey i sprawdzić phash distance:
|
|
||||||
# - fullmovies.xxx (channel/network/pornstars/categories, brak duration)
|
|
||||||
# - 4k69.com + hdporn.gg (klony freshporno — prawdopodobnie ten sam phash hit rate)
|
|
||||||
]
|
]
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
|
|
|
||||||
|
|
@ -50,6 +50,11 @@ class BaseBrowseScraper(BaseDirectTubeScraper, abc.ABC):
|
||||||
_timeout: float = 30.0
|
_timeout: float = 30.0
|
||||||
"""HTTP timeout per request."""
|
"""HTTP timeout per request."""
|
||||||
|
|
||||||
|
_proxy: str | None = None
|
||||||
|
"""Opcjonalny proxy (http://user:pass@host:port) dla listing+detail fetchy.
|
||||||
|
Ustawiany przez scrapery tubów blokujących VPS IP (superporn → Bright Data
|
||||||
|
ISP proxy). None = bezpośredni fetch (domyślnie)."""
|
||||||
|
|
||||||
@abc.abstractmethod
|
@abc.abstractmethod
|
||||||
def _listing_url(self, page: int) -> str:
|
def _listing_url(self, page: int) -> str:
|
||||||
"""URL listing page'a 'latest-vids' (page 1 = newest)."""
|
"""URL listing page'a 'latest-vids' (page 1 = newest)."""
|
||||||
|
|
@ -65,50 +70,55 @@ class BaseBrowseScraper(BaseDirectTubeScraper, abc.ABC):
|
||||||
Zwraca None gdy scena niedostępna / parse fail — caller pominie ten URL,
|
Zwraca None gdy scena niedostępna / parse fail — caller pominie ten URL,
|
||||||
nie aborti całe browse."""
|
nie aborti całe browse."""
|
||||||
|
|
||||||
def latest_scenes(self, *, max_pages: int = 5) -> Iterator[RawScene]:
|
def crawl_page(self, page: int) -> list[RawScene] | None:
|
||||||
"""Iteruje sceny od najnowszych: page 1..max_pages × N scen/page.
|
"""Crawl JEDNEJ strony listingu → lista RawScene. Wspólne dla browse_latest
|
||||||
|
(top-N) i deep_crawl (kursor). Zwraca:
|
||||||
|
None — transient fetch-fail listingu (caller: stop, NIE oznaczaj exhausted),
|
||||||
|
[] — pusty listing = koniec katalogu (caller: exhausted),
|
||||||
|
[...] — sceny z tej strony.
|
||||||
|
|
||||||
Domyślnie max_pages=5 → ~100 scen per tube per run (shyfap, freshporno
|
API-based subclasses (np. EpornerApiScraper) override'ują crawl_page bezpośrednio
|
||||||
~20 scen/page). Schedulowane raz dziennie → catch-up po 24h przerwie.
|
(call API zamiast listing→detail). HTML browse subclasses dostarczają
|
||||||
|
_listing_url/_extract_scene_urls/_parse_detail i używają tej domyślnej impl.
|
||||||
Dedup po external_id zachodzi w resolverze (path 1 same_source) — gdy
|
|
||||||
scena już była, update last_seen + skip. Więc bezpieczne nawet gdy te
|
|
||||||
same N scen pojawia się przez kilka dni.
|
|
||||||
"""
|
"""
|
||||||
# search() nie jest implementowany przez subclass dla browse-only tube'ów —
|
|
||||||
# `BaseDirectTubeScraper.search` to abstrakt, więc dodajemy stub żeby
|
|
||||||
# przepuścić abc, ale faktyczna ścieżka pracy idzie przez latest_scenes().
|
|
||||||
for page in range(1, max_pages + 1):
|
|
||||||
url = self._listing_url(page)
|
url = self._listing_url(page)
|
||||||
try:
|
try:
|
||||||
res = browser_get(url, timeout=self._timeout)
|
res = browser_get(url, timeout=self._timeout, proxy=self._proxy)
|
||||||
html = res.text if hasattr(res, "text") else res
|
html = res.text if hasattr(res, "text") else res
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.warning("%s browse listing fetch failed (page %d): %s", self.sitetag, page, e)
|
log.warning("%s browse listing fetch failed (page %d): %s", self.sitetag, page, e)
|
||||||
break
|
return None
|
||||||
|
|
||||||
urls = self._extract_scene_urls(html)
|
urls = self._extract_scene_urls(html)
|
||||||
if not urls:
|
if not urls:
|
||||||
log.info("%s browse: empty listing page %d, stopping", self.sitetag, page)
|
return []
|
||||||
break
|
|
||||||
|
|
||||||
log.info("%s browse page %d: %d scene URLs", self.sitetag, page, len(urls))
|
log.info("%s browse page %d: %d scene URLs", self.sitetag, page, len(urls))
|
||||||
|
out: list[RawScene] = []
|
||||||
for scene_url in urls:
|
for scene_url in urls:
|
||||||
try:
|
try:
|
||||||
res = browser_get(scene_url, timeout=self._timeout)
|
res = browser_get(scene_url, timeout=self._timeout, proxy=self._proxy)
|
||||||
detail_html = res.text if hasattr(res, "text") else res
|
detail_html = res.text if hasattr(res, "text") else res
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.info("%s detail fetch failed %s: %s", self.sitetag, scene_url, e)
|
log.info("%s detail fetch failed %s: %s", self.sitetag, scene_url, e)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
try:
|
try:
|
||||||
raw = self._parse_detail(scene_url, detail_html)
|
raw = self._parse_detail(scene_url, detail_html)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.warning("%s detail parse failed %s: %s", self.sitetag, scene_url, e)
|
log.warning("%s detail parse failed %s: %s", self.sitetag, scene_url, e)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
if raw is not None:
|
if raw is not None:
|
||||||
yield raw
|
out.append(raw)
|
||||||
|
return out
|
||||||
|
|
||||||
|
def latest_scenes(self, *, max_pages: int = 5) -> Iterator[RawScene]:
|
||||||
|
"""Iteruje sceny od najnowszych: page 1..max_pages (browse_latest forward-fill).
|
||||||
|
Deep-crawl używa crawl_page() z kursorem osobno. Stop na None/[] (fail/koniec)."""
|
||||||
|
for page in range(1, max_pages + 1):
|
||||||
|
scenes = self.crawl_page(page)
|
||||||
|
if not scenes: # None (fetch fail) lub [] (pusty listing = koniec) → stop
|
||||||
|
break
|
||||||
|
yield from scenes
|
||||||
|
|
||||||
# Stub `search()` — BaseDirectTubeScraper wymaga implementacji. Dla browse-only
|
# Stub `search()` — BaseDirectTubeScraper wymaga implementacji. Dla browse-only
|
||||||
# tubes nie supportujemy performer-driven search; zwracamy pusty iterator. Tube'y
|
# tubes nie supportujemy performer-driven search; zwracamy pusty iterator. Tube'y
|
||||||
|
|
|
||||||
300
app/connectors/direct_scrapers/_playtube.py
Normal file
300
app/connectors/direct_scrapers/_playtube.py
Normal file
|
|
@ -0,0 +1,300 @@
|
||||||
|
"""BasePlayTubeScraper — wspólna baza dla tube'ów na PlayTube CMS (hqfap, 4k69).
|
||||||
|
|
||||||
|
Platforma rozpoznawalna po: `/watch/<slug>_<id>.html`, sitemap index
|
||||||
|
`/sitemaps/videos/sitemap-N.xml` (z `<lastmod>`), JSON-LD VideoObject na detail
|
||||||
|
page'u (name + uploadDate + duration ISO 8601 + thumbnailUrl + contentUrl) oraz
|
||||||
|
pillach `<a class='pill' href='/videos/pornstar|category/<Name>'>`.
|
||||||
|
|
||||||
|
Listing NIE paginuje się GET-em (PlayTube doładowuje AJAX-em `aj/load-more/`),
|
||||||
|
więc crawl_page buduje katalog z sitemapów (sort lastmod desc = newest first)
|
||||||
|
i tnie na strony po `_PAGE_SIZE`. Działa dla browse_latest (pages 1-5) i
|
||||||
|
deep_crawl (kursor do końca katalogu). Minus: sitemap laguje ~dobę za
|
||||||
|
najświeższymi uploadami — akceptowalne przy dziennym harmonogramie.
|
||||||
|
|
||||||
|
Subclass ustawia `base_url` + (opcjonalnie) override'uje `_pick_studio()` —
|
||||||
|
PlayTube nie ma strukturalnego pola studio na scenie, studio siedzi w
|
||||||
|
kategoriach (hqfap: suffix " Clips"; 4k69: nazwa z listy /studios).
|
||||||
|
|
||||||
|
Cloudflare: HTML wymaga browser TLS (curl_cffi w browser_get); plain curl z VPS
|
||||||
|
dostaje 403. Sitemapy i thumbnaile schodzą bez challenge'a.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import re
|
||||||
|
from datetime import date, datetime
|
||||||
|
|
||||||
|
from app.connectors.base import (
|
||||||
|
RawFingerprint,
|
||||||
|
RawPerformer,
|
||||||
|
RawPlaybackSource,
|
||||||
|
RawScene,
|
||||||
|
RawStudio,
|
||||||
|
RawTag,
|
||||||
|
)
|
||||||
|
from app.connectors.direct_scrapers._browse_base import (
|
||||||
|
BaseBrowseScraper,
|
||||||
|
compute_thumbnail_phash,
|
||||||
|
)
|
||||||
|
from app.extractors import browser_get
|
||||||
|
from app.normalize.text import slugify
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
_PAGE_SIZE = 20
|
||||||
|
|
||||||
|
_SITEMAP_LOC_RE = re.compile(r"<loc>\s*([^<]+?)\s*</loc>")
|
||||||
|
_URL_BLOCK_RE = re.compile(r"<url>(.*?)</url>", re.DOTALL | re.IGNORECASE)
|
||||||
|
_LASTMOD_RE = re.compile(r"<lastmod>\s*([^<]+?)\s*</lastmod>")
|
||||||
|
_SCENE_ID_RE = re.compile(r"_(\d+)\.html")
|
||||||
|
|
||||||
|
_JSONLD_RE = re.compile(
|
||||||
|
r'<script[^>]+type=["\']application/ld\+json["\'][^>]*>(.*?)</script>',
|
||||||
|
re.IGNORECASE | re.DOTALL,
|
||||||
|
)
|
||||||
|
_ISO_DUR_RE = re.compile(r"^P?T?(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?$", re.IGNORECASE)
|
||||||
|
|
||||||
|
# Pille performera/kategorii. PlayTube renderuje single-quoted attrs; dopuszczamy
|
||||||
|
# oba quote'y. Nazwa z `<span itemprop='name'>` (href bywa URL-encoded / ze spacjami).
|
||||||
|
_PILL_RE = re.compile(
|
||||||
|
r"<a\s+class=['\"]pill['\"]\s+href=['\"]/videos/(?P<kind>pornstar|category)/[^'\"]*['\"]"
|
||||||
|
r".*?<span itemprop=['\"]name['\"]>(?P<name>[^<]+)</span>",
|
||||||
|
re.IGNORECASE | re.DOTALL,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_iso_duration(value: str | None) -> int | None:
|
||||||
|
"""`PT26M48S` → sekundy. None gdy format nieznany."""
|
||||||
|
if not value:
|
||||||
|
return None
|
||||||
|
m = _ISO_DUR_RE.match(value.strip())
|
||||||
|
if not m:
|
||||||
|
return None
|
||||||
|
total = int(m.group(1) or 0) * 3600 + int(m.group(2) or 0) * 60 + int(m.group(3) or 0)
|
||||||
|
return total or None
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_iso_date(value: str | None) -> date | None:
|
||||||
|
"""`2026-06-09T16:00:00+00:00` → date. None gdy parse fail."""
|
||||||
|
if not value:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
return datetime.fromisoformat(value.replace("Z", "+00:00")).date()
|
||||||
|
except ValueError:
|
||||||
|
m = re.match(r"(\d{4}-\d{2}-\d{2})", value)
|
||||||
|
if m:
|
||||||
|
try:
|
||||||
|
return date.fromisoformat(m.group(1))
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _extract_video_object(html: str) -> dict | None:
|
||||||
|
"""Pierwszy JSON-LD VideoObject w HTML (PlayTube emituje jeden, płaski dict)."""
|
||||||
|
for m in _JSONLD_RE.finditer(html):
|
||||||
|
raw = m.group(1).strip()
|
||||||
|
if not raw:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
data = json.loads(raw)
|
||||||
|
except (json.JSONDecodeError, ValueError):
|
||||||
|
continue
|
||||||
|
items = data if isinstance(data, list) else [data]
|
||||||
|
for obj in items:
|
||||||
|
if isinstance(obj, dict) and obj.get("@type") == "VideoObject":
|
||||||
|
return obj
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
class BasePlayTubeScraper(BaseBrowseScraper):
|
||||||
|
base_url: str # np. "https://hqfap.com" — subclass ustawia
|
||||||
|
|
||||||
|
def __init__(self) -> None:
|
||||||
|
super().__init__()
|
||||||
|
# Katalog URL-i scen z sitemap, newest-first. Lazy-init raz per instancję
|
||||||
|
# (browse_latest i deep_crawl tworzą instancję per run, więc kilkanaście
|
||||||
|
# fetchy XML amortyzuje się na cały run).
|
||||||
|
self._catalog: list[str] | None = None
|
||||||
|
|
||||||
|
# Hook: wybierz studio spośród nazw kategorii (display name) albo None.
|
||||||
|
# Wybrana kategoria NIE trafia do tagów.
|
||||||
|
def _pick_studio(self, category_names: list[str]) -> str | None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
# crawl_page override (jak EpornerApiScraper) — listing nie jest stronicowalny
|
||||||
|
# przez GET, źródłem paginacji jest sitemap. _listing_url/_extract_scene_urls
|
||||||
|
# nieużywane, ale abstrakcyjne — dostarczamy no-op implementacje.
|
||||||
|
def _listing_url(self, page: int) -> str: # pragma: no cover - nieużywane
|
||||||
|
return f"{self.base_url}/sitemap.xml"
|
||||||
|
|
||||||
|
def _extract_scene_urls(self, listing_html: str) -> list[str]: # pragma: no cover
|
||||||
|
return []
|
||||||
|
|
||||||
|
def _load_catalog(self) -> list[str] | None:
|
||||||
|
"""Pełna lista URL-i scen posortowana lastmod desc. None = fetch fail."""
|
||||||
|
if self._catalog is not None:
|
||||||
|
return self._catalog
|
||||||
|
index_url = f"{self.base_url}/sitemap.xml"
|
||||||
|
try:
|
||||||
|
idx = browser_get(index_url, timeout=self._timeout)
|
||||||
|
idx.raise_for_status()
|
||||||
|
except Exception as e:
|
||||||
|
log.warning("%s: sitemap index fetch failed: %s", self.sitetag, e)
|
||||||
|
return None
|
||||||
|
sitemap_urls = [
|
||||||
|
u for u in _SITEMAP_LOC_RE.findall(idx.text) if "/videos/sitemap-" in u
|
||||||
|
]
|
||||||
|
if not sitemap_urls:
|
||||||
|
log.warning("%s: sitemap index has no video sitemaps", self.sitetag)
|
||||||
|
return None
|
||||||
|
|
||||||
|
entries: list[tuple[str, str]] = [] # (lastmod, scene_url)
|
||||||
|
for sm_url in sitemap_urls:
|
||||||
|
try:
|
||||||
|
sm = browser_get(sm_url, timeout=self._timeout)
|
||||||
|
sm.raise_for_status()
|
||||||
|
except Exception as e:
|
||||||
|
# Brak jednego sitemapa ≠ fail całości — reszta katalogu wystarczy.
|
||||||
|
log.warning("%s: sitemap fetch failed %s: %s", self.sitetag, sm_url, e)
|
||||||
|
continue
|
||||||
|
for block in _URL_BLOCK_RE.findall(sm.text):
|
||||||
|
loc_m = _SITEMAP_LOC_RE.search(block)
|
||||||
|
if not loc_m or "/watch/" not in loc_m.group(1):
|
||||||
|
continue
|
||||||
|
lastmod_m = _LASTMOD_RE.search(block)
|
||||||
|
entries.append((lastmod_m.group(1) if lastmod_m else "", loc_m.group(1)))
|
||||||
|
|
||||||
|
if not entries:
|
||||||
|
return None
|
||||||
|
# Dedup po scene id (sitemap potrafi powtórzyć URL między plikami).
|
||||||
|
entries.sort(key=lambda e: e[0], reverse=True)
|
||||||
|
seen_ids: set[str] = set()
|
||||||
|
catalog: list[str] = []
|
||||||
|
for _, url in entries:
|
||||||
|
id_m = _SCENE_ID_RE.search(url)
|
||||||
|
key = id_m.group(1) if id_m else url
|
||||||
|
if key in seen_ids:
|
||||||
|
continue
|
||||||
|
seen_ids.add(key)
|
||||||
|
catalog.append(url)
|
||||||
|
log.info("%s: catalog loaded — %d scenes from %d sitemaps",
|
||||||
|
self.sitetag, len(catalog), len(sitemap_urls))
|
||||||
|
self._catalog = catalog
|
||||||
|
return catalog
|
||||||
|
|
||||||
|
def crawl_page(self, page: int) -> list[RawScene] | None:
|
||||||
|
catalog = self._load_catalog()
|
||||||
|
if catalog is None:
|
||||||
|
return None
|
||||||
|
start = (page - 1) * _PAGE_SIZE
|
||||||
|
chunk = catalog[start:start + _PAGE_SIZE]
|
||||||
|
if not chunk:
|
||||||
|
return []
|
||||||
|
out: list[RawScene] = []
|
||||||
|
for scene_url in chunk:
|
||||||
|
try:
|
||||||
|
res = browser_get(scene_url, timeout=self._timeout)
|
||||||
|
res.raise_for_status()
|
||||||
|
except Exception as e:
|
||||||
|
log.info("%s detail fetch failed %s: %s", self.sitetag, scene_url, e)
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
raw = self._parse_detail(scene_url, res.text)
|
||||||
|
except Exception as e:
|
||||||
|
log.warning("%s detail parse failed %s: %s", self.sitetag, scene_url, e)
|
||||||
|
continue
|
||||||
|
if raw is not None:
|
||||||
|
out.append(raw)
|
||||||
|
return out
|
||||||
|
|
||||||
|
def _parse_detail(self, scene_url: str, detail_html: str) -> RawScene | None:
|
||||||
|
video = _extract_video_object(detail_html)
|
||||||
|
if not video:
|
||||||
|
log.info("%s: no JSON-LD VideoObject on %s", self.sitetag, scene_url)
|
||||||
|
return None
|
||||||
|
|
||||||
|
title = (video.get("name") or "").strip()
|
||||||
|
if not title:
|
||||||
|
return None
|
||||||
|
|
||||||
|
id_m = _SCENE_ID_RE.search(scene_url)
|
||||||
|
scene_id = id_m.group(1) if id_m else None
|
||||||
|
|
||||||
|
duration_sec = _parse_iso_duration(video.get("duration"))
|
||||||
|
release_date = _parse_iso_date(video.get("uploadDate"))
|
||||||
|
thumbnail_url = video.get("thumbnailUrl") or None
|
||||||
|
|
||||||
|
# Pille: pornstar → performer; category → studio (hook `_pick_studio`)
|
||||||
|
# albo tag. Ocenzurowane nazwy (`Te***`) pomijamy — gwiazdki to nie dane.
|
||||||
|
performers: list[RawPerformer] = []
|
||||||
|
category_names: list[str] = []
|
||||||
|
seen_perf: set[str] = set()
|
||||||
|
for m in _PILL_RE.finditer(detail_html):
|
||||||
|
name = m.group("name").strip()
|
||||||
|
if not name or "*" in name:
|
||||||
|
continue
|
||||||
|
if m.group("kind").lower() == "pornstar":
|
||||||
|
slug = slugify(name)
|
||||||
|
if slug and slug not in seen_perf:
|
||||||
|
seen_perf.add(slug)
|
||||||
|
performers.append(
|
||||||
|
RawPerformer(external_id=f"{self.sitetag}:performer:{slug}", name=name)
|
||||||
|
)
|
||||||
|
elif name not in category_names:
|
||||||
|
category_names.append(name)
|
||||||
|
|
||||||
|
studio: RawStudio | None = None
|
||||||
|
studio_name = self._pick_studio(category_names)
|
||||||
|
if studio_name:
|
||||||
|
studio = RawStudio(
|
||||||
|
external_id=f"{self.sitetag}:studio:{slugify(studio_name)}",
|
||||||
|
name=studio_name,
|
||||||
|
slug=slugify(studio_name),
|
||||||
|
)
|
||||||
|
|
||||||
|
tags: list[RawTag] = []
|
||||||
|
seen_tag: set[str] = set()
|
||||||
|
picked = (studio_name or "").strip().lower()
|
||||||
|
for name in category_names:
|
||||||
|
# Studio-kategoria nie idzie do tagów (ani w wersji z suffixem " Clips").
|
||||||
|
if picked and name.strip().lower() in (picked, picked + " clips"):
|
||||||
|
continue
|
||||||
|
slug = slugify(name)
|
||||||
|
if not slug or slug in seen_tag:
|
||||||
|
continue
|
||||||
|
seen_tag.add(slug)
|
||||||
|
tags.append(RawTag(external_id=f"{self.sitetag}:tag:{slug}", name=name, slug=slug))
|
||||||
|
|
||||||
|
# Phash: thumbnaile bywają re-encodowanym studio art (szansa na match),
|
||||||
|
# dla amatorskiego contentu nie zmatchują. Graceful: miss → composite scoring.
|
||||||
|
fingerprints: list[RawFingerprint] = []
|
||||||
|
if thumbnail_url:
|
||||||
|
ph = compute_thumbnail_phash(thumbnail_url, referer=self.base_url + "/")
|
||||||
|
if ph:
|
||||||
|
fingerprints.append(RawFingerprint(kind="phash", value=ph))
|
||||||
|
|
||||||
|
# Stream: JSON-LD contentUrl wygasa (token time-bound) — NIE zapisujemy
|
||||||
|
# stream_url; extractor per-sitetag resolvuje świeży on-demand.
|
||||||
|
playback_sources = [
|
||||||
|
RawPlaybackSource(
|
||||||
|
origin=f"tube:{self.sitetag}",
|
||||||
|
page_url=scene_url,
|
||||||
|
duration_sec=duration_sec,
|
||||||
|
thumbnail_url=thumbnail_url,
|
||||||
|
)
|
||||||
|
]
|
||||||
|
|
||||||
|
return RawScene(
|
||||||
|
external_id=f"{self.sitetag}:{scene_id or scene_url}",
|
||||||
|
title=title,
|
||||||
|
release_date=release_date,
|
||||||
|
duration_sec=duration_sec,
|
||||||
|
url=scene_url,
|
||||||
|
studio=studio,
|
||||||
|
performers=performers,
|
||||||
|
tags=tags,
|
||||||
|
fingerprints=fingerprints,
|
||||||
|
playback_sources=playback_sources,
|
||||||
|
)
|
||||||
|
|
@ -198,13 +198,18 @@ class BaseSearchScraper(BaseDirectTubeScraper):
|
||||||
studio: RawStudio | None = None
|
studio: RawStudio | None = None
|
||||||
extra_performers: list[RawPerformer] = []
|
extra_performers: list[RawPerformer] = []
|
||||||
tags: list[RawTag] = []
|
tags: list[RawTag] = []
|
||||||
|
description: str | None = None
|
||||||
try:
|
try:
|
||||||
meta = self._fetch_scene_metadata(scene_url)
|
meta = self._fetch_scene_metadata(scene_url)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.debug("%s metadata fetch failed for %s: %s", self.sitetag, scene_url, e)
|
log.debug("%s metadata fetch failed for %s: %s", self.sitetag, scene_url, e)
|
||||||
meta = None
|
meta = None
|
||||||
if meta is not None:
|
if meta is not None:
|
||||||
studio, extra_performers, tags = meta
|
# Back-compat: subclass może zwrócić 3-tuple (studio, performers, tags)
|
||||||
|
# LUB 4-tuple z dodatkowym `description` (porndish). Unpack defensywnie.
|
||||||
|
studio, extra_performers, tags = meta[0], meta[1], meta[2]
|
||||||
|
if len(meta) > 3:
|
||||||
|
description = meta[3]
|
||||||
|
|
||||||
# Performer z query zawsze obecny (driver scraping). Extra performers
|
# Performer z query zawsze obecny (driver scraping). Extra performers
|
||||||
# z detail page dorzucamy — dedupe po slug/name w resolverze.
|
# z detail page dorzucamy — dedupe po slug/name w resolverze.
|
||||||
|
|
@ -213,6 +218,7 @@ class BaseSearchScraper(BaseDirectTubeScraper):
|
||||||
yield RawScene(
|
yield RawScene(
|
||||||
external_id=f"{self.sitetag}:{scene_url}",
|
external_id=f"{self.sitetag}:{scene_url}",
|
||||||
title=title,
|
title=title,
|
||||||
|
description=description,
|
||||||
url=scene_url,
|
url=scene_url,
|
||||||
playback_sources=[
|
playback_sources=[
|
||||||
RawPlaybackSource(
|
RawPlaybackSource(
|
||||||
|
|
|
||||||
|
|
@ -8,7 +8,8 @@ from app.connectors.base import RawScene
|
||||||
|
|
||||||
|
|
||||||
class BaseDirectTubeScraper(abc.ABC):
|
class BaseDirectTubeScraper(abc.ABC):
|
||||||
"""Kontrakt direct scrapera. Wszystkie scrapery feedują do `Source(name='pornapp')`
|
"""Kontrakt direct scrapera. Wszystkie scrapery feedują do
|
||||||
|
`Source(name=SCRAPER_SOURCE_NAME)` ("tube-scraper", rename z "pornapp" 2026-06-07)
|
||||||
żeby dziedziczyć logikę resolvera + idempotent merge per external_id."""
|
żeby dziedziczyć logikę resolvera + idempotent merge per external_id."""
|
||||||
|
|
||||||
sitetag: str
|
sitetag: str
|
||||||
|
|
|
||||||
124
app/connectors/direct_scrapers/eporner_api.py
Normal file
124
app/connectors/direct_scrapers/eporner_api.py
Normal file
|
|
@ -0,0 +1,124 @@
|
||||||
|
"""eporner.com — deep-crawl przez oficjalne JSON API (api/v2/video/search).
|
||||||
|
|
||||||
|
Detail HTML eporner jest JS-heavy (brak SSR duration/title/tagów) — ALE eporner ma
|
||||||
|
publiczne API zwracające KOMPLETNĄ metadatę w jednym callu: `title`, `length_sec`
|
||||||
|
(duration), `keywords` (tagi), `added` (data), thumb, embed, url. ~100k filmów,
|
||||||
|
`order=latest`, ~100/stronę → ~1000 szybkich calli (BEZ detail-fetch). To czyni
|
||||||
|
eporner idealnym SSR-bogatym źródłem deep-crawla (analiza 2026-06-03: porntrex/hqporner
|
||||||
|
odrzucone — KVS bez SSR duration; eporner-API je zastępuje).
|
||||||
|
|
||||||
|
Override `crawl_page()` (API flow). HTML-owe _listing_url/_extract/_parse to stuby
|
||||||
|
(BaseBrowseScraper ABC ich wymaga, ale nieużywane). Sitetag `epornercom` = ten sam co
|
||||||
|
search-scraper EpornerScraper → external_id namespace wspólny (dedup).
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
from datetime import date, datetime
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
|
||||||
|
from app.connectors.base import RawPlaybackSource, RawScene, RawTag
|
||||||
|
from app.connectors.direct_scrapers._browse_base import BaseBrowseScraper
|
||||||
|
from app.normalize.text import slugify
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
_API = "https://www.eporner.com/api/v2/video/search/"
|
||||||
|
_PER_PAGE = 100
|
||||||
|
_UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36"
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_added(value: str | None) -> date | None:
|
||||||
|
if not value:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
return datetime.strptime(value.strip(), "%Y-%m-%d %H:%M:%S").date()
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _keywords_to_tags(keywords: str | None, sitetag: str) -> list[RawTag]:
|
||||||
|
if not keywords:
|
||||||
|
return []
|
||||||
|
tags: list[RawTag] = []
|
||||||
|
seen: set[str] = set()
|
||||||
|
for kw in keywords.split(","):
|
||||||
|
name = kw.strip()
|
||||||
|
# Pomijamy puste + title-jak-keyword (eporner czasem wrzuca cały title jako keyword).
|
||||||
|
if not name or len(name) > 40:
|
||||||
|
continue
|
||||||
|
slug = slugify(name)
|
||||||
|
if not slug or slug in seen or len(slug) > 60:
|
||||||
|
continue
|
||||||
|
seen.add(slug)
|
||||||
|
tags.append(RawTag(external_id=f"{sitetag}:tag:{slug}", name=name, slug=slug))
|
||||||
|
return tags
|
||||||
|
|
||||||
|
|
||||||
|
class EpornerApiScraper(BaseBrowseScraper):
|
||||||
|
sitetag = "epornercom"
|
||||||
|
|
||||||
|
def crawl_page(self, page: int) -> list[RawScene] | None:
|
||||||
|
params = {
|
||||||
|
"query": "",
|
||||||
|
"per_page": _PER_PAGE,
|
||||||
|
"page": page,
|
||||||
|
"order": "latest",
|
||||||
|
"thumbsize": "medium",
|
||||||
|
"format": "json",
|
||||||
|
}
|
||||||
|
try:
|
||||||
|
with httpx.Client(timeout=self._timeout, follow_redirects=True, headers={"User-Agent": _UA}) as c:
|
||||||
|
r = c.get(_API, params=params)
|
||||||
|
if r.status_code != 200:
|
||||||
|
log.warning("eporner api page %d status %d", page, r.status_code)
|
||||||
|
return None
|
||||||
|
data = r.json()
|
||||||
|
except Exception as e:
|
||||||
|
log.warning("eporner api page %d failed: %s", page, e)
|
||||||
|
return None
|
||||||
|
|
||||||
|
videos = data.get("videos") or []
|
||||||
|
if not videos:
|
||||||
|
return [] # poza ostatnią stroną → koniec katalogu (exhausted)
|
||||||
|
|
||||||
|
out: list[RawScene] = []
|
||||||
|
for v in videos:
|
||||||
|
url = (v.get("url") or "").strip()
|
||||||
|
title = (v.get("title") or "").strip()
|
||||||
|
if not url or not title:
|
||||||
|
continue
|
||||||
|
dur = v.get("length_sec")
|
||||||
|
duration_sec = int(dur) if dur else None
|
||||||
|
thumb = (v.get("default_thumb") or {}).get("src")
|
||||||
|
out.append(
|
||||||
|
RawScene(
|
||||||
|
external_id=f"{self.sitetag}:{url}",
|
||||||
|
title=title,
|
||||||
|
duration_sec=duration_sec,
|
||||||
|
release_date=_parse_added(v.get("added")),
|
||||||
|
url=url,
|
||||||
|
tags=_keywords_to_tags(v.get("keywords"), self.sitetag),
|
||||||
|
playback_sources=[
|
||||||
|
RawPlaybackSource(
|
||||||
|
origin=f"tube:{self.sitetag}",
|
||||||
|
page_url=url,
|
||||||
|
duration_sec=duration_sec,
|
||||||
|
thumbnail_url=thumb,
|
||||||
|
)
|
||||||
|
],
|
||||||
|
raw={"source": "eporner_api", "id": v.get("id")},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return out
|
||||||
|
|
||||||
|
# HTML stuby — nieużywane (crawl_page nadpisany API-flow), ale ABC ich wymaga.
|
||||||
|
def _listing_url(self, page: int) -> str: # pragma: no cover
|
||||||
|
raise NotImplementedError("EpornerApiScraper używa crawl_page (API), nie HTML listingu")
|
||||||
|
|
||||||
|
def _extract_scene_urls(self, listing_html: str) -> list[str]: # pragma: no cover
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
|
def _parse_detail(self, scene_url: str, detail_html: str) -> RawScene | None: # pragma: no cover
|
||||||
|
raise NotImplementedError
|
||||||
|
|
@ -1,22 +1,129 @@
|
||||||
"""fpoxxx — direct HTML scrape search results.
|
"""fpo.xxx — latest-vids browse scraper (KVS engine).
|
||||||
|
|
||||||
UWAGA: dokładna domena fpoxxx (sitetag w bazie) niekoniecznie zawiera "com" ani
|
Historia: dawniej WordPress-search scraper (`?s=`), ale fpo.xxx to KVS, nie WP —
|
||||||
"net" — porn-app DEFAULT_SITETAGS używa "fpoxxx" jako sitetag. Best-guess: fpo.xxx.
|
search zwracał 0 (regex slug-URL nie pasował do `/video/<id>/`). Przerobione na
|
||||||
|
BROWSE (latest z `/new-<n>/`), 2026-06-22 (user request: ożywić zamrożone tuby).
|
||||||
|
|
||||||
Search: `https://fpo.xxx/page/<n>/?s=<q>` (WordPress).
|
Listing tile (`/new-<n>/`):
|
||||||
Scene URL: `https://fpo.xxx/<slug>/`.
|
<a href="https://www.fpo.xxx/video/<id>/<slug>/" title="<Tytuł>">
|
||||||
|
<img data-original="...screenshots/.../320x180/1.jpg"> → thumb
|
||||||
|
<span class="duration">1:59:10</span> → duration
|
||||||
|
→ tytuł, miniatura, duration, URL sceny. Performerów/tagów listing nie ma czysto
|
||||||
|
(tytuł bywa JAV-code "Imai Kaho-RKI-602 ..."), więc puste → dorabia canonical-merge.
|
||||||
|
|
||||||
|
Playback: KVS (kt_player + license_code na detail page) — token IP-bound, resolve
|
||||||
|
PO STRONIE TELEFONU (fpoxxxResolver.ts / WebView fallback, extractor `fpoxxx`).
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import html
|
||||||
|
import logging
|
||||||
import re
|
import re
|
||||||
|
|
||||||
from app.connectors.direct_scrapers._search_base import BaseSearchScraper
|
from app.connectors.base import (
|
||||||
|
RawFingerprint,
|
||||||
|
RawPlaybackSource,
|
||||||
|
RawScene,
|
||||||
|
)
|
||||||
|
from app.connectors.direct_scrapers._browse_base import (
|
||||||
|
BaseBrowseScraper,
|
||||||
|
compute_thumbnail_phash,
|
||||||
|
)
|
||||||
|
from app.extractors import browser_get
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
class FpoxxxScraper(BaseSearchScraper):
|
_BASE = "https://www.fpo.xxx"
|
||||||
sitetag = "fpoxxx"
|
# Kafelek: <a href="...fpo.xxx/video/<id>/<slug>/" title="<tytuł>">. Reszta pól w oknie.
|
||||||
_search_url_template = "https://fpo.xxx/page/{page}/?s={query}"
|
_A_RE = re.compile(
|
||||||
_scene_url_re = re.compile(
|
r'<a\s+href="(?P<url>https?://(?:www\.)?fpo\.xxx/video/\d+/[^"]*)"\s+title="(?P<title>[^"]*)"',
|
||||||
r'href="(?P<url>https://fpo\.xxx/(?P<slug>[a-z0-9][a-z0-9\-]+))/"',
|
|
||||||
re.IGNORECASE,
|
re.IGNORECASE,
|
||||||
)
|
)
|
||||||
|
_THUMB_RE = re.compile(r'data-original="([^"]+)"', re.IGNORECASE)
|
||||||
|
_DUR_RE = re.compile(r'class="duration">\s*([\d]{1,2}(?:\s*:\s*[\d]{2}){1,2})\s*<')
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_duration(text: str | None) -> int | None:
|
||||||
|
"""`1:59:10`→7150 (H:MM:SS); `40:27`→2427 (MM:SS). None gdy brak."""
|
||||||
|
if not text:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
nums = [int(p.strip()) for p in text.split(":")]
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
if len(nums) == 2:
|
||||||
|
return nums[0] * 60 + nums[1]
|
||||||
|
if len(nums) == 3:
|
||||||
|
return nums[0] * 3600 + nums[1] * 60 + nums[2]
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
class FpoxxxScraper(BaseBrowseScraper):
|
||||||
|
sitetag = "fpoxxx"
|
||||||
|
|
||||||
|
def _listing_url(self, page: int) -> str:
|
||||||
|
return f"{_BASE}/new-{page}/"
|
||||||
|
|
||||||
|
# crawl_page nadpisany → abstrakcje nieużywane, ale wymagane do instancji.
|
||||||
|
def _extract_scene_urls(self, listing_html: str) -> list[str]:
|
||||||
|
return [m.group("url") for m in _A_RE.finditer(listing_html)]
|
||||||
|
|
||||||
|
def _parse_detail(self, scene_url: str, detail_html: str) -> RawScene | None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def crawl_page(self, page: int) -> list[RawScene] | None:
|
||||||
|
url = self._listing_url(page)
|
||||||
|
try:
|
||||||
|
res = browser_get(url, timeout=self._timeout)
|
||||||
|
text = res.text if hasattr(res, "text") else res
|
||||||
|
except Exception as e:
|
||||||
|
log.warning("fpoxxx browse listing fetch failed (page %d): %s", page, e)
|
||||||
|
return None
|
||||||
|
|
||||||
|
out: list[RawScene] = []
|
||||||
|
seen: set[str] = set()
|
||||||
|
anchors = list(_A_RE.finditer(text))
|
||||||
|
for idx, m in enumerate(anchors):
|
||||||
|
scene_url = m.group("url").replace("://www.", "://").rstrip("/") + "/"
|
||||||
|
if scene_url in seen:
|
||||||
|
continue
|
||||||
|
seen.add(scene_url)
|
||||||
|
title = html.unescape(m.group("title") or "").strip()
|
||||||
|
if not title:
|
||||||
|
continue
|
||||||
|
win_end = anchors[idx + 1].start() if idx + 1 < len(anchors) else m.end() + 900
|
||||||
|
window = text[m.start():win_end]
|
||||||
|
|
||||||
|
tm = _THUMB_RE.search(window)
|
||||||
|
thumb = tm.group(1) if tm else None
|
||||||
|
dm = _DUR_RE.search(window)
|
||||||
|
duration_sec = _parse_duration(dm.group(1) if dm else None)
|
||||||
|
|
||||||
|
fingerprints: list[RawFingerprint] = []
|
||||||
|
if thumb:
|
||||||
|
ph = compute_thumbnail_phash(thumb, referer=_BASE + "/")
|
||||||
|
if ph:
|
||||||
|
fingerprints.append(RawFingerprint(kind="phash", value=ph))
|
||||||
|
|
||||||
|
out.append(
|
||||||
|
RawScene(
|
||||||
|
external_id=f"{self.sitetag}:{scene_url}",
|
||||||
|
title=title,
|
||||||
|
duration_sec=duration_sec,
|
||||||
|
url=scene_url,
|
||||||
|
performers=[],
|
||||||
|
tags=[],
|
||||||
|
fingerprints=fingerprints,
|
||||||
|
playback_sources=[
|
||||||
|
RawPlaybackSource(
|
||||||
|
origin=f"tube:{self.sitetag}",
|
||||||
|
page_url=scene_url,
|
||||||
|
duration_sec=duration_sec,
|
||||||
|
thumbnail_url=thumb,
|
||||||
|
)
|
||||||
|
],
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
log.info("fpoxxx browse page %d: %d scenes", page, len(out))
|
||||||
|
return out
|
||||||
|
|
|
||||||
|
|
@ -5,8 +5,15 @@ Pilot #2 (po shyfap fail). Hipoteza: freshporno zachowuje oryginalne studio titl
|
||||||
do canonical zadziała. Bonus: channel = studio 1:1 (Pure Taboo, Brazzers, etc.).
|
do canonical zadziała. Bonus: channel = studio 1:1 (Pure Taboo, Brazzers, etc.).
|
||||||
|
|
||||||
URL patterns:
|
URL patterns:
|
||||||
- Listing: `/` (page 1), `/2/`, `/3/`, ... (last `/391/` w czasie pisania)
|
- Listing: `/latest-updates/` (page 1), `/latest-updates/2/`, ... (chronologiczny feed)
|
||||||
- Scene: `/videos/<slug>/`
|
- Scene: `/videos/<slug>/`
|
||||||
|
|
||||||
|
Listing: świadomie `/latest-updates/` zamiast roota `/`. Root jest KVS-owym
|
||||||
|
homepage z `cache-control: no-store` i świeżym PHPSESSID per-request — serwer
|
||||||
|
rotuje tam blok "featured" i na zimnej sesji potrafi podać stary zestaw zamiast
|
||||||
|
najnowszych (zaobserwowane 2026-06-15: 3 dni browse-latest skipowało wszystko bo
|
||||||
|
root podawał sceny z 20 maja; freshporno.org report). `/latest-updates/` to jawny
|
||||||
|
feed sortowany po dacie, odporny na tę rotację. Paginacja: `/latest-updates/N/`.
|
||||||
- Channels: `/channels/<slug>/` (= studio)
|
- Channels: `/channels/<slug>/` (= studio)
|
||||||
- Models: `/models/<slug>/` (= performer)
|
- Models: `/models/<slug>/` (= performer)
|
||||||
- Tags: `/tags/<slug>/` (= category)
|
- Tags: `/tags/<slug>/` (= category)
|
||||||
|
|
@ -61,8 +68,8 @@ class FreshpornoScraper(BaseBrowseScraper):
|
||||||
|
|
||||||
def _listing_url(self, page: int) -> str:
|
def _listing_url(self, page: int) -> str:
|
||||||
if page <= 1:
|
if page <= 1:
|
||||||
return f"{_BASE}/"
|
return f"{_BASE}/latest-updates/"
|
||||||
return f"{_BASE}/{page}/"
|
return f"{_BASE}/latest-updates/{page}/"
|
||||||
|
|
||||||
def _extract_scene_urls(self, listing_html: str) -> list[str]:
|
def _extract_scene_urls(self, listing_html: str) -> list[str]:
|
||||||
seen: set[str] = set()
|
seen: set[str] = set()
|
||||||
|
|
@ -163,11 +170,25 @@ class FreshpornoScraper(BaseBrowseScraper):
|
||||||
)
|
)
|
||||||
]
|
]
|
||||||
|
|
||||||
|
# Release date — freshporno emituje `<meta itemprop="uploadDate" content="2026-05-20T...">`.
|
||||||
|
# To data wrzucenia na freshporno, NIE oryginalna release_date studio — ale dla
|
||||||
|
# świeżych scen (uploaded niedługo po publikacji) różnica ≤ 3-7 dni, mieści się w
|
||||||
|
# `date_window_days=7` w resolverze. Bez tego pola scene NULL → match score 0 →
|
||||||
|
# duplicate scene zamiast freshporno PS dodane do TPDB canonical (bug-report
|
||||||
|
# 2026-05-20: brak Brazzers Exxtra po 15-05).
|
||||||
|
release_date_parsed: date | None = None
|
||||||
|
if (m := re.search(r'itemprop="uploadDate"[^>]+content="(\d{4}-\d{2}-\d{2})', detail_html)):
|
||||||
|
try:
|
||||||
|
release_date_parsed = date.fromisoformat(m.group(1))
|
||||||
|
except ValueError:
|
||||||
|
pass
|
||||||
|
|
||||||
return RawScene(
|
return RawScene(
|
||||||
external_id=f"{self.sitetag}:{scene_url}",
|
external_id=f"{self.sitetag}:{scene_url}",
|
||||||
title=title,
|
title=title,
|
||||||
description=description,
|
description=description,
|
||||||
duration_sec=duration_sec,
|
duration_sec=duration_sec,
|
||||||
|
release_date=release_date_parsed,
|
||||||
url=scene_url,
|
url=scene_url,
|
||||||
studio=studio,
|
studio=studio,
|
||||||
performers=performers,
|
performers=performers,
|
||||||
|
|
|
||||||
|
|
@ -21,6 +21,7 @@ from app.connectors.direct_scrapers._browse_base import (
|
||||||
compute_thumbnail_phash,
|
compute_thumbnail_phash,
|
||||||
meta_content,
|
meta_content,
|
||||||
)
|
)
|
||||||
|
from app.normalize.text import slugify
|
||||||
|
|
||||||
_BASE = "https://www.fullmovies.xxx"
|
_BASE = "https://www.fullmovies.xxx"
|
||||||
_SCENE_URL_RE = re.compile(r'href="(https://www\.fullmovies\.xxx/videos/[a-z0-9\-]+/)"', re.IGNORECASE)
|
_SCENE_URL_RE = re.compile(r'href="(https://www\.fullmovies\.xxx/videos/[a-z0-9\-]+/)"', re.IGNORECASE)
|
||||||
|
|
@ -68,23 +69,25 @@ class FullmoviesScraper(BaseBrowseScraper):
|
||||||
if dur_meta and dur_meta.isdigit():
|
if dur_meta and dur_meta.isdigit():
|
||||||
duration_sec = int(dur_meta)
|
duration_sec = int(dur_meta)
|
||||||
|
|
||||||
|
# Studio z PREFIKSU tytułu ("Studio - Scene Title"), nie z sidebara /networks/.
|
||||||
|
# Sidebar listuje WSZYSTKIE sieci → `_NETWORK_LINK_RE.finditer().first()` zawsze
|
||||||
|
# zwracał pierwszą z listy (Brazzers) dla każdej sceny — mis-attribution. Tytuł
|
||||||
|
# po oczyszczeniu ma format "Studio - Opis" (np. "Fake Hostel - ...").
|
||||||
studio: RawStudio | None = None
|
studio: RawStudio | None = None
|
||||||
for m in _NETWORK_LINK_RE.finditer(detail_html):
|
if " - " in title:
|
||||||
slug, name = m.group(1), m.group(2).strip()
|
studio_name = title.split(" - ", 1)[0].strip()
|
||||||
if name.lower() in ("networks", ""):
|
if studio_name and len(studio_name) <= 50:
|
||||||
continue
|
|
||||||
studio = RawStudio(
|
studio = RawStudio(
|
||||||
external_id=f"fullmoviesxxx:network:{slug}",
|
external_id=f"fullmoviesxxx:studio:{slugify(studio_name)}",
|
||||||
name=name,
|
name=studio_name,
|
||||||
slug=slug,
|
slug=slugify(studio_name),
|
||||||
)
|
)
|
||||||
break
|
|
||||||
|
|
||||||
performers: list[RawPerformer] = []
|
performers: list[RawPerformer] = []
|
||||||
seen_perf: set[str] = set()
|
seen_perf: set[str] = set()
|
||||||
for m in _MODEL_LINK_RE.finditer(detail_html):
|
for m in _MODEL_LINK_RE.finditer(detail_html):
|
||||||
slug, name = m.group(1), m.group(2).strip()
|
slug, name = m.group(1), m.group(2).strip()
|
||||||
if slug in seen_perf or name.lower() in ("pornstars", "models"):
|
if not name or slug in seen_perf or name.lower() in ("pornstars", "models"):
|
||||||
continue
|
continue
|
||||||
seen_perf.add(slug)
|
seen_perf.add(slug)
|
||||||
performers.append(
|
performers.append(
|
||||||
|
|
@ -100,11 +103,11 @@ class FullmoviesScraper(BaseBrowseScraper):
|
||||||
seen_tag.add(slug)
|
seen_tag.add(slug)
|
||||||
tags.append(RawTag(external_id=f"fullmoviesxxx:tag:{slug}", name=name, slug=slug))
|
tags.append(RawTag(external_id=f"fullmoviesxxx:tag:{slug}", name=name, slug=slug))
|
||||||
|
|
||||||
|
# Phash WYŁĄCZONY (pilot 2026-06-01: 0% trafień ≤5, mediana Hamming 14 do
|
||||||
|
# canonical — auto-screenshoty img.fullmovies.xxx, nie hot-linkowane studio
|
||||||
|
# thumbnaile). Matching trzyma się na title+performer+duration (seed: 92% tagged),
|
||||||
|
# więc download thumbnaila pod phash to czysty narzut. thumbnail_url zostaje (display).
|
||||||
fingerprints: list[RawFingerprint] = []
|
fingerprints: list[RawFingerprint] = []
|
||||||
if thumbnail_url:
|
|
||||||
ph = compute_thumbnail_phash(thumbnail_url, referer=_BASE + "/")
|
|
||||||
if ph:
|
|
||||||
fingerprints.append(RawFingerprint(kind="phash", value=ph))
|
|
||||||
|
|
||||||
playback_sources = [
|
playback_sources = [
|
||||||
RawPlaybackSource(
|
RawPlaybackSource(
|
||||||
|
|
|
||||||
|
|
@ -25,6 +25,7 @@ from app.connectors.direct_scrapers._browse_base import (
|
||||||
compute_thumbnail_phash,
|
compute_thumbnail_phash,
|
||||||
meta_content,
|
meta_content,
|
||||||
)
|
)
|
||||||
|
from app.normalize.text import slugify
|
||||||
|
|
||||||
_BASE = "https://www.hdporn.gg"
|
_BASE = "https://www.hdporn.gg"
|
||||||
_SCENE_URL_RE = re.compile(r'href="(https://www\.hdporn\.gg/videos/[a-z0-9\-]+/)"', re.IGNORECASE)
|
_SCENE_URL_RE = re.compile(r'href="(https://www\.hdporn\.gg/videos/[a-z0-9\-]+/)"', re.IGNORECASE)
|
||||||
|
|
@ -75,27 +76,24 @@ class HDPornGGScraper(BaseBrowseScraper):
|
||||||
if dur_meta and dur_meta.isdigit():
|
if dur_meta and dur_meta.isdigit():
|
||||||
duration_sec = int(dur_meta)
|
duration_sec = int(dur_meta)
|
||||||
|
|
||||||
# Studio z /networks/. Skip nav anchors typu "Networks" / "Pornstars".
|
# Studio z PREFIKSU tytułu ("Studio - Scene Title"), nie z sidebara /networks/.
|
||||||
|
# Sidebar listuje WSZYSTKIE sieci → pierwszy match zawsze ten sam (Brazzers) dla
|
||||||
|
# każdej sceny. Tytuł po oczyszczeniu ma format "Studio - Opis" (np. "Dad Crush - ...").
|
||||||
studio: RawStudio | None = None
|
studio: RawStudio | None = None
|
||||||
for m in _NETWORK_LINK_RE.finditer(detail_html):
|
if " - " in title:
|
||||||
slug, name = m.group(1), m.group(2).strip()
|
studio_name = title.split(" - ", 1)[0].strip()
|
||||||
if name.lower() in ("networks", ""):
|
if studio_name and len(studio_name) <= 50:
|
||||||
continue
|
|
||||||
# Pierwszy NETWORK link w body to studio sceny (nav sidebar też ma networks
|
|
||||||
# listę — bierzemy gdy `class="btn_sponsor_group"` lub po prostu pierwszy
|
|
||||||
# NIE z sidebara). hdporn.gg pokazuje btn_sponsor_group w main scene area.
|
|
||||||
studio = RawStudio(
|
studio = RawStudio(
|
||||||
external_id=f"hdporngg:network:{slug}",
|
external_id=f"hdporngg:studio:{slugify(studio_name)}",
|
||||||
name=name,
|
name=studio_name,
|
||||||
slug=slug,
|
slug=slugify(studio_name),
|
||||||
)
|
)
|
||||||
break
|
|
||||||
|
|
||||||
performers: list[RawPerformer] = []
|
performers: list[RawPerformer] = []
|
||||||
seen_perf: set[str] = set()
|
seen_perf: set[str] = set()
|
||||||
for m in _MODEL_LINK_RE.finditer(detail_html):
|
for m in _MODEL_LINK_RE.finditer(detail_html):
|
||||||
slug, name = m.group(1), m.group(2).strip()
|
slug, name = m.group(1), m.group(2).strip()
|
||||||
if slug in seen_perf or name.lower() in ("pornstars", "models"):
|
if not name or slug in seen_perf or name.lower() in ("pornstars", "models"):
|
||||||
continue
|
continue
|
||||||
seen_perf.add(slug)
|
seen_perf.add(slug)
|
||||||
performers.append(
|
performers.append(
|
||||||
|
|
@ -113,11 +111,11 @@ class HDPornGGScraper(BaseBrowseScraper):
|
||||||
RawTag(external_id=f"hdporngg:tag:{slug}", name=name, slug=slug)
|
RawTag(external_id=f"hdporngg:tag:{slug}", name=name, slug=slug)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Phash WYŁĄCZONY (pilot 2026-06-01: 0% trafień ≤5, mediana Hamming 14 do
|
||||||
|
# canonical — auto-screenshoty img.hdporn.gg, nie hot-linkowane studio thumbnaile).
|
||||||
|
# Matching trzyma się na title+performer+duration (seed: 92% tagged), więc download
|
||||||
|
# thumbnaila pod phash to czysty narzut. thumbnail_url zostaje (display).
|
||||||
fingerprints: list[RawFingerprint] = []
|
fingerprints: list[RawFingerprint] = []
|
||||||
if thumbnail_url:
|
|
||||||
ph = compute_thumbnail_phash(thumbnail_url, referer=_BASE + "/")
|
|
||||||
if ph:
|
|
||||||
fingerprints.append(RawFingerprint(kind="phash", value=ph))
|
|
||||||
|
|
||||||
playback_sources = [
|
playback_sources = [
|
||||||
RawPlaybackSource(
|
RawPlaybackSource(
|
||||||
|
|
|
||||||
28
app/connectors/direct_scrapers/hqfap.py
Normal file
28
app/connectors/direct_scrapers/hqfap.py
Normal file
|
|
@ -0,0 +1,28 @@
|
||||||
|
"""hqfap.com — latest-vids browse scraper (PlayTube CMS, patrz _playtube.py).
|
||||||
|
|
||||||
|
Dołączony 2026-06-10 (user request), usunięty 2026-06-25 (cała biblioteka CDN padła
|
||||||
|
na stub), przywrócony 2026-07-07 (strona wróciła na CDN vstor.top z realnymi plikami,
|
||||||
|
user request). Re-uploader katalogu pornhd.pet (~120k scen, thumbnaile to base64-encoded
|
||||||
|
oryginalne URL-e w `/uploads/images/`).
|
||||||
|
|
||||||
|
Specyfika vs baza: studio siedzi w kategoriach z suffixem " Clips"
|
||||||
|
("Filthy Kings Clips" → studio "Filthy Kings"); reszta kategorii → tagi.
|
||||||
|
Playback: direct mp4 z JSON-LD contentUrl (vstor.top obecnie, cdnde.com/okcdn.ru
|
||||||
|
starsze), tokeny time-bound i portable cross-IP → natywny extractor `hqfapcom`.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from app.connectors.direct_scrapers._playtube import BasePlayTubeScraper
|
||||||
|
|
||||||
|
|
||||||
|
class HQFapScraper(BasePlayTubeScraper):
|
||||||
|
sitetag = "hqfapcom"
|
||||||
|
base_url = "https://hqfap.com"
|
||||||
|
|
||||||
|
def _pick_studio(self, category_names: list[str]) -> str | None:
|
||||||
|
for name in category_names:
|
||||||
|
if name.lower().endswith(" clips"):
|
||||||
|
studio_name = name[: -len(" clips")].strip()
|
||||||
|
if studio_name:
|
||||||
|
return studio_name
|
||||||
|
return None
|
||||||
|
|
@ -47,8 +47,12 @@ class HQPornerScraper(BaseDirectTubeScraper):
|
||||||
log.debug("hqporner search %s status=%d", url, r.status_code)
|
log.debug("hqporner search %s status=%d", url, r.status_code)
|
||||||
return
|
return
|
||||||
|
|
||||||
# Filtr: slug musi zawierać przynajmniej jedno z słów query (case-insensitive)
|
# Filtr: slug musi zawierać WSZYSTKIE słowa query (≥3 znaki), case-insensitive.
|
||||||
# Eliminuje totalnie niezwiązane wyniki gdy fuzzy search szumi.
|
# Wcześniej `any` (≥1 token) → przy 2-słownych nazwach match na jednym pospolitym
|
||||||
|
# tokenie (np. "anna"/"mia") atrybutował scenę do KAŻDEGO performera dzielącego ten
|
||||||
|
# token → sceny z setkami błędnych aktorek (do 503; hqporner = 5659/5897 takich scen,
|
||||||
|
# bug-report 2026-06-07). `all` wymaga pełnego dopasowania nazwy → precyzja.
|
||||||
|
# Pojedyncze nazwy ("Belladonna") nadal działają (jeden token musi być).
|
||||||
query_tokens = {tok for tok in query.lower().split() if len(tok) >= 3}
|
query_tokens = {tok for tok in query.lower().split() if len(tok) >= 3}
|
||||||
|
|
||||||
seen_urls: set[str] = set()
|
seen_urls: set[str] = set()
|
||||||
|
|
@ -63,7 +67,7 @@ class HQPornerScraper(BaseDirectTubeScraper):
|
||||||
|
|
||||||
# Title-token filter
|
# Title-token filter
|
||||||
slug_lower = slug_part.lower()
|
slug_lower = slug_part.lower()
|
||||||
if query_tokens and not any(tok in slug_lower for tok in query_tokens):
|
if query_tokens and not all(tok in slug_lower for tok in query_tokens):
|
||||||
continue
|
continue
|
||||||
|
|
||||||
title = slug_part.replace("_", " ").replace("-", " ").strip()
|
title = slug_part.replace("_", " ").replace("-", " ").strip()
|
||||||
|
|
|
||||||
101
app/connectors/direct_scrapers/javflix.py
Normal file
101
app/connectors/direct_scrapers/javflix.py
Normal file
|
|
@ -0,0 +1,101 @@
|
||||||
|
"""javflix.cc — JAV browse scraper (WordPress, English-subbed JAV).
|
||||||
|
|
||||||
|
Osobna pula JAV (kody typu BKD-368, tytuły azjatyckie), origin `tube:javflix`.
|
||||||
|
NIE deduplikuje się z zachodnim katalogiem — świadomie orphan vertical (sekcja JAV).
|
||||||
|
|
||||||
|
Struktura (RE 2026-07-10):
|
||||||
|
- listing: `/page/N/` (WordPress archive), posty pod `/<slug>/` (np. /bkd-368-english-subtitle/)
|
||||||
|
- detail: metadane w `itemprop` (name/thumbnailUrl/uploadDate; BRAK duration),
|
||||||
|
playback = przyciski serwerów `<a class="myLink" name="<hoster>" href="<embed>">`
|
||||||
|
(streamtape/voe/doodstream/emturbovid). Href jest w SUROWYM HTML (theme strippuje go
|
||||||
|
po renderze JS, ale scraper widzi surowy). Embed-hostery obsługuje generyczny
|
||||||
|
ekstraktor `_embed_iframe` (anchor-hoster pattern) → rejestrujemy go pod `javflix`.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import html
|
||||||
|
import re
|
||||||
|
|
||||||
|
from app.connectors.base import RawPlaybackSource, RawScene, RawTag
|
||||||
|
from app.connectors.direct_scrapers._browse_base import BaseBrowseScraper
|
||||||
|
from app.connectors.direct_scrapers._playtube import _parse_iso_date
|
||||||
|
|
||||||
|
_BASE = "https://javflix.cc"
|
||||||
|
# Posty to `javflix.cc/<slug>/`. Odsiewamy strony nie-postowe (taksonomie, statyczne).
|
||||||
|
_POST_RE = re.compile(r'href="(https://javflix\.cc/[a-z0-9][a-z0-9\-]{4,}/)"', re.IGNORECASE)
|
||||||
|
_NON_POST = (
|
||||||
|
"/page/", "/category/", "/categories/", "/genre/", "/maker/", "/actress/",
|
||||||
|
"/actors/", "/tag/", "/tags/", "/studio/", "/label/", "/series/", "/wp-",
|
||||||
|
"/18-usc", "/dmca", "/contact", "/privacy", "/about", "/2257",
|
||||||
|
"/terms", "/faq", "/policy", "/disclaimer", "/sitemap",
|
||||||
|
)
|
||||||
|
_CODE_RE = re.compile(r"^([a-z]+-?\d+[a-z]?)", re.IGNORECASE)
|
||||||
|
|
||||||
|
|
||||||
|
def _itemprop(html: str, name: str) -> str | None:
|
||||||
|
m = re.search(
|
||||||
|
rf'itemprop="{name}"\s+content="([^"]+)"', html, re.IGNORECASE
|
||||||
|
)
|
||||||
|
return m.group(1).strip() if m else None
|
||||||
|
|
||||||
|
|
||||||
|
class JavflixScraper(BaseBrowseScraper):
|
||||||
|
sitetag = "javflix"
|
||||||
|
|
||||||
|
def _listing_url(self, page: int) -> str:
|
||||||
|
return f"{_BASE}/" if page <= 1 else f"{_BASE}/page/{page}/"
|
||||||
|
|
||||||
|
def _extract_scene_urls(self, listing_html: str) -> list[str]:
|
||||||
|
seen: set[str] = set()
|
||||||
|
out: list[str] = []
|
||||||
|
for m in _POST_RE.finditer(listing_html):
|
||||||
|
url = m.group(1)
|
||||||
|
if any(x in url for x in _NON_POST):
|
||||||
|
continue
|
||||||
|
if url.rstrip("/") == _BASE:
|
||||||
|
continue
|
||||||
|
if url not in seen:
|
||||||
|
seen.add(url)
|
||||||
|
out.append(url)
|
||||||
|
return out
|
||||||
|
|
||||||
|
def _parse_detail(self, scene_url: str, detail_html: str) -> RawScene | None:
|
||||||
|
# Prawdziwy post video ma przyciski serwerów (`class="myLink"`). Strony statyczne
|
||||||
|
# (Terms/FAQ/DMCA) ich nie mają → pomijamy (URL-filter nie łapie wszystkich).
|
||||||
|
if 'class="myLink"' not in detail_html:
|
||||||
|
return None
|
||||||
|
title = _itemprop(detail_html, "name")
|
||||||
|
if not title:
|
||||||
|
tm = re.search(r"<title>([^<]+)</title>", detail_html)
|
||||||
|
title = tm.group(1).split(" – ")[0].strip() if tm else None
|
||||||
|
if not title:
|
||||||
|
return None
|
||||||
|
title = html.unescape(title).strip()
|
||||||
|
|
||||||
|
thumb = _itemprop(detail_html, "thumbnailUrl")
|
||||||
|
up = _itemprop(detail_html, "uploadDate")
|
||||||
|
release_date = _parse_iso_date(up) if up else None
|
||||||
|
|
||||||
|
# JAV code (BKD-368) ze sluga — kanoniczny identyfikator, dodajemy jako tag
|
||||||
|
# (searchable) bo javflix nie ma osobnego pola kodu.
|
||||||
|
slug = scene_url.rstrip("/").rsplit("/", 1)[-1]
|
||||||
|
cm = _CODE_RE.match(slug)
|
||||||
|
tags: list[RawTag] = []
|
||||||
|
if cm:
|
||||||
|
code = cm.group(1).upper()
|
||||||
|
tags.append(RawTag(external_id=f"javcode:{code}", name=code, slug=cm.group(1).lower()))
|
||||||
|
|
||||||
|
return RawScene(
|
||||||
|
external_id=f"{self.sitetag}:{scene_url}",
|
||||||
|
title=title,
|
||||||
|
release_date=release_date,
|
||||||
|
url=scene_url,
|
||||||
|
tags=tags,
|
||||||
|
playback_sources=[
|
||||||
|
RawPlaybackSource(
|
||||||
|
origin=f"tube:{self.sitetag}",
|
||||||
|
page_url=scene_url,
|
||||||
|
thumbnail_url=thumb,
|
||||||
|
)
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
@ -1,19 +1,180 @@
|
||||||
"""latestpornvideo.com — direct HTML scrape.
|
"""latestpornvideo.com — latest-vids browse scraper.
|
||||||
|
|
||||||
Search: `https://latestpornvideo.com/page/<n>/?s=<q>`.
|
Historia: dawniej performer-driven search scraper (`/actor/<slug>/`). Problem
|
||||||
Scene URL: `https://latestpornvideo.com/<slug>/`.
|
(user-report 1da0375e): search-scraper ingestuje TYLKO sceny performerów, których
|
||||||
|
akurat szukamy → feed strony "latest" nigdy nie wpada, w apce widać stary zestaw,
|
||||||
|
a na stronie jest świeży. Przerobione na BROWSE (latest chronologicznie z
|
||||||
|
`/page/<n>/`, page 1 = `/`), 2026-06-22.
|
||||||
|
|
||||||
|
Listing card (zero detail-fetchy — detail page nie ma performerów ani duration):
|
||||||
|
<article class="... post-<id> ... category-<cat> tag-<x> tag-<y> ...">
|
||||||
|
<a href="https://latestpornvideo.com/<id>/" title="<Tytuł>">
|
||||||
|
data-main-thumb="<Studio>-YYYY-MM-DD-...-cover.jpg"
|
||||||
|
→ tytuł, miniatura, studio+release_date (z nazwy thumba albo z tytułu
|
||||||
|
"<Studio> YY MM DD ..."). Performerzy: listing ICH NIE MA czysto
|
||||||
|
(homepage karty bez `actors-*`, jak na stronach /actor/), a `tag-*` miesza
|
||||||
|
fragmenty imion z gatunkami → NIE ufamy tagom jako performerom; performera
|
||||||
|
dorabia canonical-merge po tytule+duration. Tagi bierzemy ostrożnie.
|
||||||
|
|
||||||
|
Playback: luluvid (filemoon family) iframe → extractor `latestpornvideocom`
|
||||||
|
(_embed_iframe → type='hoster'), telefon resolwuje phone-side. page_url = /<id>/.
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import html
|
||||||
|
import logging
|
||||||
import re
|
import re
|
||||||
|
from datetime import date
|
||||||
|
|
||||||
from app.connectors.direct_scrapers._search_base import BaseSearchScraper
|
from app.connectors.base import (
|
||||||
|
RawPlaybackSource,
|
||||||
|
RawScene,
|
||||||
class LatestPornVideoScraper(BaseSearchScraper):
|
RawStudio,
|
||||||
sitetag = "latestpornvideocom"
|
RawTag,
|
||||||
_search_url_template = "https://latestpornvideo.com/page/{page}/?s={query}"
|
|
||||||
_scene_url_re = re.compile(
|
|
||||||
r'href="(?P<url>https://latestpornvideo\.com/(?P<slug>[a-z0-9][a-z0-9\-]+))/"',
|
|
||||||
re.IGNORECASE,
|
|
||||||
)
|
)
|
||||||
|
from app.connectors.direct_scrapers._browse_base import (
|
||||||
|
BaseBrowseScraper,
|
||||||
|
compute_thumbnail_phash,
|
||||||
|
)
|
||||||
|
from app.extractors import browser_get
|
||||||
|
from app.normalize.text import slugify
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
_BASE = "https://latestpornvideo.com"
|
||||||
|
_ARTICLE_RE = re.compile(r'<article[^>]*\bclass="([^"]+)"', re.IGNORECASE)
|
||||||
|
_LINK_RE = re.compile(r'<a\s+href="([^"]+)"\s+title="([^"]+)"', re.IGNORECASE)
|
||||||
|
_THUMB_RE = re.compile(r'data-main-thumb="([^"]+)"', re.IGNORECASE)
|
||||||
|
_CLASS_TAG_RE = re.compile(r"\btag-([a-z0-9-]+)")
|
||||||
|
_CLASS_CAT_RE = re.compile(r"\bcategory-([a-z0-9-]+)")
|
||||||
|
# Nazwa thumba: `<Studio>-YYYY-MM-DD-<rest>-cover.jpg`.
|
||||||
|
_THUMB_NAME_RE = re.compile(r"/([A-Za-z0-9][A-Za-z0-9-]*?)-(\d{4})-(\d{2})-(\d{2})-", re.IGNORECASE)
|
||||||
|
# Tytuł: `<Studio> YY MM DD <rest>` (np. "MySexMobile 20 10 23 Abella Danger").
|
||||||
|
_TITLE_DATE_RE = re.compile(r"^(.*?)\s*\b(\d{2})\s+(\d{2})\s+(\d{2})\b")
|
||||||
|
# Karty homepage zawsze siedzą w kategorii "latest-porn-videos" — to nie jest tag.
|
||||||
|
_CAT_SKIP = {"latest-porn-videos", "uncategorized", ""}
|
||||||
|
|
||||||
|
|
||||||
|
def _name_from_slug(slug: str) -> str:
|
||||||
|
return " ".join(w.capitalize() for w in slug.split("-") if w)
|
||||||
|
|
||||||
|
|
||||||
|
class LatestPornVideoScraper(BaseBrowseScraper):
|
||||||
|
sitetag = "latestpornvideocom"
|
||||||
|
|
||||||
|
def _listing_url(self, page: int) -> str:
|
||||||
|
return _BASE + "/" if page <= 1 else f"{_BASE}/page/{page}/"
|
||||||
|
|
||||||
|
# crawl_page nadpisany → poniższe abstrakcje nieużywane, ale wymagane do instancji.
|
||||||
|
def _extract_scene_urls(self, listing_html: str) -> list[str]:
|
||||||
|
return [m.group(1) for m in _LINK_RE.finditer(listing_html)]
|
||||||
|
|
||||||
|
def _parse_detail(self, scene_url: str, detail_html: str) -> RawScene | None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def crawl_page(self, page: int) -> list[RawScene] | None:
|
||||||
|
url = self._listing_url(page)
|
||||||
|
try:
|
||||||
|
res = browser_get(url, timeout=self._timeout)
|
||||||
|
text = res.text if hasattr(res, "text") else res
|
||||||
|
except Exception as e:
|
||||||
|
log.warning("latestpornvideo browse listing fetch failed (page %d): %s", page, e)
|
||||||
|
return None
|
||||||
|
|
||||||
|
anchors = list(_ARTICLE_RE.finditer(text))
|
||||||
|
out: list[RawScene] = []
|
||||||
|
seen: set[str] = set()
|
||||||
|
for idx, m in enumerate(anchors):
|
||||||
|
cls = m.group(1)
|
||||||
|
win_end = anchors[idx + 1].start() if idx + 1 < len(anchors) else m.end() + 1500
|
||||||
|
window = text[m.start():win_end]
|
||||||
|
|
||||||
|
link_m = _LINK_RE.search(window)
|
||||||
|
if not link_m:
|
||||||
|
continue
|
||||||
|
scene_url = link_m.group(1).rstrip("/") + "/"
|
||||||
|
# tylko właściwe posty scen (/<digits>/), bez nav/kategorii
|
||||||
|
if not re.fullmatch(rf"{re.escape(_BASE)}/\d+/", scene_url) or scene_url in seen:
|
||||||
|
continue
|
||||||
|
seen.add(scene_url)
|
||||||
|
title = html.unescape(link_m.group(2)).strip()
|
||||||
|
if not title:
|
||||||
|
continue
|
||||||
|
|
||||||
|
thumb_m = _THUMB_RE.search(window)
|
||||||
|
thumb = thumb_m.group(1) if thumb_m else None
|
||||||
|
|
||||||
|
# Studio + release_date z nazwy thumba (`<Studio>-YYYY-MM-DD-`).
|
||||||
|
studio: RawStudio | None = None
|
||||||
|
release_date: date | None = None
|
||||||
|
if thumb and (tn := _THUMB_NAME_RE.search(thumb)):
|
||||||
|
studio_raw = tn.group(1).replace("-", " ").strip()
|
||||||
|
if studio_raw:
|
||||||
|
studio = RawStudio(
|
||||||
|
external_id=f"{self.sitetag}:studio:{slugify(studio_raw)}",
|
||||||
|
name=studio_raw, slug=slugify(studio_raw),
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
release_date = date(int(tn.group(2)), int(tn.group(3)), int(tn.group(4)))
|
||||||
|
except ValueError:
|
||||||
|
release_date = None
|
||||||
|
|
||||||
|
# Fallback z tytułu: `<Studio> YY MM DD ...`.
|
||||||
|
if studio is None or release_date is None:
|
||||||
|
if tm2 := _TITLE_DATE_RE.search(title):
|
||||||
|
if release_date is None:
|
||||||
|
try:
|
||||||
|
release_date = date(
|
||||||
|
2000 + int(tm2.group(2)), int(tm2.group(3)), int(tm2.group(4))
|
||||||
|
)
|
||||||
|
except ValueError:
|
||||||
|
release_date = None
|
||||||
|
studio_raw = tm2.group(1).strip(" -–")
|
||||||
|
if studio is None and 2 <= len(studio_raw) <= 30:
|
||||||
|
studio = RawStudio(
|
||||||
|
external_id=f"{self.sitetag}:studio:{slugify(studio_raw)}",
|
||||||
|
name=studio_raw, slug=slugify(studio_raw),
|
||||||
|
)
|
||||||
|
|
||||||
|
# Tagi: tylko prawdziwe kategorie (category-*), bez "latest-porn-videos".
|
||||||
|
# `tag-*` POMIJAMY — to mieszanka fragmentów imion performerów i gatunków,
|
||||||
|
# bez `actors-*` (jak na /actor/) nie da się ich rozdzielić → byłby szum.
|
||||||
|
tags: list[RawTag] = []
|
||||||
|
seen_tag: set[str] = set()
|
||||||
|
for cm in _CLASS_CAT_RE.finditer(cls):
|
||||||
|
sl = cm.group(1)
|
||||||
|
if sl in _CAT_SKIP or sl in seen_tag:
|
||||||
|
continue
|
||||||
|
seen_tag.add(sl)
|
||||||
|
tags.append(RawTag(external_id=f"{self.sitetag}:tag:{sl}", name=_name_from_slug(sl), slug=sl))
|
||||||
|
|
||||||
|
fingerprints = []
|
||||||
|
if thumb:
|
||||||
|
ph = compute_thumbnail_phash(thumb, referer=_BASE + "/")
|
||||||
|
if ph:
|
||||||
|
from app.connectors.base import RawFingerprint
|
||||||
|
|
||||||
|
fingerprints.append(RawFingerprint(kind="phash", value=ph))
|
||||||
|
|
||||||
|
out.append(
|
||||||
|
RawScene(
|
||||||
|
external_id=f"{self.sitetag}:{scene_url}",
|
||||||
|
title=title,
|
||||||
|
release_date=release_date,
|
||||||
|
url=scene_url,
|
||||||
|
studio=studio,
|
||||||
|
performers=[],
|
||||||
|
tags=tags,
|
||||||
|
fingerprints=fingerprints,
|
||||||
|
playback_sources=[
|
||||||
|
RawPlaybackSource(
|
||||||
|
origin=f"tube:{self.sitetag}",
|
||||||
|
page_url=scene_url,
|
||||||
|
thumbnail_url=thumb,
|
||||||
|
)
|
||||||
|
],
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
log.info("latestpornvideo browse page %d: %d scenes", page, len(out))
|
||||||
|
return out
|
||||||
|
|
|
||||||
|
|
@ -1,19 +1,144 @@
|
||||||
"""mypornerleak.com — direct HTML scrape.
|
"""mypornerleak.com — performer-page listing scrape (search-based, performer-driven).
|
||||||
|
|
||||||
Search: `https://mypornerleak.com/page/<n>/?s=<q>`.
|
2026-06-16 fix (zamrożony od 05-07): treść serwowana jest pod subdomeną
|
||||||
Scene URL: `https://mypornerleak.com/<slug>/`.
|
`w8.mypornerleak.com` (load-balancer wN.), a stary regex szukał scen na gołym
|
||||||
|
`mypornerleak.com/<slug>/` → 0 trafień. Kanoniczny host i tak serwuje stronę
|
||||||
|
listingu i sceny, więc fetchujemy `mypornerleak.com/actor/<slug>/` a linki scen
|
||||||
|
normalizujemy wN.→ kanoniczny (stabilny page_url/dedup).
|
||||||
|
|
||||||
|
Wszystko z LISTINGU (bez detail-fetcha) — karta `<article>` koduje metadane:
|
||||||
|
- klasa `actors-<slug>` (multi) → performerzy; `category-<slug>` (multi) → tagi
|
||||||
|
- `<a href title="...">` → URL sceny + czysty tytuł
|
||||||
|
- `<span class="duration">MM:SS|HH:MM:SS`
|
||||||
|
- `<img data-src>` → thumbnail
|
||||||
|
|
||||||
|
Bez studia: mypornerleak to repost leaków OnlyFans/amatorskich (brak studyjnego
|
||||||
|
źródła). Playback przez extractor `mypornerleakcom` (_embed_iframe → hoster).
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import html
|
||||||
|
import logging
|
||||||
import re
|
import re
|
||||||
|
from collections.abc import Iterator
|
||||||
|
|
||||||
|
from app.connectors.base import RawPerformer, RawPlaybackSource, RawScene, RawTag
|
||||||
from app.connectors.direct_scrapers._search_base import BaseSearchScraper
|
from app.connectors.direct_scrapers._search_base import BaseSearchScraper
|
||||||
|
from app.extractors import browser_get
|
||||||
|
from app.normalize.text import slugify
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
_BASE = "https://mypornerleak.com"
|
||||||
|
_ARTICLE_RE = re.compile(r'<article[^>]*\bclass="([^"]+)"', re.IGNORECASE)
|
||||||
|
_LINK_RE = re.compile(r'<a\s+href="([^"]+)"\s+title="([^"]+)"', re.IGNORECASE)
|
||||||
|
_DURATION_RE = re.compile(r'class="duration">(?:<[^>]+>)*\s*(\d{1,2}:\d{2}(?::\d{2})?)', re.IGNORECASE)
|
||||||
|
_THUMB_RE = re.compile(r'data-src="([^"]+)"', re.IGNORECASE)
|
||||||
|
_WN_HOST_RE = re.compile(r"https?://w\d+\.mypornerleak\.com", re.IGNORECASE)
|
||||||
|
_CLASS_ACTOR_RE = re.compile(r"\bactors-([a-z0-9-]+)")
|
||||||
|
_CLASS_CAT_RE = re.compile(r"\bcategory-([a-z0-9-]+)")
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_duration(s: str) -> int | None:
|
||||||
|
parts = s.split(":")
|
||||||
|
try:
|
||||||
|
if len(parts) == 2:
|
||||||
|
return int(parts[0]) * 60 + int(parts[1])
|
||||||
|
if len(parts) == 3:
|
||||||
|
return int(parts[0]) * 3600 + int(parts[1]) * 60 + int(parts[2])
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _name_from_slug(slug: str) -> str:
|
||||||
|
return " ".join(w.capitalize() for w in slug.split("-") if w)
|
||||||
|
|
||||||
|
|
||||||
class MyPornerLeakScraper(BaseSearchScraper):
|
class MyPornerLeakScraper(BaseSearchScraper):
|
||||||
sitetag = "mypornerleakcom"
|
sitetag = "mypornerleakcom"
|
||||||
_search_url_template = "https://mypornerleak.com/page/{page}/?s={query}"
|
|
||||||
_scene_url_re = re.compile(
|
def search(
|
||||||
r'href="(?P<url>https://mypornerleak\.com/(?P<slug>[a-z0-9][a-z0-9\-]+))/"',
|
self, query: str, *, page: int = 1, limit: int | None = None
|
||||||
re.IGNORECASE,
|
) -> Iterator[RawScene]:
|
||||||
|
actor_slug = slugify(query)
|
||||||
|
if not actor_slug:
|
||||||
|
return
|
||||||
|
url = f"{_BASE}/actor/{actor_slug}/" + (f"page/{page}/" if page > 1 else "")
|
||||||
|
try:
|
||||||
|
r = browser_get(url, timeout=self._timeout)
|
||||||
|
except Exception as e:
|
||||||
|
log.warning("mypornerleak actor-page fetch failed (%s): %s", url, e)
|
||||||
|
return
|
||||||
|
if r.status_code != 200:
|
||||||
|
return
|
||||||
|
|
||||||
|
text = r.text
|
||||||
|
anchors = list(_ARTICLE_RE.finditer(text))
|
||||||
|
seen: set[str] = set()
|
||||||
|
yielded = 0
|
||||||
|
for idx, m in enumerate(anchors):
|
||||||
|
cls = m.group(1)
|
||||||
|
win_end = anchors[idx + 1].start() if idx + 1 < len(anchors) else m.end() + 1500
|
||||||
|
window = text[m.start():win_end]
|
||||||
|
|
||||||
|
link_m = _LINK_RE.search(window)
|
||||||
|
if not link_m:
|
||||||
|
continue
|
||||||
|
scene_url = _WN_HOST_RE.sub(_BASE, link_m.group(1)).rstrip("/") + "/"
|
||||||
|
if scene_url in seen:
|
||||||
|
continue
|
||||||
|
seen.add(scene_url)
|
||||||
|
title = html.unescape(link_m.group(2)).strip()
|
||||||
|
if not title:
|
||||||
|
continue
|
||||||
|
|
||||||
|
dur_m = _DURATION_RE.search(window)
|
||||||
|
duration_sec = _parse_duration(dur_m.group(1)) if dur_m else None
|
||||||
|
thumb_m = _THUMB_RE.search(window)
|
||||||
|
thumb = thumb_m.group(1) if thumb_m else None
|
||||||
|
|
||||||
|
# Performerzy + tagi z klasy <article>.
|
||||||
|
performers: list[RawPerformer] = []
|
||||||
|
seen_perf: set[str] = set()
|
||||||
|
for am in _CLASS_ACTOR_RE.finditer(cls):
|
||||||
|
sl = am.group(1)
|
||||||
|
if sl in seen_perf:
|
||||||
|
continue
|
||||||
|
seen_perf.add(sl)
|
||||||
|
performers.append(
|
||||||
|
RawPerformer(external_id=f"{self.sitetag}:performer:{sl}", name=_name_from_slug(sl))
|
||||||
)
|
)
|
||||||
|
if not performers:
|
||||||
|
performers.append(
|
||||||
|
RawPerformer(external_id=f"{self.sitetag}:performer:{actor_slug}", name=query.strip())
|
||||||
|
)
|
||||||
|
|
||||||
|
tags: list[RawTag] = []
|
||||||
|
seen_tag: set[str] = set()
|
||||||
|
for cm in _CLASS_CAT_RE.finditer(cls):
|
||||||
|
sl = re.sub(r"-(porn|leaks?|videos?)$", "", cm.group(1))
|
||||||
|
if not sl or sl in seen_tag:
|
||||||
|
continue
|
||||||
|
seen_tag.add(sl)
|
||||||
|
tags.append(RawTag(external_id=f"{self.sitetag}:tag:{sl}", name=_name_from_slug(sl), slug=sl))
|
||||||
|
|
||||||
|
yield RawScene(
|
||||||
|
external_id=f"{self.sitetag}:{scene_url}",
|
||||||
|
title=title,
|
||||||
|
duration_sec=duration_sec,
|
||||||
|
url=scene_url,
|
||||||
|
performers=performers,
|
||||||
|
tags=tags,
|
||||||
|
playback_sources=[
|
||||||
|
RawPlaybackSource(
|
||||||
|
origin=f"tube:{self.sitetag}",
|
||||||
|
page_url=scene_url,
|
||||||
|
duration_sec=duration_sec,
|
||||||
|
thumbnail_url=thumb,
|
||||||
|
)
|
||||||
|
],
|
||||||
|
)
|
||||||
|
yielded += 1
|
||||||
|
if limit is not None and yielded >= limit:
|
||||||
|
return
|
||||||
|
|
|
||||||
150
app/connectors/direct_scrapers/mypornerleak_browse.py
Normal file
150
app/connectors/direct_scrapers/mypornerleak_browse.py
Normal file
|
|
@ -0,0 +1,150 @@
|
||||||
|
"""mypornerleak.com — latest BROWSE scraper via WordPress REST API, obok search scrapera.
|
||||||
|
|
||||||
|
MyPornerLeakScraper (search) zostaje w ALL_DIRECT_SCRAPERS; ten browse dokłada
|
||||||
|
świeżość wprost z WP REST (`/wp-json/wp/v2/posts?_embed=1`). W odróżnieniu od
|
||||||
|
perverzija/porndish, mypornerleak WYSTAWIA custom taksonomię `actors` w REST →
|
||||||
|
mamy też performerów (nie tylko studio z `category` + tagi z `post_tag`).
|
||||||
|
|
||||||
|
Playback: post page embeduje hoster iframe → extractor `mypornerleakcom` →
|
||||||
|
`_embed_iframe`, resolwowany phone-side (bez zmian).
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import html
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
from datetime import date, datetime
|
||||||
|
|
||||||
|
from app.connectors.base import (
|
||||||
|
RawFingerprint,
|
||||||
|
RawPerformer,
|
||||||
|
RawPlaybackSource,
|
||||||
|
RawScene,
|
||||||
|
RawStudio,
|
||||||
|
RawTag,
|
||||||
|
)
|
||||||
|
from app.connectors.direct_scrapers._browse_base import (
|
||||||
|
BaseBrowseScraper,
|
||||||
|
compute_thumbnail_phash,
|
||||||
|
)
|
||||||
|
from app.extractors import browser_get
|
||||||
|
from app.normalize.text import slugify
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
_BASE = "https://mypornerleak.com"
|
||||||
|
_PER_PAGE = 20
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_date(value: str | None) -> date | None:
|
||||||
|
if not value:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
return datetime.fromisoformat(value.replace("Z", "+00:00")).date()
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
class MyPornerLeakBrowseScraper(BaseBrowseScraper):
|
||||||
|
sitetag = "mypornerleakcom"
|
||||||
|
|
||||||
|
def _listing_url(self, page: int) -> str:
|
||||||
|
return f"{_BASE}/wp-json/wp/v2/posts?per_page={_PER_PAGE}&page={page}&_embed=1"
|
||||||
|
|
||||||
|
def _extract_scene_urls(self, listing_html: str) -> list[str]:
|
||||||
|
return []
|
||||||
|
|
||||||
|
def _parse_detail(self, scene_url: str, detail_html: str) -> RawScene | None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def crawl_page(self, page: int) -> list[RawScene] | None:
|
||||||
|
url = self._listing_url(page)
|
||||||
|
try:
|
||||||
|
res = browser_get(url, timeout=self._timeout)
|
||||||
|
except Exception as e:
|
||||||
|
log.warning("mypornerleak REST fetch failed (page %d): %s", page, e)
|
||||||
|
return None
|
||||||
|
if res.status_code != 200:
|
||||||
|
return []
|
||||||
|
try:
|
||||||
|
posts = json.loads(res.text)
|
||||||
|
except (json.JSONDecodeError, ValueError):
|
||||||
|
log.warning("mypornerleak REST: bad JSON page %d", page)
|
||||||
|
return None
|
||||||
|
if not isinstance(posts, list) or not posts:
|
||||||
|
return []
|
||||||
|
|
||||||
|
out: list[RawScene] = []
|
||||||
|
for p in posts:
|
||||||
|
link = (p.get("link") or "").strip()
|
||||||
|
title = html.unescape((p.get("title") or {}).get("rendered", "")).strip()
|
||||||
|
if not link or not title:
|
||||||
|
continue
|
||||||
|
release_date = _parse_date(p.get("date"))
|
||||||
|
|
||||||
|
emb = p.get("_embedded") or {}
|
||||||
|
fm = emb.get("wp:featuredmedia") or []
|
||||||
|
thumb = (fm[0].get("source_url") if fm and isinstance(fm[0], dict) else None) or None
|
||||||
|
|
||||||
|
studio: RawStudio | None = None
|
||||||
|
tags: list[RawTag] = []
|
||||||
|
performers: list[RawPerformer] = []
|
||||||
|
seen_tag: set[str] = set()
|
||||||
|
seen_perf: set[str] = set()
|
||||||
|
for group in emb.get("wp:term") or []:
|
||||||
|
if not group:
|
||||||
|
continue
|
||||||
|
tax = group[0].get("taxonomy")
|
||||||
|
if tax == "category" and studio is None:
|
||||||
|
sname = (group[0].get("name") or "").strip()
|
||||||
|
if sname:
|
||||||
|
studio = RawStudio(
|
||||||
|
external_id=f"{self.sitetag}:studio:{slugify(sname)}",
|
||||||
|
name=sname, slug=slugify(sname),
|
||||||
|
)
|
||||||
|
elif tax == "actors":
|
||||||
|
for g in group:
|
||||||
|
name = (g.get("name") or "").strip()
|
||||||
|
sl = slugify(name)
|
||||||
|
if not name or sl in seen_perf:
|
||||||
|
continue
|
||||||
|
seen_perf.add(sl)
|
||||||
|
performers.append(
|
||||||
|
RawPerformer(external_id=f"{self.sitetag}:performer:{sl}", name=name)
|
||||||
|
)
|
||||||
|
elif tax == "post_tag":
|
||||||
|
for g in group:
|
||||||
|
name = (g.get("name") or "").strip()
|
||||||
|
sl = (g.get("slug") or slugify(name)).strip()
|
||||||
|
if not name or sl in seen_tag:
|
||||||
|
continue
|
||||||
|
seen_tag.add(sl)
|
||||||
|
tags.append(RawTag(external_id=f"{self.sitetag}:tag:{sl}", name=name, slug=sl))
|
||||||
|
|
||||||
|
fingerprints: list[RawFingerprint] = []
|
||||||
|
if thumb:
|
||||||
|
ph = compute_thumbnail_phash(thumb, referer=_BASE + "/")
|
||||||
|
if ph:
|
||||||
|
fingerprints.append(RawFingerprint(kind="phash", value=ph))
|
||||||
|
|
||||||
|
out.append(
|
||||||
|
RawScene(
|
||||||
|
external_id=f"{self.sitetag}:{link}",
|
||||||
|
title=title,
|
||||||
|
release_date=release_date,
|
||||||
|
url=link,
|
||||||
|
studio=studio,
|
||||||
|
performers=performers,
|
||||||
|
tags=tags,
|
||||||
|
fingerprints=fingerprints,
|
||||||
|
playback_sources=[
|
||||||
|
RawPlaybackSource(
|
||||||
|
origin=f"tube:{self.sitetag}",
|
||||||
|
page_url=link,
|
||||||
|
thumbnail_url=thumb,
|
||||||
|
)
|
||||||
|
],
|
||||||
|
)
|
||||||
|
)
|
||||||
|
log.info("mypornerleak REST page %d: %d scenes", page, len(out))
|
||||||
|
return out
|
||||||
191
app/connectors/direct_scrapers/neporn.py
Normal file
191
app/connectors/direct_scrapers/neporn.py
Normal file
|
|
@ -0,0 +1,191 @@
|
||||||
|
"""neporn.com — latest-vids browse scraper (KVS engine).
|
||||||
|
|
||||||
|
Dołączony 2026-06-10 (user request). Paysite re-upload (HardX, DAP itp.),
|
||||||
|
~40k+ scen po numeracji video id.
|
||||||
|
|
||||||
|
Sygnały per scena:
|
||||||
|
- JSON-LD VideoObject: name, description, uploadDate, thumbnailUrl
|
||||||
|
(uwaga: bywa zmanglowany "https:https://..." — normalizujemy)
|
||||||
|
- `<meta property="video:duration">` = sekundy wprost
|
||||||
|
- Performerzy: `<a class="link" href="https://neporn.com/models/<slug>/">`
|
||||||
|
(nazwa w spanie z ikoną — bierzemy slug i tytułujemy). Świeże uploady NIE mają
|
||||||
|
jeszcze linków modelek — fallback: flashvars `video_tags` (na świeżych scenach
|
||||||
|
to czyste nazwiska, np. "emily willis, gianna dior"); bierzemy wpisy 2-3 słowa
|
||||||
|
obecne w tytule i bez generycznych fraz (stoplist), bo starsze sceny mają tam
|
||||||
|
też szum ("deep throat", "natural tits", "hd porn").
|
||||||
|
- Kategorie: linki `/categories/<slug>/` w bloku info (tagi z `/tags/` to szum
|
||||||
|
typu "hd porn", "2020" — pomijamy)
|
||||||
|
|
||||||
|
Studio: brak strukturalnego pola — tytuł bywa "- HardX Update - ..." ale format
|
||||||
|
niespójny, zostawiamy fuzzy matchowi po tytule.
|
||||||
|
|
||||||
|
Listing: standard KVS `/latest-updates/N/` (24 sceny/strona).
|
||||||
|
Playback: KVS function/0 + license → `_kvs.resolve_kvs` server-side; finalny
|
||||||
|
`data001.neporn.com/remote_control.php?time=...` portable cross-IP
|
||||||
|
(test 2026-06-10: VPS resolve → lokalny ISP 206 video/mp4).
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import re
|
||||||
|
|
||||||
|
from app.connectors.base import (
|
||||||
|
RawFingerprint,
|
||||||
|
RawPerformer,
|
||||||
|
RawPlaybackSource,
|
||||||
|
RawScene,
|
||||||
|
RawTag,
|
||||||
|
)
|
||||||
|
from app.connectors.direct_scrapers._browse_base import (
|
||||||
|
BaseBrowseScraper,
|
||||||
|
compute_thumbnail_phash,
|
||||||
|
meta_content,
|
||||||
|
)
|
||||||
|
from app.connectors.direct_scrapers._playtube import (
|
||||||
|
_extract_video_object,
|
||||||
|
_parse_iso_date,
|
||||||
|
)
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
_BASE = "https://neporn.com"
|
||||||
|
|
||||||
|
_SCENE_URL_RE = re.compile(r'href="(https://neporn\.com/video/(\d+)/[^"]+)"')
|
||||||
|
_MODEL_LINK_RE = re.compile(r'href="https://neporn\.com/models/([a-z0-9\-]+)/"', re.IGNORECASE)
|
||||||
|
_CATEGORY_LINK_RE = re.compile(
|
||||||
|
r'href="https://neporn\.com/categories/([a-z0-9\-]+)/"\s*>\s*([^<]+?)\s*<', re.IGNORECASE
|
||||||
|
)
|
||||||
|
_VIDEO_TAGS_RE = re.compile(r"video_tags:\s*'([^']*)'")
|
||||||
|
|
||||||
|
# Słowa dyskwalifikujące wpis z video_tags jako nazwisko (generyczne frazy typu
|
||||||
|
# "deep throat" / "natural tits" przechodzą test "jest w tytule" zbyt często).
|
||||||
|
_PERF_STOPWORDS = frozenset(
|
||||||
|
"porn sex tits ass anal throat cum blow blowjob dick cock pussy fuck fucking "
|
||||||
|
"scene scenes hd milf teen big small double penetration facial creampie "
|
||||||
|
"threesome amateur petite latina blonde brunette".split()
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _name_from_slug(slug: str) -> str:
|
||||||
|
"""`emily-willis` → `Emily Willis`."""
|
||||||
|
return " ".join(w.capitalize() for w in slug.split("-") if w)
|
||||||
|
|
||||||
|
|
||||||
|
class NepornScraper(BaseBrowseScraper):
|
||||||
|
sitetag = "neporncom"
|
||||||
|
|
||||||
|
def _listing_url(self, page: int) -> str:
|
||||||
|
if page <= 1:
|
||||||
|
return f"{_BASE}/latest-updates/"
|
||||||
|
return f"{_BASE}/latest-updates/{page}/"
|
||||||
|
|
||||||
|
def _extract_scene_urls(self, listing_html: str) -> list[str]:
|
||||||
|
seen: set[str] = set()
|
||||||
|
out: list[str] = []
|
||||||
|
for m in _SCENE_URL_RE.finditer(listing_html):
|
||||||
|
url = m.group(1)
|
||||||
|
if url in seen:
|
||||||
|
continue
|
||||||
|
seen.add(url)
|
||||||
|
out.append(url)
|
||||||
|
return out
|
||||||
|
|
||||||
|
def _parse_detail(self, scene_url: str, detail_html: str) -> RawScene | None:
|
||||||
|
video = _extract_video_object(detail_html) or {}
|
||||||
|
|
||||||
|
title = (video.get("name") or meta_content(detail_html, property="og:title") or "").strip()
|
||||||
|
# Tytuły bywają z wiszącym separatorem na początku ("- HardX Update - ...").
|
||||||
|
title = title.lstrip("- ").strip()
|
||||||
|
if not title:
|
||||||
|
return None
|
||||||
|
|
||||||
|
id_m = re.search(r"/video/(\d+)/", scene_url)
|
||||||
|
video_id = id_m.group(1) if id_m else None
|
||||||
|
|
||||||
|
duration_sec: int | None = None
|
||||||
|
dur_meta = meta_content(detail_html, property="video:duration")
|
||||||
|
if dur_meta and dur_meta.isdigit():
|
||||||
|
duration_sec = int(dur_meta)
|
||||||
|
|
||||||
|
release_date = _parse_iso_date(video.get("uploadDate"))
|
||||||
|
description = (video.get("description") or "").strip() or None
|
||||||
|
thumbnail_url = (video.get("thumbnailUrl") or "").strip() or None
|
||||||
|
if thumbnail_url:
|
||||||
|
# JSON-LD emituje "https:https://cdn..." — utnij zdublowany scheme.
|
||||||
|
thumbnail_url = re.sub(r"^https?:(?=https?://)", "", thumbnail_url)
|
||||||
|
|
||||||
|
performers: list[RawPerformer] = []
|
||||||
|
seen_perf: set[str] = set()
|
||||||
|
for m in _MODEL_LINK_RE.finditer(detail_html):
|
||||||
|
slug = m.group(1)
|
||||||
|
if slug in seen_perf:
|
||||||
|
continue
|
||||||
|
seen_perf.add(slug)
|
||||||
|
performers.append(
|
||||||
|
RawPerformer(
|
||||||
|
external_id=f"{self.sitetag}:model:{slug}",
|
||||||
|
name=_name_from_slug(slug),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
# Fallback dla świeżych uploadów bez linków modelek: nazwiska z flashvars
|
||||||
|
# video_tags — 2-3 słowa, obecne w tytule, bez generycznych fraz.
|
||||||
|
title_cf = title.casefold()
|
||||||
|
tags_m = _VIDEO_TAGS_RE.search(detail_html)
|
||||||
|
for entry in (tags_m.group(1).split(",") if tags_m else []):
|
||||||
|
entry = entry.strip()
|
||||||
|
words = entry.split()
|
||||||
|
if not (2 <= len(words) <= 3):
|
||||||
|
continue
|
||||||
|
if any(w.casefold() in _PERF_STOPWORDS for w in words):
|
||||||
|
continue
|
||||||
|
if entry.casefold() not in title_cf:
|
||||||
|
continue
|
||||||
|
slug = entry.casefold().replace(" ", "-")
|
||||||
|
if slug in seen_perf:
|
||||||
|
continue
|
||||||
|
seen_perf.add(slug)
|
||||||
|
performers.append(
|
||||||
|
RawPerformer(
|
||||||
|
external_id=f"{self.sitetag}:model:{slug}",
|
||||||
|
name=_name_from_slug(slug),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
tags: list[RawTag] = []
|
||||||
|
seen_tag: set[str] = set()
|
||||||
|
for m in _CATEGORY_LINK_RE.finditer(detail_html):
|
||||||
|
slug, name = m.group(1), m.group(2).strip()
|
||||||
|
if not name or slug in seen_tag:
|
||||||
|
continue
|
||||||
|
seen_tag.add(slug)
|
||||||
|
tags.append(RawTag(external_id=f"{self.sitetag}:tag:{slug}", name=name, slug=slug))
|
||||||
|
|
||||||
|
# Phash z KVS screenshotów — niski hit-rate oczekiwany (jak porn00),
|
||||||
|
# graceful miss → composite scoring (performer+title+duration).
|
||||||
|
fingerprints: list[RawFingerprint] = []
|
||||||
|
if thumbnail_url:
|
||||||
|
ph = compute_thumbnail_phash(thumbnail_url, referer=_BASE + "/")
|
||||||
|
if ph:
|
||||||
|
fingerprints.append(RawFingerprint(kind="phash", value=ph))
|
||||||
|
|
||||||
|
playback_sources = [
|
||||||
|
RawPlaybackSource(
|
||||||
|
origin=f"tube:{self.sitetag}",
|
||||||
|
page_url=scene_url,
|
||||||
|
duration_sec=duration_sec,
|
||||||
|
thumbnail_url=thumbnail_url,
|
||||||
|
)
|
||||||
|
]
|
||||||
|
|
||||||
|
return RawScene(
|
||||||
|
external_id=f"{self.sitetag}:{video_id or scene_url}",
|
||||||
|
title=title,
|
||||||
|
description=description,
|
||||||
|
release_date=release_date,
|
||||||
|
duration_sec=duration_sec,
|
||||||
|
url=scene_url,
|
||||||
|
performers=performers,
|
||||||
|
tags=tags,
|
||||||
|
fingerprints=fingerprints,
|
||||||
|
playback_sources=playback_sources,
|
||||||
|
)
|
||||||
|
|
@ -1,21 +1,148 @@
|
||||||
"""perverzija.com — direct HTML scrape search results.
|
"""perverzija.com — latest browse scraper via WordPress REST API.
|
||||||
|
|
||||||
Search: `https://www.perverzija.com/page/<n>/?s=<q>` (WordPress + Cloudflare).
|
Historia: dawniej search scraper (`?s=`), ale 2026-06 perverzija rate-limituje search
|
||||||
Scene URL: `https://www.perverzija.com/<slug>/`.
|
(429) a homepage jest JS-renderowane (brak linków postów w surowym HTML) → search
|
||||||
|
zwracał 0. To WordPress, więc czysty kanał to REST API: `/wp-json/wp/v2/posts` daje
|
||||||
|
ustrukturyzowany JSON (link, date, title, featured thumb, taksonomie) jednym requestem
|
||||||
|
na stronę. VPS dociera (curl_cffi bypassuje JA3; 200 nie 403). Przerobione na browse
|
||||||
|
2026-06-22 (user request).
|
||||||
|
|
||||||
CF-protected: `browser_get` (curl_cffi) bypassuje JA3 fingerprint blocks.
|
Z REST `?_embed=1` bierzemy: tytuł, datę, miniaturę (featured_media), STUDIO
|
||||||
|
(taksonomia `category` — np. "DadCrush"/"TeamSkeet", to studyjny re-up) i tagi
|
||||||
|
(`post_tag`). Performerów REST nie wystawia (custom taksonomia `stars` bez show_in_rest)
|
||||||
|
→ puste, dorabia canonical-merge (content studyjny dobrze matchuje TPDB/StashDB; tytuł
|
||||||
|
i tak ma nazwiska).
|
||||||
|
|
||||||
|
Playback: post page (tube.perverzija.com/<slug>/) embeduje xtremestream iframe →
|
||||||
|
extractor `perverzijacom` → `_embed_iframe` → hoster resolwowany phone-side.
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import re
|
import html
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
from datetime import date, datetime
|
||||||
|
|
||||||
from app.connectors.direct_scrapers._search_base import BaseSearchScraper
|
from app.connectors.base import (
|
||||||
|
RawFingerprint,
|
||||||
|
RawPlaybackSource,
|
||||||
class PerverzijaScraper(BaseSearchScraper):
|
RawScene,
|
||||||
sitetag = "perverzijacom"
|
RawStudio,
|
||||||
_search_url_template = "https://www.perverzija.com/page/{page}/?s={query}"
|
RawTag,
|
||||||
_scene_url_re = re.compile(
|
|
||||||
r'href="(?P<url>https://www\.perverzija\.com/(?P<slug>[a-z0-9][a-z0-9\-]+))/"',
|
|
||||||
re.IGNORECASE,
|
|
||||||
)
|
)
|
||||||
|
from app.connectors.direct_scrapers._browse_base import (
|
||||||
|
BaseBrowseScraper,
|
||||||
|
compute_thumbnail_phash,
|
||||||
|
)
|
||||||
|
from app.extractors import browser_get
|
||||||
|
from app.normalize.text import slugify
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
_BASE = "https://www.perverzija.com"
|
||||||
|
_PER_PAGE = 20
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_date(value: str | None) -> date | None:
|
||||||
|
if not value:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
return datetime.fromisoformat(value.replace("Z", "+00:00")).date()
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
class PerverzijaScraper(BaseBrowseScraper):
|
||||||
|
sitetag = "perverzijacom"
|
||||||
|
|
||||||
|
def _listing_url(self, page: int) -> str:
|
||||||
|
return f"{_BASE}/wp-json/wp/v2/posts?per_page={_PER_PAGE}&page={page}&_embed=1"
|
||||||
|
|
||||||
|
# crawl_page nadpisany (REST JSON, nie HTML) → abstrakcje nieużywane.
|
||||||
|
def _extract_scene_urls(self, listing_html: str) -> list[str]:
|
||||||
|
return []
|
||||||
|
|
||||||
|
def _parse_detail(self, scene_url: str, detail_html: str) -> RawScene | None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def crawl_page(self, page: int) -> list[RawScene] | None:
|
||||||
|
url = self._listing_url(page)
|
||||||
|
try:
|
||||||
|
res = browser_get(url, timeout=self._timeout)
|
||||||
|
except Exception as e:
|
||||||
|
log.warning("perverzija REST fetch failed (page %d): %s", page, e)
|
||||||
|
return None
|
||||||
|
# WP zwraca 400 (rest_post_invalid_page_number) za ostatnią stroną → exhausted.
|
||||||
|
if res.status_code != 200:
|
||||||
|
return []
|
||||||
|
try:
|
||||||
|
posts = json.loads(res.text)
|
||||||
|
except (json.JSONDecodeError, ValueError):
|
||||||
|
log.warning("perverzija REST: bad JSON page %d", page)
|
||||||
|
return None
|
||||||
|
if not isinstance(posts, list) or not posts:
|
||||||
|
return []
|
||||||
|
|
||||||
|
out: list[RawScene] = []
|
||||||
|
for p in posts:
|
||||||
|
link = (p.get("link") or "").strip()
|
||||||
|
title = html.unescape((p.get("title") or {}).get("rendered", "")).strip()
|
||||||
|
if not link or not title:
|
||||||
|
continue
|
||||||
|
release_date = _parse_date(p.get("date"))
|
||||||
|
|
||||||
|
emb = p.get("_embedded") or {}
|
||||||
|
fm = emb.get("wp:featuredmedia") or []
|
||||||
|
thumb = (fm[0].get("source_url") if fm and isinstance(fm[0], dict) else None) or None
|
||||||
|
|
||||||
|
studio: RawStudio | None = None
|
||||||
|
tags: list[RawTag] = []
|
||||||
|
seen_tag: set[str] = set()
|
||||||
|
for group in emb.get("wp:term") or []:
|
||||||
|
if not group:
|
||||||
|
continue
|
||||||
|
tax = group[0].get("taxonomy")
|
||||||
|
if tax == "category" and studio is None:
|
||||||
|
sname = (group[0].get("name") or "").strip()
|
||||||
|
if sname:
|
||||||
|
studio = RawStudio(
|
||||||
|
external_id=f"{self.sitetag}:studio:{slugify(sname)}",
|
||||||
|
name=sname, slug=slugify(sname),
|
||||||
|
)
|
||||||
|
elif tax == "post_tag":
|
||||||
|
for g in group:
|
||||||
|
name = (g.get("name") or "").strip()
|
||||||
|
sl = (g.get("slug") or slugify(name)).strip()
|
||||||
|
if not name or sl in seen_tag:
|
||||||
|
continue
|
||||||
|
seen_tag.add(sl)
|
||||||
|
tags.append(RawTag(external_id=f"{self.sitetag}:tag:{sl}", name=name, slug=sl))
|
||||||
|
|
||||||
|
fingerprints: list[RawFingerprint] = []
|
||||||
|
if thumb:
|
||||||
|
ph = compute_thumbnail_phash(thumb, referer=_BASE + "/")
|
||||||
|
if ph:
|
||||||
|
fingerprints.append(RawFingerprint(kind="phash", value=ph))
|
||||||
|
|
||||||
|
out.append(
|
||||||
|
RawScene(
|
||||||
|
external_id=f"{self.sitetag}:{link}",
|
||||||
|
title=title,
|
||||||
|
release_date=release_date,
|
||||||
|
url=link,
|
||||||
|
studio=studio,
|
||||||
|
performers=[],
|
||||||
|
tags=tags,
|
||||||
|
fingerprints=fingerprints,
|
||||||
|
playback_sources=[
|
||||||
|
RawPlaybackSource(
|
||||||
|
origin=f"tube:{self.sitetag}",
|
||||||
|
page_url=link,
|
||||||
|
thumbnail_url=thumb,
|
||||||
|
)
|
||||||
|
],
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
log.info("perverzija REST page %d: %d scenes", page, len(out))
|
||||||
|
return out
|
||||||
|
|
|
||||||
|
|
@ -1,19 +1,147 @@
|
||||||
"""porndish.com — direct HTML scrape.
|
"""porndish.com — latest browse scraper via WordPress REST API.
|
||||||
|
|
||||||
Search: `https://porndish.com/page/<n>/?s=<q>`.
|
Historia: dawniej search scraper (`?s=`), zamarzł 2026-05-07 (search przestał dawać
|
||||||
Scene URL: `https://porndish.com/<slug>/`.
|
nowe sceny — 1151h cisza, watchdog GOON-16). To WordPress (g1/bimber theme), VPS
|
||||||
|
dociera, więc czysty kanał to REST API: `/wp-json/wp/v2/posts?_embed=1` daje
|
||||||
|
ustrukturyzowany JSON jednym requestem na stronę. Przerobione na browse 2026-06-24
|
||||||
|
(ten sam wzorzec co perverzija).
|
||||||
|
|
||||||
|
Z REST `_embed`: tytuł, data, miniatura (featured_media), STUDIO (taksonomia
|
||||||
|
`category` — np. "Freeuse Fantasy", content studyjny) i tagi (`post_tag` — porndish
|
||||||
|
miesza w nich performerów z gatunkami, bierzemy jak jest; canonical-merge i tak
|
||||||
|
dorabia performerów z TPDB/StashDB, a tytuł ma nazwiska). Performerów osobno nie
|
||||||
|
wyciągamy (post_tag ich nie rozdziela od gatunków bez listy known-performers).
|
||||||
|
|
||||||
|
Playback: post page embeduje hoster iframe → extractor `porndishcom` → `_embed_iframe`
|
||||||
|
→ resolwowany phone-side.
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import re
|
import html
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
from datetime import date, datetime
|
||||||
|
|
||||||
from app.connectors.direct_scrapers._search_base import BaseSearchScraper
|
from app.connectors.base import (
|
||||||
|
RawFingerprint,
|
||||||
|
RawPlaybackSource,
|
||||||
class PornDishScraper(BaseSearchScraper):
|
RawScene,
|
||||||
sitetag = "porndishcom"
|
RawStudio,
|
||||||
_search_url_template = "https://porndish.com/page/{page}/?s={query}"
|
RawTag,
|
||||||
_scene_url_re = re.compile(
|
|
||||||
r'href="(?P<url>https://porndish\.com/(?P<slug>[a-z0-9][a-z0-9\-]+))/"',
|
|
||||||
re.IGNORECASE,
|
|
||||||
)
|
)
|
||||||
|
from app.connectors.direct_scrapers._browse_base import (
|
||||||
|
BaseBrowseScraper,
|
||||||
|
compute_thumbnail_phash,
|
||||||
|
)
|
||||||
|
from app.extractors import browser_get
|
||||||
|
from app.normalize.text import slugify
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
_BASE = "https://www.porndish.com"
|
||||||
|
_PER_PAGE = 20
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_date(value: str | None) -> date | None:
|
||||||
|
if not value:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
return datetime.fromisoformat(value.replace("Z", "+00:00")).date()
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
class PornDishScraper(BaseBrowseScraper):
|
||||||
|
sitetag = "porndishcom"
|
||||||
|
|
||||||
|
def _listing_url(self, page: int) -> str:
|
||||||
|
return f"{_BASE}/wp-json/wp/v2/posts?per_page={_PER_PAGE}&page={page}&_embed=1"
|
||||||
|
|
||||||
|
# crawl_page nadpisany (REST JSON, nie HTML) → abstrakcje nieużywane.
|
||||||
|
def _extract_scene_urls(self, listing_html: str) -> list[str]:
|
||||||
|
return []
|
||||||
|
|
||||||
|
def _parse_detail(self, scene_url: str, detail_html: str) -> RawScene | None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def crawl_page(self, page: int) -> list[RawScene] | None:
|
||||||
|
url = self._listing_url(page)
|
||||||
|
try:
|
||||||
|
res = browser_get(url, timeout=self._timeout)
|
||||||
|
except Exception as e:
|
||||||
|
log.warning("porndish REST fetch failed (page %d): %s", page, e)
|
||||||
|
return None
|
||||||
|
# WP zwraca 400 (rest_post_invalid_page_number) za ostatnią stroną → exhausted.
|
||||||
|
if res.status_code != 200:
|
||||||
|
return []
|
||||||
|
try:
|
||||||
|
posts = json.loads(res.text)
|
||||||
|
except (json.JSONDecodeError, ValueError):
|
||||||
|
log.warning("porndish REST: bad JSON page %d", page)
|
||||||
|
return None
|
||||||
|
if not isinstance(posts, list) or not posts:
|
||||||
|
return []
|
||||||
|
|
||||||
|
out: list[RawScene] = []
|
||||||
|
for p in posts:
|
||||||
|
link = (p.get("link") or "").strip()
|
||||||
|
title = html.unescape((p.get("title") or {}).get("rendered", "")).strip()
|
||||||
|
if not link or not title:
|
||||||
|
continue
|
||||||
|
release_date = _parse_date(p.get("date"))
|
||||||
|
|
||||||
|
emb = p.get("_embedded") or {}
|
||||||
|
fm = emb.get("wp:featuredmedia") or []
|
||||||
|
thumb = (fm[0].get("source_url") if fm and isinstance(fm[0], dict) else None) or None
|
||||||
|
|
||||||
|
studio: RawStudio | None = None
|
||||||
|
tags: list[RawTag] = []
|
||||||
|
seen_tag: set[str] = set()
|
||||||
|
for group in emb.get("wp:term") or []:
|
||||||
|
if not group:
|
||||||
|
continue
|
||||||
|
tax = group[0].get("taxonomy")
|
||||||
|
if tax == "category" and studio is None:
|
||||||
|
sname = (group[0].get("name") or "").strip()
|
||||||
|
if sname:
|
||||||
|
studio = RawStudio(
|
||||||
|
external_id=f"{self.sitetag}:studio:{slugify(sname)}",
|
||||||
|
name=sname, slug=slugify(sname),
|
||||||
|
)
|
||||||
|
elif tax == "post_tag":
|
||||||
|
for g in group:
|
||||||
|
name = (g.get("name") or "").strip()
|
||||||
|
sl = (g.get("slug") or slugify(name)).strip()
|
||||||
|
if not name or sl in seen_tag:
|
||||||
|
continue
|
||||||
|
seen_tag.add(sl)
|
||||||
|
tags.append(RawTag(external_id=f"{self.sitetag}:tag:{sl}", name=name, slug=sl))
|
||||||
|
|
||||||
|
fingerprints: list[RawFingerprint] = []
|
||||||
|
if thumb:
|
||||||
|
ph = compute_thumbnail_phash(thumb, referer=_BASE + "/")
|
||||||
|
if ph:
|
||||||
|
fingerprints.append(RawFingerprint(kind="phash", value=ph))
|
||||||
|
|
||||||
|
out.append(
|
||||||
|
RawScene(
|
||||||
|
external_id=f"{self.sitetag}:{link}",
|
||||||
|
title=title,
|
||||||
|
release_date=release_date,
|
||||||
|
url=link,
|
||||||
|
studio=studio,
|
||||||
|
performers=[],
|
||||||
|
tags=tags,
|
||||||
|
fingerprints=fingerprints,
|
||||||
|
playback_sources=[
|
||||||
|
RawPlaybackSource(
|
||||||
|
origin=f"tube:{self.sitetag}",
|
||||||
|
page_url=link,
|
||||||
|
thumbnail_url=thumb,
|
||||||
|
)
|
||||||
|
],
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
log.info("porndish REST page %d: %d scenes", page, len(out))
|
||||||
|
return out
|
||||||
|
|
|
||||||
|
|
@ -1,26 +0,0 @@
|
||||||
"""pornditt.com — direct HTML scrape.
|
|
||||||
|
|
||||||
KVS-style site (kt_player engine). Search URL: `/search/<slug>/?from=<page>` z slug-style
|
|
||||||
zapytaniem (spacje → `-`). Sceny renderują się na subdomenie `v.pornditt.com/videos/<id>/<slug>/`,
|
|
||||||
więc regex matchuje oba (z i bez `v.` prefix).
|
|
||||||
|
|
||||||
Sitetag `porndittcom` (legacy z porn-app DEFAULT_SITETAGS — suffix-stripped name).
|
|
||||||
"""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import re
|
|
||||||
|
|
||||||
from app.connectors.direct_scrapers._search_base import BaseSearchScraper
|
|
||||||
|
|
||||||
|
|
||||||
class PornDittScraper(BaseSearchScraper):
|
|
||||||
sitetag = "porndittcom"
|
|
||||||
_search_url_template = "https://pornditt.com/search/{query}/?from={page}"
|
|
||||||
_scene_url_re = re.compile(
|
|
||||||
r'href="(?P<url>https://(?:v\.)?pornditt\.com/videos/(?P<sid>\d+)/(?P<slug>[a-z0-9\-]+))/"',
|
|
||||||
re.IGNORECASE,
|
|
||||||
)
|
|
||||||
|
|
||||||
def _format_query_for_url(self, query: str) -> str:
|
|
||||||
# KVS slug: lowercase, spacja/interpunkcja → `-`. URL-encoded (`+`) tu nie zadziała.
|
|
||||||
return re.sub(r"[^a-z0-9]+", "-", query.lower()).strip("-")
|
|
||||||
271
app/connectors/direct_scrapers/porndoe.py
Normal file
271
app/connectors/direct_scrapers/porndoe.py
Normal file
|
|
@ -0,0 +1,271 @@
|
||||||
|
"""porndoe.com — latest-vids browse scraper.
|
||||||
|
|
||||||
|
Dołączony 2026-05-21 (theporndude audit). Jedyny verified high-value candidate
|
||||||
|
z 172 tube'ów na theporndude.com/top-porn-tube-sites + /full-porn-movies-sites.
|
||||||
|
|
||||||
|
Czemu wart: każda scena ma kompletny **JSON-LD VideoObject** schema:
|
||||||
|
- name (title), description, uploadDate (ISO timestamp), duration (ISO 8601)
|
||||||
|
- producer + publisher → named studio z `/channel-profile/<slug>` URL
|
||||||
|
- actor[] → named performers z `/pornstars-profile/<slug>` URL
|
||||||
|
- thumbnailUrl (CDN p.cdnc.porndoe.com)
|
||||||
|
|
||||||
|
To wystarczy do composite fuzzy match w resolverze (studio + performer Jaccard +
|
||||||
|
date proximity + title token-set + duration). Phash hit-rate niski (porndoe robi
|
||||||
|
własne crop-thumbnaile 390x219, nie hot-linkuje studio art) — ale rich metadata
|
||||||
|
nadrabia, jak pornxp/porn00.
|
||||||
|
|
||||||
|
URL patterns:
|
||||||
|
- Listing: `/videos/most-recent?page=N` (page 1 = newest, ~31 scen/page)
|
||||||
|
- Scene: `/watch/<id>` gdzie id = `pd` + 10 alfanum (stable)
|
||||||
|
- Studio: `/channel-profile/<slug>`
|
||||||
|
- Performer: `/pornstars-profile/<slug>`
|
||||||
|
- Tags/categories: `/categories/<slug>`
|
||||||
|
|
||||||
|
Playback: stream URL NIE jest inline w SSR HTML — player JS init dopiero po user
|
||||||
|
"Play" click. Dajemy playback_source z page_url + origin `tube:porndoecom`;
|
||||||
|
extractor w `_REGISTRY` mapuje na `_vps_blocked_fallback.extract` → mobile WebView
|
||||||
|
INJECTED_JS scrapuje `<video>.src` po phone IP (0 VPS bandwidth, zgodne z
|
||||||
|
pre-public bandwidth/anonimowość priorytet).
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import re
|
||||||
|
from datetime import date, datetime
|
||||||
|
|
||||||
|
from app.connectors.base import (
|
||||||
|
RawFingerprint,
|
||||||
|
RawPerformer,
|
||||||
|
RawPlaybackSource,
|
||||||
|
RawScene,
|
||||||
|
RawStudio,
|
||||||
|
RawTag,
|
||||||
|
)
|
||||||
|
from app.connectors.direct_scrapers._browse_base import (
|
||||||
|
BaseBrowseScraper,
|
||||||
|
compute_thumbnail_phash,
|
||||||
|
)
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
_BASE = "https://porndoe.com"
|
||||||
|
|
||||||
|
# Scene listing — `<a href="/watch/pd7a3o0e8v2b">`. Id = `pd` + alfanum.
|
||||||
|
_SCENE_URL_RE = re.compile(r'href="(/watch/[a-z0-9]+)"', re.IGNORECASE)
|
||||||
|
_WATCH_ID_RE = re.compile(r"/watch/([a-z0-9]+)", re.IGNORECASE)
|
||||||
|
|
||||||
|
# JSON-LD <script> bloki.
|
||||||
|
_JSONLD_RE = re.compile(
|
||||||
|
r'<script[^>]+type=["\']application/ld\+json["\'][^>]*>(.*?)</script>',
|
||||||
|
re.IGNORECASE | re.DOTALL,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Tagi/kategorie z DOM (JSON-LD genre bywa pusty). porndoe URL: `/category/<id>/<slug>`.
|
||||||
|
_TAG_LINK_RE = re.compile(
|
||||||
|
r'href="/category/\d+/([a-z0-9\-]+)"[^>]*>([^<]+)</a>', re.IGNORECASE
|
||||||
|
)
|
||||||
|
|
||||||
|
# ISO 8601 duration — porndoe emituje "PT8M0S" (czasem "T8M0S" bez P).
|
||||||
|
_ISO_DUR_RE = re.compile(
|
||||||
|
r"^P?T?(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?$", re.IGNORECASE
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_iso_duration(value: str | None) -> int | None:
|
||||||
|
"""`PT11M7S` / `T8M0S` → sekundy. None gdy format nieznany."""
|
||||||
|
if not value:
|
||||||
|
return None
|
||||||
|
m = _ISO_DUR_RE.match(value.strip())
|
||||||
|
if not m:
|
||||||
|
return None
|
||||||
|
h = int(m.group(1) or 0)
|
||||||
|
mn = int(m.group(2) or 0)
|
||||||
|
s = int(m.group(3) or 0)
|
||||||
|
total = h * 3600 + mn * 60 + s
|
||||||
|
return total or None
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_iso_date(value: str | None) -> date | None:
|
||||||
|
"""`2026-05-20T14:55:13+00:00` → date. None gdy parse fail."""
|
||||||
|
if not value:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
return datetime.fromisoformat(value.replace("Z", "+00:00")).date()
|
||||||
|
except ValueError:
|
||||||
|
# Fallback: pierwsze 10 znaków YYYY-MM-DD
|
||||||
|
m = re.match(r"(\d{4}-\d{2}-\d{2})", value)
|
||||||
|
if m:
|
||||||
|
try:
|
||||||
|
return date.fromisoformat(m.group(1))
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _slug_from_url(url: str | None) -> str | None:
|
||||||
|
"""`https://porndoe.com/channel-profile/fantasy-girl-pass` → `fantasy-girl-pass`."""
|
||||||
|
if not url:
|
||||||
|
return None
|
||||||
|
m = re.search(r"/(?:channel-profile|pornstars-profile)/([a-z0-9\-]+)", url, re.IGNORECASE)
|
||||||
|
return m.group(1) if m else None
|
||||||
|
|
||||||
|
|
||||||
|
def _iter_jsonld_objects(data: object):
|
||||||
|
"""Spłaszcza JSON-LD: dict / list / @graph → strumień dict-ów."""
|
||||||
|
if isinstance(data, dict):
|
||||||
|
graph = data.get("@graph")
|
||||||
|
if isinstance(graph, list):
|
||||||
|
for item in graph:
|
||||||
|
yield from _iter_jsonld_objects(item)
|
||||||
|
else:
|
||||||
|
yield data
|
||||||
|
elif isinstance(data, list):
|
||||||
|
for item in data:
|
||||||
|
yield from _iter_jsonld_objects(item)
|
||||||
|
|
||||||
|
|
||||||
|
def _extract_video_object(html: str) -> dict | None:
|
||||||
|
"""Znajdź pierwszy JSON-LD VideoObject w HTML."""
|
||||||
|
for m in _JSONLD_RE.finditer(html):
|
||||||
|
raw = m.group(1).strip()
|
||||||
|
if not raw:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
data = json.loads(raw)
|
||||||
|
except (json.JSONDecodeError, ValueError):
|
||||||
|
continue
|
||||||
|
for obj in _iter_jsonld_objects(data):
|
||||||
|
if obj.get("@type") == "VideoObject":
|
||||||
|
return obj
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
class PornDoeScraper(BaseBrowseScraper):
|
||||||
|
sitetag = "porndoecom"
|
||||||
|
|
||||||
|
def _listing_url(self, page: int) -> str:
|
||||||
|
if page <= 1:
|
||||||
|
return f"{_BASE}/videos/most-recent"
|
||||||
|
return f"{_BASE}/videos/most-recent?page={page}"
|
||||||
|
|
||||||
|
def _extract_scene_urls(self, listing_html: str) -> list[str]:
|
||||||
|
seen: set[str] = set()
|
||||||
|
out: list[str] = []
|
||||||
|
for m in _SCENE_URL_RE.finditer(listing_html):
|
||||||
|
url = f"{_BASE}{m.group(1)}"
|
||||||
|
if url in seen:
|
||||||
|
continue
|
||||||
|
seen.add(url)
|
||||||
|
out.append(url)
|
||||||
|
return out
|
||||||
|
|
||||||
|
def _parse_detail(self, scene_url: str, detail_html: str) -> RawScene | None:
|
||||||
|
video = _extract_video_object(detail_html)
|
||||||
|
if not video:
|
||||||
|
log.info("porndoe: no JSON-LD VideoObject on %s", scene_url)
|
||||||
|
return None
|
||||||
|
|
||||||
|
title = (video.get("name") or "").strip()
|
||||||
|
if not title:
|
||||||
|
return None
|
||||||
|
|
||||||
|
watch_id_m = _WATCH_ID_RE.search(scene_url)
|
||||||
|
watch_id = watch_id_m.group(1) if watch_id_m else None
|
||||||
|
|
||||||
|
description = (video.get("description") or "").strip() or None
|
||||||
|
duration_sec = _parse_iso_duration(video.get("duration"))
|
||||||
|
release_date = _parse_iso_date(
|
||||||
|
video.get("uploadDate") or video.get("datePublished")
|
||||||
|
)
|
||||||
|
thumbnail_url = video.get("thumbnailUrl") or None
|
||||||
|
|
||||||
|
# Studio: producer / publisher (Organization). Preferuj producer.
|
||||||
|
studio: RawStudio | None = None
|
||||||
|
for key in ("producer", "publisher"):
|
||||||
|
org = video.get(key)
|
||||||
|
if isinstance(org, dict) and org.get("name"):
|
||||||
|
name = org["name"].strip()
|
||||||
|
slug = _slug_from_url(org.get("url")) or re.sub(
|
||||||
|
r"[^a-z0-9]+", "-", name.lower()
|
||||||
|
).strip("-")
|
||||||
|
if name:
|
||||||
|
studio = RawStudio(
|
||||||
|
external_id=f"{self.sitetag}:channel:{slug}",
|
||||||
|
name=name,
|
||||||
|
slug=slug,
|
||||||
|
)
|
||||||
|
break
|
||||||
|
|
||||||
|
# Performers: actor[] (lista Person lub pojedynczy Person).
|
||||||
|
performers: list[RawPerformer] = []
|
||||||
|
seen_perf: set[str] = set()
|
||||||
|
actors = video.get("actor")
|
||||||
|
if isinstance(actors, dict):
|
||||||
|
actors = [actors]
|
||||||
|
if isinstance(actors, list):
|
||||||
|
for actor in actors:
|
||||||
|
if not isinstance(actor, dict):
|
||||||
|
continue
|
||||||
|
name = (actor.get("name") or "").strip()
|
||||||
|
if not name:
|
||||||
|
continue
|
||||||
|
slug = _slug_from_url(actor.get("url")) or re.sub(
|
||||||
|
r"[^a-z0-9]+", "-", name.lower()
|
||||||
|
).strip("-")
|
||||||
|
if slug in seen_perf:
|
||||||
|
continue
|
||||||
|
seen_perf.add(slug)
|
||||||
|
performers.append(
|
||||||
|
RawPerformer(
|
||||||
|
external_id=f"{self.sitetag}:performer:{slug}",
|
||||||
|
name=name,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
# Tagi: z DOM (`/categories/<slug>` / `/tags/<slug>`).
|
||||||
|
tags: list[RawTag] = []
|
||||||
|
seen_tag: set[str] = set()
|
||||||
|
for m in _TAG_LINK_RE.finditer(detail_html):
|
||||||
|
slug, name = m.group(1), m.group(2).strip()
|
||||||
|
if not name or name.lower() in ("categories", "tags", ""):
|
||||||
|
continue
|
||||||
|
if slug in seen_tag or len(slug) > 60:
|
||||||
|
continue
|
||||||
|
seen_tag.add(slug)
|
||||||
|
tags.append(
|
||||||
|
RawTag(external_id=f"{self.sitetag}:tag:{slug}", name=name, slug=slug)
|
||||||
|
)
|
||||||
|
|
||||||
|
# Phash z thumbnail (porndoe robi własne crop-thumbnaile — niski hit-rate
|
||||||
|
# oczekiwany, ale graceful: brak match → resolver spada do composite scoring).
|
||||||
|
fingerprints: list[RawFingerprint] = []
|
||||||
|
if thumbnail_url:
|
||||||
|
ph = compute_thumbnail_phash(thumbnail_url, referer=_BASE + "/")
|
||||||
|
if ph:
|
||||||
|
fingerprints.append(RawFingerprint(kind="phash", value=ph))
|
||||||
|
|
||||||
|
# Playback — page_url do scena strony. Stream JS-rendered, więc extractor
|
||||||
|
# `porndoecom` → `_vps_blocked_fallback.extract` (mobile WebView scrape).
|
||||||
|
playback_sources = [
|
||||||
|
RawPlaybackSource(
|
||||||
|
origin=f"tube:{self.sitetag}",
|
||||||
|
page_url=scene_url,
|
||||||
|
duration_sec=duration_sec,
|
||||||
|
thumbnail_url=thumbnail_url,
|
||||||
|
)
|
||||||
|
]
|
||||||
|
|
||||||
|
return RawScene(
|
||||||
|
external_id=f"{self.sitetag}:{watch_id or scene_url}",
|
||||||
|
title=title,
|
||||||
|
description=description,
|
||||||
|
release_date=release_date,
|
||||||
|
duration_sec=duration_sec,
|
||||||
|
url=scene_url,
|
||||||
|
studio=studio,
|
||||||
|
performers=performers,
|
||||||
|
tags=tags,
|
||||||
|
fingerprints=fingerprints,
|
||||||
|
playback_sources=playback_sources,
|
||||||
|
)
|
||||||
|
|
@ -1,99 +0,0 @@
|
||||||
"""pornhat.com — search-mode scraper (performer-driven backfill).
|
|
||||||
|
|
||||||
KVS engine. Search URL: `/search/<query>/` z `+` jako space separator. Scene URLs
|
|
||||||
to `/video/<slug>/` (slug bez ID prefix, w przeciwieństwie do 3Movs/OK.xxx). Slug
|
|
||||||
zawiera tokens query gdy match jest relevant, więc filtruje się automatycznie.
|
|
||||||
|
|
||||||
Auto-screenshot thumbnaile (`static.pornhat.com/contents/videos_screenshots/.../1.jpg`)
|
|
||||||
— do canonical match przez phash NIE nadają się (sprawdzone w probe 2026-05-12, 8%).
|
|
||||||
Ale wartość scrapera: discovering nowych scen performera których inne tube'y/canonical
|
|
||||||
nie mają. Mostly orphan ingest, ale dla popular performers może łapać studio scenes
|
|
||||||
których nie mamy w TPDB jeszcze.
|
|
||||||
|
|
||||||
Metadata enrich: scene page ma `class="info-video js-ajax-{dvd,model,tag}"` div'y
|
|
||||||
z `data-setup='{"title": ..., "url": ..., "dir": ...}'` JSON. Parsujemy w
|
|
||||||
`_fetch_scene_metadata()` żeby insertować studio (dvd), dodatkowych performerów
|
|
||||||
(models), i tagi do każdej sceny.
|
|
||||||
"""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import json
|
|
||||||
import logging
|
|
||||||
import re
|
|
||||||
|
|
||||||
from app.connectors.base import RawPerformer, RawStudio, RawTag
|
|
||||||
from app.connectors.direct_scrapers._search_base import BaseSearchScraper
|
|
||||||
from app.extractors import browser_get
|
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
|
|
||||||
# `class="info-video js-ajax-<kind>"` ... `data-setup='<json>'`. JSON jest
|
|
||||||
# single-quoted (HTML attribute), z double-quotes wewnątrz dla string values.
|
|
||||||
# `\1` w replacement: backreference do `<kind>` żeby wiedzieć co matchujemy.
|
|
||||||
_AJAX_DATA_RE = re.compile(
|
|
||||||
r"class=\"info-video js-ajax-(?P<kind>dvd|model|tag)[^\"]*\"[^>]*data-setup='(?P<json>[^']+)'",
|
|
||||||
re.IGNORECASE,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class PornHatScraper(BaseSearchScraper):
|
|
||||||
sitetag = "pornhatcom"
|
|
||||||
# Pagination KVS-style: /search/<query>/<page>/ (page=1 ALSO works z explicit `/1/`)
|
|
||||||
_search_url_template = "https://www.pornhat.com/search/{query}/{page}/"
|
|
||||||
# PornHat search HTML używa relative hrefs `/video/<slug>/`. BaseSearchScraper
|
|
||||||
# automatycznie konwertuje relative → absolute via urlparse(search_url).netloc.
|
|
||||||
_scene_url_re = re.compile(
|
|
||||||
r'href="(?P<url>(?:https://www\.pornhat\.com)?/video/(?P<slug>[a-z0-9\-]+)/)"',
|
|
||||||
re.IGNORECASE,
|
|
||||||
)
|
|
||||||
|
|
||||||
def _format_query_for_url(self, query: str) -> str:
|
|
||||||
# KVS: lowercase + spaces → `-` (slug-style), działa też `+`
|
|
||||||
return query.strip().lower().replace(" ", "-")
|
|
||||||
|
|
||||||
def _fetch_scene_metadata(
|
|
||||||
self, scene_url: str
|
|
||||||
) -> tuple[RawStudio | None, list[RawPerformer], list[RawTag]] | None:
|
|
||||||
"""Fetch scene detail + parse `js-ajax-{dvd,model,tag}` data-setup JSON."""
|
|
||||||
try:
|
|
||||||
r = browser_get(scene_url, timeout=self._timeout)
|
|
||||||
if r.status_code != 200:
|
|
||||||
return None
|
|
||||||
except Exception as e:
|
|
||||||
log.debug("pornhat detail fetch failed %s: %s", scene_url, e)
|
|
||||||
return None
|
|
||||||
|
|
||||||
studio: RawStudio | None = None
|
|
||||||
performers: list[RawPerformer] = []
|
|
||||||
tags: list[RawTag] = []
|
|
||||||
|
|
||||||
for m in _AJAX_DATA_RE.finditer(r.text):
|
|
||||||
kind = m.group("kind").lower()
|
|
||||||
try:
|
|
||||||
data = json.loads(m.group("json"))
|
|
||||||
except json.JSONDecodeError:
|
|
||||||
continue
|
|
||||||
name = (data.get("title") or "").strip()
|
|
||||||
slug = (data.get("dir") or "").strip() or None
|
|
||||||
if not name:
|
|
||||||
continue
|
|
||||||
if kind == "dvd":
|
|
||||||
# `dvd` to studio/series wrapper (np. "Adult Time"). Pierwsze
|
|
||||||
# wystąpienie bierzemy jako studio sceny — rzadko jest ich więcej.
|
|
||||||
if studio is None:
|
|
||||||
studio = RawStudio(
|
|
||||||
external_id=f"pornhatcom:dvd:{slug or name.lower()}",
|
|
||||||
name=name,
|
|
||||||
slug=slug,
|
|
||||||
)
|
|
||||||
elif kind == "model":
|
|
||||||
performers.append(RawPerformer(name=name))
|
|
||||||
elif kind == "tag":
|
|
||||||
tags.append(RawTag(
|
|
||||||
external_id=f"pornhatcom:tag:{slug or name.lower()}",
|
|
||||||
name=name,
|
|
||||||
slug=slug,
|
|
||||||
))
|
|
||||||
|
|
||||||
return studio, performers, tags
|
|
||||||
|
|
@ -1,24 +0,0 @@
|
||||||
"""PornHub.com — direct HTML scrape search results.
|
|
||||||
|
|
||||||
Search: `https://www.pornhub.com/video/search?search=<q>&page=<n>`
|
|
||||||
Scene URL: `https://www.pornhub.com/view_video.php?viewkey=<id>`
|
|
||||||
"""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import re
|
|
||||||
|
|
||||||
from app.connectors.direct_scrapers._search_base import BaseSearchScraper
|
|
||||||
|
|
||||||
|
|
||||||
class PornHubScraper(BaseSearchScraper):
|
|
||||||
sitetag = "pornhubcom"
|
|
||||||
_search_url_template = "https://www.pornhub.com/video/search?search={query}&page={page}"
|
|
||||||
_scene_url_re = re.compile(
|
|
||||||
r'href="(?P<url>/view_video\.php\?viewkey=[A-Za-z0-9]+)"',
|
|
||||||
)
|
|
||||||
|
|
||||||
def _slug_from_match(self, m, scene_url):
|
|
||||||
# Pornhub URL nie ma slugu — używamy viewkey jako slug do query token filtering.
|
|
||||||
# Tytuł będzie derived z viewkey (krótki ID), ale faktyczny title backfilluje
|
|
||||||
# się przy resolve (yt-dlp ma metadata).
|
|
||||||
return m.group("url").split("=")[-1]
|
|
||||||
122
app/connectors/direct_scrapers/porntrex_browse.py
Normal file
122
app/connectors/direct_scrapers/porntrex_browse.py
Normal file
|
|
@ -0,0 +1,122 @@
|
||||||
|
"""porntrex.com — latest-vids BROWSE scraper (KVS), obok istniejącego search scrapera.
|
||||||
|
|
||||||
|
PornTrexScraper (search, performer-driven) zostaje w ALL_DIRECT_SCRAPERS — daje
|
||||||
|
pokrycie back-catalogu performerów. Ten browse dokłada gwarancję świeżości wprost
|
||||||
|
z feedu `/latest-updates/<n>/` (próg watchdog 48h zamiast 168h, nie zależy od kolejki
|
||||||
|
performerów). Wzorzec jak xvideos (search + browse równolegle).
|
||||||
|
|
||||||
|
KVS listing tile:
|
||||||
|
<div ... data-item-id="<id>"><a href="https://www.porntrex.com/video/<id>/<slug>">
|
||||||
|
<img data-src="//ptx.cdntrex.com/contents/.../300x168/1.jpg" alt="<Tytuł>">
|
||||||
|
<div class="duration">MM:SS</div>
|
||||||
|
Playback: KVS, natywny extractor `porntrexcom` (token expires+md5, portable) — bez zmian.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import html
|
||||||
|
import logging
|
||||||
|
import re
|
||||||
|
|
||||||
|
from app.connectors.base import RawFingerprint, RawPlaybackSource, RawScene
|
||||||
|
from app.connectors.direct_scrapers._browse_base import (
|
||||||
|
BaseBrowseScraper,
|
||||||
|
compute_thumbnail_phash,
|
||||||
|
)
|
||||||
|
from app.extractors import browser_get
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
_BASE = "https://www.porntrex.com"
|
||||||
|
_A_RE = re.compile(
|
||||||
|
r'<a\s+href="(?P<url>https?://(?:www\.)?porntrex\.com/video/\d+/[^"]*)"', re.IGNORECASE
|
||||||
|
)
|
||||||
|
_ALT_RE = re.compile(r'alt="([^"]*)"')
|
||||||
|
_THUMB_RE = re.compile(r'data-src="(//[^"]+\.(?:jpg|jpeg|webp|png)[^"]*)"', re.IGNORECASE)
|
||||||
|
_DUR_RE = re.compile(r'class="duration">\s*([\d]{1,2}(?:\s*:\s*[\d]{2}){1,2})\s*<')
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_duration(text: str | None) -> int | None:
|
||||||
|
if not text:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
nums = [int(p.strip()) for p in text.split(":")]
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
if len(nums) == 2:
|
||||||
|
return nums[0] * 60 + nums[1]
|
||||||
|
if len(nums) == 3:
|
||||||
|
return nums[0] * 3600 + nums[1] * 60 + nums[2]
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
class PornTrexBrowseScraper(BaseBrowseScraper):
|
||||||
|
sitetag = "porntrexcom"
|
||||||
|
|
||||||
|
def _listing_url(self, page: int) -> str:
|
||||||
|
return f"{_BASE}/latest-updates/{page}/"
|
||||||
|
|
||||||
|
def _extract_scene_urls(self, listing_html: str) -> list[str]:
|
||||||
|
return [m.group("url") for m in _A_RE.finditer(listing_html)]
|
||||||
|
|
||||||
|
def _parse_detail(self, scene_url: str, detail_html: str) -> RawScene | None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def crawl_page(self, page: int) -> list[RawScene] | None:
|
||||||
|
url = self._listing_url(page)
|
||||||
|
try:
|
||||||
|
res = browser_get(url, timeout=self._timeout)
|
||||||
|
text = res.text if hasattr(res, "text") else res
|
||||||
|
except Exception as e:
|
||||||
|
log.warning("porntrex browse fetch failed (page %d): %s", page, e)
|
||||||
|
return None
|
||||||
|
|
||||||
|
out: list[RawScene] = []
|
||||||
|
seen: set[str] = set()
|
||||||
|
anchors = list(_A_RE.finditer(text))
|
||||||
|
for idx, m in enumerate(anchors):
|
||||||
|
scene_url = m.group("url").replace("://www.", "://").rstrip("/")
|
||||||
|
if scene_url in seen:
|
||||||
|
continue
|
||||||
|
seen.add(scene_url)
|
||||||
|
win = text[m.start(): (anchors[idx + 1].start() if idx + 1 < len(anchors) else m.end() + 700)]
|
||||||
|
|
||||||
|
am = _ALT_RE.search(win)
|
||||||
|
title = html.unescape(am.group(1)).strip() if am else ""
|
||||||
|
if not title:
|
||||||
|
# fallback: slug → tytuł
|
||||||
|
sl = re.search(r"/video/\d+/([a-z0-9\-]+)", scene_url)
|
||||||
|
title = sl.group(1).replace("-", " ").strip().title() if sl else ""
|
||||||
|
if not title:
|
||||||
|
continue
|
||||||
|
tm = _THUMB_RE.search(win)
|
||||||
|
thumb = ("https:" + tm.group(1)) if tm else None
|
||||||
|
dm = _DUR_RE.search(win)
|
||||||
|
duration_sec = _parse_duration(dm.group(1) if dm else None)
|
||||||
|
|
||||||
|
fingerprints: list[RawFingerprint] = []
|
||||||
|
if thumb:
|
||||||
|
ph = compute_thumbnail_phash(thumb, referer=_BASE + "/")
|
||||||
|
if ph:
|
||||||
|
fingerprints.append(RawFingerprint(kind="phash", value=ph))
|
||||||
|
|
||||||
|
out.append(
|
||||||
|
RawScene(
|
||||||
|
external_id=f"{self.sitetag}:{scene_url}",
|
||||||
|
title=title,
|
||||||
|
duration_sec=duration_sec,
|
||||||
|
url=scene_url,
|
||||||
|
performers=[],
|
||||||
|
tags=[],
|
||||||
|
fingerprints=fingerprints,
|
||||||
|
playback_sources=[
|
||||||
|
RawPlaybackSource(
|
||||||
|
origin=f"tube:{self.sitetag}",
|
||||||
|
page_url=scene_url,
|
||||||
|
duration_sec=duration_sec,
|
||||||
|
thumbnail_url=thumb,
|
||||||
|
)
|
||||||
|
],
|
||||||
|
)
|
||||||
|
)
|
||||||
|
log.info("porntrex browse page %d: %d scenes", page, len(out))
|
||||||
|
return out
|
||||||
|
|
@ -1,22 +0,0 @@
|
||||||
"""RedTube.com — direct HTML scrape search results.
|
|
||||||
|
|
||||||
Search: `https://www.redtube.com/?search=<q>&page=<n>`
|
|
||||||
Scene URL: `https://www.redtube.com/<id>` (slug nie ma w URL — viewkey-only).
|
|
||||||
"""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import re
|
|
||||||
|
|
||||||
from app.connectors.direct_scrapers._search_base import BaseSearchScraper
|
|
||||||
|
|
||||||
|
|
||||||
class RedTubeScraper(BaseSearchScraper):
|
|
||||||
sitetag = "redtubecom"
|
|
||||||
_search_url_template = "https://www.redtube.com/?search={query}&page={page}"
|
|
||||||
_scene_url_re = re.compile(
|
|
||||||
r'href="(?P<url>https://www\.redtube\.com/(?P<slug>\d+))"',
|
|
||||||
)
|
|
||||||
|
|
||||||
def _title_from_slug(self, slug):
|
|
||||||
# Numeric ID jako tytuł nie ma sensu — placeholder, title backfill przy resolve.
|
|
||||||
return f"redtube:{slug}"
|
|
||||||
|
|
@ -1,19 +1,178 @@
|
||||||
"""siska.video — direct HTML scrape.
|
"""siska.video — latest-vids browse scraper.
|
||||||
|
|
||||||
Search: `https://siska.video/page/<n>/?s=<q>`.
|
Historia: dawniej performer-driven search scraper (`?s=<q>`), ale siska zepsuła
|
||||||
Scene URL: `https://siska.video/<slug>/`.
|
wyszukiwarkę (zwraca latest niezależnie od query). Przerobione na BROWSE (latest
|
||||||
|
chronologicznie z `/page/<n>/`), re-enabled 2026-06-20 (user fa4083a2).
|
||||||
|
|
||||||
|
Cały blok kafelka listingu ma komplet metadanych (zero detail-fetchy):
|
||||||
|
<a title='<Tytuł>' href='https://siska.video/video.php?videoID=<n>'>
|
||||||
|
<span class='th_video_duration'>40 : 27</span>
|
||||||
|
<img data-src='https://siska.video/category/<Kat>/<id>.jpg'
|
||||||
|
alt='<Performer> - <Tytuł> - <Studio>'>
|
||||||
|
→ tytuł, duration, miniatura, performer+studio (alt), kategoria (ścieżka thumba).
|
||||||
|
|
||||||
|
Playback: świeże filmy embedują playmogo (DoodStream clone) + luluvid (filemoon
|
||||||
|
family). Extractor `siskavideo` → `_embed_iframe.extract` oddaje type='hoster';
|
||||||
|
telefon resolwuje phone-side. page_url = video.php?videoID=<n>.
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
import re
|
import re
|
||||||
|
|
||||||
from app.connectors.direct_scrapers._search_base import BaseSearchScraper
|
from app.connectors.base import (
|
||||||
|
RawFingerprint,
|
||||||
|
RawPerformer,
|
||||||
|
RawPlaybackSource,
|
||||||
|
RawScene,
|
||||||
|
RawStudio,
|
||||||
|
RawTag,
|
||||||
|
)
|
||||||
|
from app.connectors.direct_scrapers._browse_base import (
|
||||||
|
BaseBrowseScraper,
|
||||||
|
compute_thumbnail_phash,
|
||||||
|
)
|
||||||
|
from app.extractors import browser_get
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
class SiskaScraper(BaseSearchScraper):
|
_BASE = "https://siska.video"
|
||||||
sitetag = "siskavideo"
|
|
||||||
_search_url_template = "https://siska.video/page/{page}/?s={query}"
|
# Kafelek: <a title='..' href='..videoID=N'>. Reszta pól w oknie po tym matchu.
|
||||||
_scene_url_re = re.compile(
|
_A_RE = re.compile(
|
||||||
r'href="(?P<url>https://siska\.video/(?P<slug>[a-z0-9][a-z0-9\-]+))/"',
|
r"<a\s+title='(?P<title>[^']*)'\s+href='(?P<url>https://siska\.video/video\.php\?videoID=\d+)'",
|
||||||
re.IGNORECASE,
|
re.IGNORECASE,
|
||||||
)
|
)
|
||||||
|
_DUR_RE = re.compile(r"th_video_duration'>\s*([\d]{1,2}(?:\s*:\s*[\d]{2}){1,2})\s*<")
|
||||||
|
_THUMB_RE = re.compile(r"data-src='([^']+\.(?:jpg|jpeg|webp|png))'", re.IGNORECASE)
|
||||||
|
_ALT_RE = re.compile(r"alt='([^']*)'")
|
||||||
|
_CAT_RE = re.compile(r"/category/([^/]+)/", re.IGNORECASE)
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_duration(text: str | None) -> int | None:
|
||||||
|
"""`40 : 27` → 2427 (MM:SS); `1 : 05 : 30` → H:MM:SS. None gdy brak."""
|
||||||
|
if not text:
|
||||||
|
return None
|
||||||
|
parts = [p.strip() for p in text.split(":")]
|
||||||
|
try:
|
||||||
|
nums = [int(p) for p in parts]
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
if len(nums) == 2:
|
||||||
|
return nums[0] * 60 + nums[1]
|
||||||
|
if len(nums) == 3:
|
||||||
|
return nums[0] * 3600 + nums[1] * 60 + nums[2]
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _slugify(name: str) -> str:
|
||||||
|
return re.sub(r"[^a-z0-9]+", "-", name.lower()).strip("-")
|
||||||
|
|
||||||
|
|
||||||
|
class SiskaScraper(BaseBrowseScraper):
|
||||||
|
sitetag = "siskavideo"
|
||||||
|
|
||||||
|
def _listing_url(self, page: int) -> str:
|
||||||
|
return f"{_BASE}/page/{page}/"
|
||||||
|
|
||||||
|
# crawl_page nadpisany → poniższe abstrakcje nieużywane, ale wymagane do instancji.
|
||||||
|
def _extract_scene_urls(self, listing_html: str) -> list[str]:
|
||||||
|
return [m.group("url") for m in _A_RE.finditer(listing_html)]
|
||||||
|
|
||||||
|
def _parse_detail(self, scene_url: str, detail_html: str) -> RawScene | None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def crawl_page(self, page: int) -> list[RawScene] | None:
|
||||||
|
url = self._listing_url(page)
|
||||||
|
try:
|
||||||
|
res = browser_get(url, timeout=self._timeout)
|
||||||
|
html = res.text if hasattr(res, "text") else res
|
||||||
|
except Exception as e:
|
||||||
|
log.warning("siska browse listing fetch failed (page %d): %s", page, e)
|
||||||
|
return None
|
||||||
|
|
||||||
|
out: list[RawScene] = []
|
||||||
|
seen: set[str] = set()
|
||||||
|
for m in _A_RE.finditer(html):
|
||||||
|
scene_url = m.group("url")
|
||||||
|
if scene_url in seen:
|
||||||
|
continue
|
||||||
|
seen.add(scene_url)
|
||||||
|
title = (m.group("title") or "").strip()
|
||||||
|
if not title:
|
||||||
|
continue
|
||||||
|
window = html[m.end():m.end() + 700]
|
||||||
|
|
||||||
|
dm = _DUR_RE.search(window)
|
||||||
|
duration_sec = _parse_duration(dm.group(1) if dm else None)
|
||||||
|
tm = _THUMB_RE.search(window)
|
||||||
|
thumbnail_url = tm.group(1) if tm else None
|
||||||
|
|
||||||
|
# alt='Performer - Tytuł - Studio' → performer (pierwszy), studio (ostatni).
|
||||||
|
performers: list[RawPerformer] = []
|
||||||
|
studio: RawStudio | None = None
|
||||||
|
am = _ALT_RE.search(window)
|
||||||
|
if am and " - " in am.group(1):
|
||||||
|
parts = [p.strip() for p in am.group(1).split(" - ") if p.strip()]
|
||||||
|
if len(parts) >= 2:
|
||||||
|
pname = parts[0]
|
||||||
|
sname = parts[-1]
|
||||||
|
if pname:
|
||||||
|
performers.append(
|
||||||
|
RawPerformer(
|
||||||
|
external_id=f"{self.sitetag}:performer:{_slugify(pname)}",
|
||||||
|
name=pname,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if sname and len(parts) >= 3:
|
||||||
|
studio = RawStudio(
|
||||||
|
external_id=f"{self.sitetag}:studio:{_slugify(sname)}",
|
||||||
|
name=sname,
|
||||||
|
slug=_slugify(sname),
|
||||||
|
)
|
||||||
|
|
||||||
|
# Kategoria ze ścieżki miniatury (/category/<Kat>/<id>.jpg).
|
||||||
|
tags: list[RawTag] = []
|
||||||
|
if thumbnail_url:
|
||||||
|
cm = _CAT_RE.search(thumbnail_url)
|
||||||
|
if cm and cm.group(1).lower() not in ("uncategorized", ""):
|
||||||
|
cat = cm.group(1).replace("-", " ").replace("_", " ").strip()
|
||||||
|
tags.append(
|
||||||
|
RawTag(
|
||||||
|
external_id=f"{self.sitetag}:tag:{_slugify(cat)}",
|
||||||
|
name=cat,
|
||||||
|
slug=_slugify(cat),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
fingerprints: list[RawFingerprint] = []
|
||||||
|
if thumbnail_url:
|
||||||
|
ph = compute_thumbnail_phash(thumbnail_url, referer=_BASE + "/")
|
||||||
|
if ph:
|
||||||
|
fingerprints.append(RawFingerprint(kind="phash", value=ph))
|
||||||
|
|
||||||
|
playback_sources = [
|
||||||
|
RawPlaybackSource(
|
||||||
|
origin=f"tube:{self.sitetag}",
|
||||||
|
page_url=scene_url,
|
||||||
|
duration_sec=duration_sec,
|
||||||
|
thumbnail_url=thumbnail_url,
|
||||||
|
)
|
||||||
|
]
|
||||||
|
|
||||||
|
out.append(
|
||||||
|
RawScene(
|
||||||
|
external_id=f"{self.sitetag}:{scene_url}",
|
||||||
|
title=title,
|
||||||
|
duration_sec=duration_sec,
|
||||||
|
url=scene_url,
|
||||||
|
studio=studio,
|
||||||
|
performers=performers,
|
||||||
|
tags=tags,
|
||||||
|
fingerprints=fingerprints,
|
||||||
|
playback_sources=playback_sources,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
log.info("siska browse page %d: %d scenes", page, len(out))
|
||||||
|
return out
|
||||||
|
|
|
||||||
194
app/connectors/direct_scrapers/superporn.py
Normal file
194
app/connectors/direct_scrapers/superporn.py
Normal file
|
|
@ -0,0 +1,194 @@
|
||||||
|
"""superporn.com — latest-vids browse scraper (przez Bright Data ISP proxy).
|
||||||
|
|
||||||
|
Dołączony 2026-06-10 (user request). superporn twardo blokuje VPS IP Cloudflarem
|
||||||
|
(403 na KAŻDEJ impersonacji TLS — chrome/safari/firefox), więc ingest HTML idzie
|
||||||
|
przez Bright Data ISP proxy (`settings.brightdata_proxy_url`). Gdy proxy
|
||||||
|
nieskonfigurowane → scraper no-op (pusty iterator, log warning).
|
||||||
|
|
||||||
|
Proxy używamy TYLKO do scrape HTML. Playback NIE idzie przez proxy: `<source>` mp4
|
||||||
|
(cdnst*.superporn.com) ma token IP-bound do fetchera (403 cross-IP), więc resolve
|
||||||
|
musi nastąpić po stronie telefonu — extractor `superporncom` → `_vps_blocked_fallback`
|
||||||
|
(mobile WebView ładuje stronę z residential IP telefonu, INJECTED_JS bierze video.src).
|
||||||
|
Thumbnaile (img*.superporn.com) schodzą z VPS bez proxy (image proxy działa).
|
||||||
|
|
||||||
|
Sygnały (SSR HTML):
|
||||||
|
- JSON-LD VideoObject: name, description, uploadDate, thumbnailUrl, duration
|
||||||
|
ISO 8601 (`P0DT0H38M48S`). `author` = uploader (NIE performer — pomijamy).
|
||||||
|
- Performerzy: chip `<a class="chip-link" href=".../pornstar/<slug>"><span>Name</span>`
|
||||||
|
- Kategorie: chip `<a class="chip-link" href="/<slug>"><span>Name</span>` (bez
|
||||||
|
`/pornstar/`) — w bloku `#collapse-categories`.
|
||||||
|
- Duration backup: `<video ... data-video-duration="2328">` (sekundy).
|
||||||
|
|
||||||
|
Listing: `?page=N` (newest-first; sitemap lastmod jest z 2024, bezużyteczny).
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import html
|
||||||
|
import logging
|
||||||
|
import re
|
||||||
|
|
||||||
|
from app.config import get_settings
|
||||||
|
from app.connectors.base import (
|
||||||
|
RawFingerprint,
|
||||||
|
RawPerformer,
|
||||||
|
RawPlaybackSource,
|
||||||
|
RawScene,
|
||||||
|
RawTag,
|
||||||
|
)
|
||||||
|
from app.connectors.direct_scrapers._browse_base import (
|
||||||
|
BaseBrowseScraper,
|
||||||
|
compute_thumbnail_phash,
|
||||||
|
)
|
||||||
|
from app.connectors.direct_scrapers._playtube import (
|
||||||
|
_extract_video_object,
|
||||||
|
_parse_iso_date,
|
||||||
|
)
|
||||||
|
from app.normalize.text import slugify
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
_BASE = "https://www.superporn.com"
|
||||||
|
|
||||||
|
_SCENE_URL_RE = re.compile(r'href="(https://www\.superporn\.com/video/[a-z0-9\-]+)"', re.IGNORECASE)
|
||||||
|
# Chip pornstar: href .../pornstar/<slug> (abs lub rel), nazwa w <span>.
|
||||||
|
_PERF_CHIP_RE = re.compile(
|
||||||
|
r'<a[^>]+class="chip-link"[^>]+href="[^"]*/pornstar/[^"]*"[^>]*>.*?<span>([^<]+)</span>',
|
||||||
|
re.IGNORECASE | re.DOTALL,
|
||||||
|
)
|
||||||
|
# Chip kategorii: href="/<slug>" (NIE /pornstar/, NIE /video/, NIE absolutny http).
|
||||||
|
_CAT_CHIP_RE = re.compile(
|
||||||
|
r'<a[^>]+class="chip-link"[^>]+href="/(?!pornstar/|video/)([a-z0-9\-]+)"[^>]*>.*?<span>([^<]+)</span>',
|
||||||
|
re.IGNORECASE | re.DOTALL,
|
||||||
|
)
|
||||||
|
_DATA_DUR_RE = re.compile(r'data-video-duration="(\d+)"')
|
||||||
|
# ISO 8601 z dniami: `P0DT0H38M48S`.
|
||||||
|
_ISO_DUR_FULL_RE = re.compile(
|
||||||
|
r"P(?:(\d+)D)?T?(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?", re.IGNORECASE
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _clean(text: str) -> str:
|
||||||
|
"""superporn dwukrotnie HTML-koduje encje w JSON-LD (`&#39;` zamiast `'`).
|
||||||
|
Unescape iteracyjnie (max 2) — bez over-decode dla zwykłych nazw."""
|
||||||
|
for _ in range(2):
|
||||||
|
new = html.unescape(text)
|
||||||
|
if new == text:
|
||||||
|
break
|
||||||
|
text = new
|
||||||
|
return text.strip()
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_iso_duration_days(value: str | None) -> int | None:
|
||||||
|
"""`P0DT0H38M48S` → 2328. None gdy zero/parse fail."""
|
||||||
|
if not value:
|
||||||
|
return None
|
||||||
|
m = _ISO_DUR_FULL_RE.match(value.strip())
|
||||||
|
if not m:
|
||||||
|
return None
|
||||||
|
d, h, mn, s = (int(g or 0) for g in m.groups())
|
||||||
|
total = d * 86400 + h * 3600 + mn * 60 + s
|
||||||
|
return total or None
|
||||||
|
|
||||||
|
|
||||||
|
class SuperpornScraper(BaseBrowseScraper):
|
||||||
|
sitetag = "superporncom"
|
||||||
|
|
||||||
|
def __init__(self) -> None:
|
||||||
|
super().__init__()
|
||||||
|
# Bright Data ISP proxy — bez niego superporn jest nieosiągalny z VPS.
|
||||||
|
self._proxy = get_settings().brightdata_proxy_url
|
||||||
|
if not self._proxy:
|
||||||
|
log.warning("superporn: BRIGHTDATA_PROXY_URL unset — scraper disabled")
|
||||||
|
|
||||||
|
def _listing_url(self, page: int) -> str:
|
||||||
|
if page <= 1:
|
||||||
|
return f"{_BASE}/"
|
||||||
|
return f"{_BASE}/?page={page}"
|
||||||
|
|
||||||
|
def crawl_page(self, page: int):
|
||||||
|
# Bez proxy nie ma sensu uderzać (gwarantowany CF 403) — sygnalizuj
|
||||||
|
# "exhausted" (pusta lista), żeby deep-crawl nie retry'ował w kółko.
|
||||||
|
if not self._proxy:
|
||||||
|
return []
|
||||||
|
return super().crawl_page(page)
|
||||||
|
|
||||||
|
def _extract_scene_urls(self, listing_html: str) -> list[str]:
|
||||||
|
seen: set[str] = set()
|
||||||
|
out: list[str] = []
|
||||||
|
for m in _SCENE_URL_RE.finditer(listing_html):
|
||||||
|
url = m.group(1)
|
||||||
|
if url in seen:
|
||||||
|
continue
|
||||||
|
seen.add(url)
|
||||||
|
out.append(url)
|
||||||
|
return out
|
||||||
|
|
||||||
|
def _parse_detail(self, scene_url: str, detail_html: str) -> RawScene | None:
|
||||||
|
video = _extract_video_object(detail_html) or {}
|
||||||
|
|
||||||
|
title = _clean(video.get("name") or "")
|
||||||
|
if not title:
|
||||||
|
return None
|
||||||
|
|
||||||
|
slug_m = re.search(r"/video/([a-z0-9\-]+)", scene_url, re.IGNORECASE)
|
||||||
|
scene_slug = slug_m.group(1) if slug_m else None
|
||||||
|
|
||||||
|
duration_sec = _parse_iso_duration_days(video.get("duration"))
|
||||||
|
if duration_sec is None:
|
||||||
|
dm = _DATA_DUR_RE.search(detail_html)
|
||||||
|
if dm:
|
||||||
|
duration_sec = int(dm.group(1)) or None
|
||||||
|
|
||||||
|
release_date = _parse_iso_date(video.get("uploadDate"))
|
||||||
|
description = _clean(video.get("description") or "") or None
|
||||||
|
thumbnail_url = (video.get("thumbnailUrl") or "").strip() or None
|
||||||
|
|
||||||
|
performers: list[RawPerformer] = []
|
||||||
|
seen_perf: set[str] = set()
|
||||||
|
for m in _PERF_CHIP_RE.finditer(detail_html):
|
||||||
|
name = _clean(m.group(1))
|
||||||
|
slug = slugify(name)
|
||||||
|
if not slug or slug in seen_perf:
|
||||||
|
continue
|
||||||
|
seen_perf.add(slug)
|
||||||
|
performers.append(
|
||||||
|
RawPerformer(external_id=f"{self.sitetag}:performer:{slug}", name=name)
|
||||||
|
)
|
||||||
|
|
||||||
|
tags: list[RawTag] = []
|
||||||
|
seen_tag: set[str] = set()
|
||||||
|
for m in _CAT_CHIP_RE.finditer(detail_html):
|
||||||
|
slug, name = m.group(1).strip(), _clean(m.group(2))
|
||||||
|
if not name or slug in seen_tag:
|
||||||
|
continue
|
||||||
|
seen_tag.add(slug)
|
||||||
|
tags.append(RawTag(external_id=f"{self.sitetag}:tag:{slug}", name=name, slug=slug))
|
||||||
|
|
||||||
|
# Phash z thumbnaila — proxy NIE potrzebny (img*.superporn.com działa z VPS).
|
||||||
|
fingerprints: list[RawFingerprint] = []
|
||||||
|
if thumbnail_url:
|
||||||
|
ph = compute_thumbnail_phash(thumbnail_url, referer=_BASE + "/")
|
||||||
|
if ph:
|
||||||
|
fingerprints.append(RawFingerprint(kind="phash", value=ph))
|
||||||
|
|
||||||
|
playback_sources = [
|
||||||
|
RawPlaybackSource(
|
||||||
|
origin=f"tube:{self.sitetag}",
|
||||||
|
page_url=scene_url,
|
||||||
|
duration_sec=duration_sec,
|
||||||
|
thumbnail_url=thumbnail_url,
|
||||||
|
)
|
||||||
|
]
|
||||||
|
|
||||||
|
return RawScene(
|
||||||
|
external_id=f"{self.sitetag}:{scene_slug or scene_url}",
|
||||||
|
title=title,
|
||||||
|
description=description,
|
||||||
|
release_date=release_date,
|
||||||
|
duration_sec=duration_sec,
|
||||||
|
url=scene_url,
|
||||||
|
performers=performers,
|
||||||
|
tags=tags,
|
||||||
|
fingerprints=fingerprints,
|
||||||
|
playback_sources=playback_sources,
|
||||||
|
)
|
||||||
126
app/connectors/direct_scrapers/supjav.py
Normal file
126
app/connectors/direct_scrapers/supjav.py
Normal file
|
|
@ -0,0 +1,126 @@
|
||||||
|
"""supjav.com — JAV browse scraper (przez Bright Data proxy, CF-blokuje VPS IP).
|
||||||
|
|
||||||
|
Osobna pula JAV (sekcja JAV), origin `tube:supjav` w JAV_ORIGINS. supjav agreguje
|
||||||
|
embedy (serwery TV/FST/ST/VOE), gdzie realny hoster jest schowany za `data-link`
|
||||||
|
(reverse-hex → lk1.supremejav.com); rozwiązuje go ekstraktor `app/extractors/tubes/supjav.py`.
|
||||||
|
|
||||||
|
Browse (RE 2026-07-10):
|
||||||
|
- listing: homepage `/` (najnowsze ~66, WordPress). Głębsza paginacja `/page/N/` leci
|
||||||
|
CF 403 nawet przez proxy → browse ogranicza się do strony 1 (scheduler odświeża).
|
||||||
|
- karty: `<div class="post"><a href="/<id>.html" title="<title>" rel="bookmark"
|
||||||
|
class="img"><img src=".../images/YYYY/MM/<CODE>.jpg!320x216.jpg" class="thumb">`.
|
||||||
|
- Metadane bierzemy Z KARTY (tytuł/thumb/URL/kod JAV/data z roku-miesiąca ścieżki
|
||||||
|
thumb), NIE z detalu — detal przez proxy jest flaky (CF przepuszcza ~40% IP), a
|
||||||
|
per-post fetch byłby wolny i lossy. Studio/tagi (poza kodem) dorabia enrich później.
|
||||||
|
|
||||||
|
supjav blokuje VPS IP twardym CF → listing idzie przez Bright Data ISP proxy z retry
|
||||||
|
(`fetch_supjav_html`, rotacja IP aż trafi przepuszczający). Bez proxy scraper no-op.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import html as _html
|
||||||
|
import logging
|
||||||
|
import re
|
||||||
|
from datetime import date
|
||||||
|
|
||||||
|
from app.config import get_settings
|
||||||
|
from app.connectors.base import RawPlaybackSource, RawScene, RawTag
|
||||||
|
from app.connectors.direct_scrapers._browse_base import BaseBrowseScraper
|
||||||
|
from app.extractors.tubes.supjav import fetch_supjav_html
|
||||||
|
from app.normalize.text import slugify
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
_BASE = "https://supjav.com"
|
||||||
|
# Karta: link-obraz z href + title + thumb. title="([^"]*)" jest bezpieczne (wartość
|
||||||
|
# atrybutu nie zawiera surowego "), więc tytuły z `>` (np. `<<3-Day Offer>>`) przechodzą.
|
||||||
|
# Link-obraz karty: <a href="URL" ... title="TITLE"><img src|data-original="THUMB">.
|
||||||
|
# supjav miesza kolejność atrybutów (href/rel/class="img"/title w dwóch wariantach), więc
|
||||||
|
# NIE wymagamy pozycji class="img" — wymóg `>\s*<img` i tak izoluje link-obraz od
|
||||||
|
# link-tekstu (ten drugi po `>` ma tekst, nie <img>). Thumb eager `src=` lub lazy
|
||||||
|
# `data-original=` (karty poniżej folda: src="data:..." placeholder + realny data-original).
|
||||||
|
_CARD_RE = re.compile(
|
||||||
|
r'href="(https://supjav\.com/\d+\.html)"[^>]*?\btitle="([^"]*)"[^>]*?>\s*'
|
||||||
|
r'<img[^>]*?\b(?:src|data-original|data-src|data-lazy-src)="(https://img\.supjav\.com/[^"]+)"',
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
_THUMB_DATE_RE = re.compile(r"/images/(\d{4})/(\d{2})/")
|
||||||
|
# Kod JAV: FC2PPV ma cyfrę w prefiksie (FC2PPV 4931572), studia mają myślnik (MGOLD-053,
|
||||||
|
# DLDSS-510, SNOS-258). Prefiks liter+cyfr, separator - lub spacja, numer.
|
||||||
|
_CODE_RE = re.compile(r"^([A-Z][A-Z0-9]{1,7}[-\s]\d{2,7})", re.IGNORECASE)
|
||||||
|
|
||||||
|
|
||||||
|
class SupjavScraper(BaseBrowseScraper):
|
||||||
|
sitetag = "supjav"
|
||||||
|
_timeout = 70.0
|
||||||
|
|
||||||
|
def __init__(self, *args, **kwargs) -> None:
|
||||||
|
super().__init__(*args, **kwargs)
|
||||||
|
# CF-blok VPS IP → listing przez Bright Data proxy (jak superporn).
|
||||||
|
self._proxy = get_settings().brightdata_proxy_url
|
||||||
|
if not self._proxy:
|
||||||
|
log.warning("supjav: BRIGHTDATA_PROXY_URL unset — scraper disabled")
|
||||||
|
|
||||||
|
def _listing_url(self, page: int) -> str:
|
||||||
|
return f"{_BASE}/" if page <= 1 else f"{_BASE}/page/{page}/"
|
||||||
|
|
||||||
|
def crawl_page(self, page: int) -> list[RawScene] | None:
|
||||||
|
if not self._proxy:
|
||||||
|
return [] # bez proxy gwarantowany CF 403 (exhausted)
|
||||||
|
html = fetch_supjav_html(self._listing_url(page), proxy=self._proxy, timeout=self._timeout)
|
||||||
|
if not html or len(html) < 15000:
|
||||||
|
# sam challenge / fetch fail. page 1 = transient (None, retry potem);
|
||||||
|
# page > 1 = najpewniej CF-403 głębszej paginacji = koniec (exhausted).
|
||||||
|
return None if page <= 1 else []
|
||||||
|
seen: set[str] = set()
|
||||||
|
out: list[RawScene] = []
|
||||||
|
for m in _CARD_RE.finditer(html):
|
||||||
|
url, title_raw, thumb_raw = m.group(1), m.group(2), m.group(3)
|
||||||
|
if url in seen:
|
||||||
|
continue
|
||||||
|
seen.add(url)
|
||||||
|
raw = self._card_to_scene(url, title_raw, thumb_raw)
|
||||||
|
if raw is not None:
|
||||||
|
out.append(raw)
|
||||||
|
return out
|
||||||
|
|
||||||
|
def _card_to_scene(self, url: str, title_raw: str, thumb_raw: str) -> RawScene | None:
|
||||||
|
title = _html.unescape(title_raw).strip()
|
||||||
|
if not title:
|
||||||
|
return None
|
||||||
|
thumb = thumb_raw.split("!")[0] or None # utnij `!320x216.jpg` → pełna rozdz.
|
||||||
|
release_date: date | None = None
|
||||||
|
dm = _THUMB_DATE_RE.search(thumb_raw)
|
||||||
|
if dm:
|
||||||
|
try:
|
||||||
|
release_date = date(int(dm.group(1)), int(dm.group(2)), 1)
|
||||||
|
except ValueError:
|
||||||
|
release_date = None
|
||||||
|
|
||||||
|
tags: list[RawTag] = []
|
||||||
|
cm = _CODE_RE.match(title)
|
||||||
|
if cm:
|
||||||
|
code = re.sub(r"[ ]", "-", cm.group(1).upper())
|
||||||
|
tags.append(RawTag(external_id=f"javcode:{code}", name=code, slug=slugify(code)))
|
||||||
|
|
||||||
|
return RawScene(
|
||||||
|
external_id=f"{self.sitetag}:{url}",
|
||||||
|
title=title,
|
||||||
|
release_date=release_date,
|
||||||
|
url=url,
|
||||||
|
tags=tags,
|
||||||
|
playback_sources=[
|
||||||
|
RawPlaybackSource(
|
||||||
|
origin=f"tube:{self.sitetag}",
|
||||||
|
page_url=url,
|
||||||
|
thumbnail_url=thumb,
|
||||||
|
)
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
# HTML detail nieużywany (browse z kart), ale ABC wymaga.
|
||||||
|
def _extract_scene_urls(self, listing_html: str) -> list[str]: # pragma: no cover
|
||||||
|
return []
|
||||||
|
|
||||||
|
def _parse_detail(self, scene_url: str, detail_html: str) -> RawScene | None: # pragma: no cover
|
||||||
|
return None
|
||||||
|
|
@ -1,78 +1,220 @@
|
||||||
"""SxyLandScraper — direct HTML scrape sxyland.com search.
|
"""sxyland.com — performer-page scrape (search-based, performer-driven).
|
||||||
|
|
||||||
Search: `https://sxyland.com/?s=<query>` zwraca wyniki w formacie
|
2026-06-16 fix (zamrożony od 06-07): sxyland porzucił URL scen `/<numeric_id>/<slug>/`
|
||||||
`https://sxyland.com/<numeric_id>/<slug>/`. Filtrujemy linki bez numeric ID
|
na rzecz `/<slug>/`, więc stary regex (wymagał cyfry w ścieżce) dawał 0. WordPress `?s=`
|
||||||
(legal pages typu /18-u-s-c-2257/).
|
filtruje, ale miesza — czystsze są **strony performera** `/actor/<slug>/`
|
||||||
|
(performer-driven query = nazwa performera → slugify → /actor/<slug>/).
|
||||||
|
|
||||||
|
Bogate metadane (per-scene detail fetch — sxyland to WP tube, taksonomie na scenie):
|
||||||
|
- performerzy: WSZYSTKIE `/actor/<slug>/` linki (z co-starami; `title="Name"`)
|
||||||
|
- tagi: `/tag/` + `/category/` (`title="Name"`); część to studia (BangBros/BLACKED/...)
|
||||||
|
- studio: heurystycznie z tagów-paysite (`_STUDIO_TAGS`); brak match → bez studio
|
||||||
|
- duration: `itemprop="duration"` ISO 8601 z dniami (P0DT0H41M12S)
|
||||||
|
- release date: `itemprop="uploadDate"`
|
||||||
|
- title: `og:title` / `itemprop="name"`
|
||||||
|
|
||||||
|
Playback przez extractor `sxylandcom` (_embed_iframe → playmogo/dood, phone-side).
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import html
|
||||||
import logging
|
import logging
|
||||||
import re
|
import re
|
||||||
import urllib.parse
|
|
||||||
from collections.abc import Iterator
|
from collections.abc import Iterator
|
||||||
|
from datetime import date, datetime
|
||||||
|
|
||||||
from app.connectors.base import RawPerformer, RawPlaybackSource, RawScene
|
from app.connectors.base import (
|
||||||
|
RawPerformer,
|
||||||
|
RawPlaybackSource,
|
||||||
|
RawScene,
|
||||||
|
RawStudio,
|
||||||
|
RawTag,
|
||||||
|
)
|
||||||
from app.connectors.direct_scrapers.base import BaseDirectTubeScraper
|
from app.connectors.direct_scrapers.base import BaseDirectTubeScraper
|
||||||
from app.extractors import browser_get
|
from app.extractors import browser_get
|
||||||
|
from app.normalize.text import slugify
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
_BASE = "https://sxyland.com"
|
||||||
|
|
||||||
_SCENE_URL_RE = re.compile(r'href="(https://sxyland\.com/(\d+)/([^"/]+))/?"')
|
# Linki scen na stronie performera: /<slug>/ (multi-word). Wykluczamy taksonomie/nav.
|
||||||
|
_SCENE_URL_RE = re.compile(r'href="https://sxyland\.com/([a-z0-9][a-z0-9-]+)/"')
|
||||||
|
_NAV_SLUGS = frozenset({
|
||||||
|
"actor", "actors", "category", "categories", "tag", "tags", "page", "author",
|
||||||
|
"models", "studios", "search", "home", "login", "register", "18-u-s-c-2257",
|
||||||
|
"privacy-policy", "cookie-policy", "dmca", "dmca-notice", "contact", "contact-us",
|
||||||
|
"terms", "terms-of-use", "about", "about-us", "2257",
|
||||||
|
})
|
||||||
|
# Scena-tagi siedzą w pierwszym <div class="tags-list">...</div> (NIE w sidebarze/
|
||||||
|
# popular-tags widgetcie). Bez scope'u studio łapało globalny "bangbros" na każdej scenie.
|
||||||
|
_TAGS_BLOCK_RE = re.compile(r'<div class="tags-list">(.*?)</div>', re.IGNORECASE | re.DOTALL)
|
||||||
|
|
||||||
|
_ACTOR_LINK_RE = re.compile(
|
||||||
|
r'href="https://sxyland\.com/actor/[^"/]+/"\s+title="([^"]+)"', re.IGNORECASE
|
||||||
|
)
|
||||||
|
_TAG_LINK_RE = re.compile(
|
||||||
|
r'href="https://sxyland\.com/(?:tag|category)/[^"/]+/"[^>]*title="([^"]+)"', re.IGNORECASE
|
||||||
|
)
|
||||||
|
_DURATION_RE = re.compile(r'itemprop="duration"\s+content="([^"]+)"', re.IGNORECASE)
|
||||||
|
_UPLOADDATE_RE = re.compile(r'itemprop="uploadDate"\s+content="([^"]+)"', re.IGNORECASE)
|
||||||
|
_OGTITLE_RE = re.compile(r'property="og:title"\s+content="([^"]+)"', re.IGNORECASE)
|
||||||
|
_ISO_DUR_RE = re.compile(
|
||||||
|
r"P(?:(\d+)D)?T?(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?", re.IGNORECASE
|
||||||
|
)
|
||||||
|
|
||||||
|
def _studio_from_title(title: str, performers: list[RawPerformer]) -> RawStudio | None:
|
||||||
|
"""Studio z prefiksu "Studio - ..." tytułu (jak hdporngg: paysite reposty mają
|
||||||
|
"BraZZers - ...", "MilfCoach - ..."). Guard: prefiks NIE może być performerem
|
||||||
|
(tytuł "Amirah Adara - X" → prefiks to imię, nie studio). Brak " - " → brak studio."""
|
||||||
|
if " - " not in title:
|
||||||
|
return None
|
||||||
|
prefix = title.split(" - ", 1)[0].strip()
|
||||||
|
if not (2 <= len(prefix) <= 30):
|
||||||
|
return None
|
||||||
|
pl = prefix.lower()
|
||||||
|
for p in performers:
|
||||||
|
if pl == p.name.lower() or pl in p.name.lower():
|
||||||
|
return None
|
||||||
|
return RawStudio(external_id=f"sxylandcom:studio:{slugify(prefix)}", name=prefix, slug=slugify(prefix))
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_iso_duration(value: str | None) -> int | None:
|
||||||
|
"""`P0DT0H41M12S` → 2472. None gdy zero/parse fail."""
|
||||||
|
if not value:
|
||||||
|
return None
|
||||||
|
m = _ISO_DUR_RE.match(value.strip())
|
||||||
|
if not m:
|
||||||
|
return None
|
||||||
|
d, h, mn, s = (int(g or 0) for g in m.groups())
|
||||||
|
total = d * 86400 + h * 3600 + mn * 60 + s
|
||||||
|
return total or None
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_date(value: str | None) -> date | None:
|
||||||
|
if not value:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
return datetime.fromisoformat(value.replace("Z", "+00:00")).date()
|
||||||
|
except ValueError:
|
||||||
|
m = re.match(r"(\d{4}-\d{2}-\d{2})", value)
|
||||||
|
return date.fromisoformat(m.group(1)) if m else None
|
||||||
|
|
||||||
|
|
||||||
class SxyLandScraper(BaseDirectTubeScraper):
|
class SxyLandScraper(BaseDirectTubeScraper):
|
||||||
sitetag = "sxylandcom"
|
sitetag = "sxylandcom"
|
||||||
|
_timeout: float = 30.0
|
||||||
|
|
||||||
def search(
|
def search(
|
||||||
self,
|
self, query: str, *, page: int = 1, limit: int | None = None
|
||||||
query: str,
|
|
||||||
*,
|
|
||||||
page: int = 1,
|
|
||||||
limit: int | None = None,
|
|
||||||
) -> Iterator[RawScene]:
|
) -> Iterator[RawScene]:
|
||||||
q = urllib.parse.quote_plus(query.strip())
|
actor_slug = slugify(query)
|
||||||
url = f"https://sxyland.com/page/{page}/?s={q}"
|
if not actor_slug:
|
||||||
|
return
|
||||||
|
listing = f"{_BASE}/actor/{actor_slug}/" + (f"page/{page}/" if page > 1 else "")
|
||||||
try:
|
try:
|
||||||
r = browser_get(url, timeout=30)
|
r = browser_get(listing, timeout=self._timeout)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.warning("sxyland search fetch failed: %s", e)
|
log.warning("sxyland actor-page fetch failed (%s): %s", listing, e)
|
||||||
return
|
return
|
||||||
if r.status_code != 200:
|
if r.status_code != 200:
|
||||||
|
log.debug("sxyland %s status=%d", listing, r.status_code)
|
||||||
return
|
return
|
||||||
|
|
||||||
query_tokens = {tok for tok in query.lower().split() if len(tok) >= 3}
|
scene_urls: list[str] = []
|
||||||
|
|
||||||
seen: set[str] = set()
|
seen: set[str] = set()
|
||||||
yielded = 0
|
|
||||||
for m in _SCENE_URL_RE.finditer(r.text):
|
for m in _SCENE_URL_RE.finditer(r.text):
|
||||||
scene_url = m.group(1) + "/"
|
slug = m.group(1)
|
||||||
slug = m.group(3)
|
if slug in _NAV_SLUGS or slug in seen:
|
||||||
if scene_url in seen:
|
|
||||||
continue
|
continue
|
||||||
seen.add(scene_url)
|
seen.add(slug)
|
||||||
|
scene_urls.append(f"{_BASE}/{slug}/")
|
||||||
|
|
||||||
slug_lower = slug.lower()
|
yielded = 0
|
||||||
if query_tokens and not any(tok in slug_lower for tok in query_tokens):
|
for scene_url in scene_urls:
|
||||||
|
scene = self._parse_scene(scene_url, query)
|
||||||
|
if scene is None:
|
||||||
continue
|
continue
|
||||||
|
yield scene
|
||||||
title = slug.replace("-", " ").strip()
|
|
||||||
|
|
||||||
yield RawScene(
|
|
||||||
external_id=f"sxylandcom:{scene_url}",
|
|
||||||
title=title,
|
|
||||||
url=scene_url,
|
|
||||||
playback_sources=[
|
|
||||||
RawPlaybackSource(origin="tube:sxylandcom", page_url=scene_url)
|
|
||||||
],
|
|
||||||
performers=[RawPerformer(name=query.strip())],
|
|
||||||
raw={
|
|
||||||
"source": "direct_scraper:sxyland",
|
|
||||||
"query": query,
|
|
||||||
"page": page,
|
|
||||||
"url": scene_url,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
yielded += 1
|
yielded += 1
|
||||||
if limit is not None and yielded >= limit:
|
if limit is not None and yielded >= limit:
|
||||||
return
|
return
|
||||||
|
|
||||||
|
def _parse_scene(self, scene_url: str, query: str) -> RawScene | None:
|
||||||
|
try:
|
||||||
|
r = browser_get(scene_url, timeout=self._timeout)
|
||||||
|
if r.status_code != 200:
|
||||||
|
return None
|
||||||
|
detail = r.text
|
||||||
|
except Exception as e:
|
||||||
|
log.info("sxyland scene fetch failed %s: %s", scene_url, e)
|
||||||
|
return None
|
||||||
|
|
||||||
|
title = _OGTITLE_RE.search(detail)
|
||||||
|
title_s = html.unescape(title.group(1)).strip() if title else ""
|
||||||
|
if not title_s:
|
||||||
|
return None
|
||||||
|
|
||||||
|
dm = _DURATION_RE.search(detail)
|
||||||
|
duration_sec = _parse_iso_duration(dm.group(1)) if dm else None
|
||||||
|
um = _UPLOADDATE_RE.search(detail)
|
||||||
|
release_date = _parse_date(um.group(1)) if um else None
|
||||||
|
|
||||||
|
# Performerzy: wszystkie /actor/ linki (z co-starami).
|
||||||
|
performers: list[RawPerformer] = []
|
||||||
|
seen_perf: set[str] = set()
|
||||||
|
for m in _ACTOR_LINK_RE.finditer(detail):
|
||||||
|
name = html.unescape(m.group(1)).strip()
|
||||||
|
sl = slugify(name)
|
||||||
|
if not sl or sl in seen_perf:
|
||||||
|
continue
|
||||||
|
seen_perf.add(sl)
|
||||||
|
performers.append(
|
||||||
|
RawPerformer(external_id=f"{self.sitetag}:performer:{sl}", name=name)
|
||||||
|
)
|
||||||
|
if not performers:
|
||||||
|
# Fallback: query (jesteśmy na /actor/<query>/, więc to na pewno ona).
|
||||||
|
performers.append(
|
||||||
|
RawPerformer(
|
||||||
|
external_id=f"{self.sitetag}:performer:{slugify(query)}",
|
||||||
|
name=query.strip(),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
# Tagi — TYLKO z bloku tagów sceny (nie z sidebara/popular widgetu).
|
||||||
|
tags: list[RawTag] = []
|
||||||
|
seen_tag: set[str] = set()
|
||||||
|
block_m = _TAGS_BLOCK_RE.search(detail)
|
||||||
|
tags_html = block_m.group(1) if block_m else ""
|
||||||
|
for m in _TAG_LINK_RE.finditer(tags_html):
|
||||||
|
name = html.unescape(m.group(1)).strip()
|
||||||
|
sl = slugify(name)
|
||||||
|
if not sl or sl in seen_tag:
|
||||||
|
continue
|
||||||
|
seen_tag.add(sl)
|
||||||
|
tags.append(RawTag(external_id=f"{self.sitetag}:tag:{sl}", name=name, slug=sl))
|
||||||
|
|
||||||
|
# Guard "to realna scena wideo": nav/legal pages (Terms of Use itp.) mają
|
||||||
|
# sidebar z aktorami (fałszywi performerzy) ale ZERO duration i ZERO tagów.
|
||||||
|
if duration_sec is None and not tags:
|
||||||
|
return None
|
||||||
|
|
||||||
|
studio = _studio_from_title(title_s, performers)
|
||||||
|
|
||||||
|
return RawScene(
|
||||||
|
external_id=f"{self.sitetag}:{scene_url}",
|
||||||
|
title=title_s,
|
||||||
|
duration_sec=duration_sec,
|
||||||
|
release_date=release_date,
|
||||||
|
url=scene_url,
|
||||||
|
studio=studio,
|
||||||
|
performers=performers,
|
||||||
|
tags=tags,
|
||||||
|
playback_sources=[
|
||||||
|
RawPlaybackSource(
|
||||||
|
origin=f"tube:{self.sitetag}",
|
||||||
|
page_url=scene_url,
|
||||||
|
duration_sec=duration_sec,
|
||||||
|
)
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
|
||||||
|
|
@ -1,24 +1,193 @@
|
||||||
"""sxyprn.com — direct HTML scrape search results.
|
"""sxyprn.com — performer-page scrape (search-based, performer-driven).
|
||||||
|
|
||||||
Sxyprn search jest oparte na `?type=videos&query=<q>` GET endpoint który zwraca
|
2026-06-16 fix (zamrożony od 05-07): sxyprn NIE ma free-text searcha. Stary endpoint
|
||||||
HTML strony z linkami. Scene URL format: `https://sxyprn.com/post/<post_id>.html`.
|
`?type=videos&query=<q>` oddawał TRENDING (0 trafień dla performera → strict token
|
||||||
|
filtr słusznie wycinał wszystko → 0 ingestu → freshness zamrożona). Realny "search"
|
||||||
|
to **strona performera** `https://sxyprn.com/<Imie-Nazwisko>.html` (spacje → myślniki),
|
||||||
|
zwraca ~30 kart scen tego performera.
|
||||||
|
|
||||||
Page'owanie sxyprn niespójne — często single-page results dla query (~24 wyników).
|
Bonus tej ścieżki: w trybie performer-driven query = czysta nazwa performera, więc
|
||||||
|
omijamy fragmentację sxyprn (post taguje "Dallas Rae" jako osobne `/Dallas.html` +
|
||||||
|
`/Rae.html` / `aria-label='Dallas,Rae'`). Performera bierzemy z query (po potwierdzeniu
|
||||||
|
token-filtrem na tytule), a NIE z połamanych tagów.
|
||||||
|
|
||||||
|
Metadane z karty (bez +1 fetch detalu):
|
||||||
|
- URL sceny: `/post/<id>.html`
|
||||||
|
- duration: `<span class='duration_small'>MM:SS|HH:MM:SS`
|
||||||
|
- studio: `<span class='post_el_small_subcat'>Channel` (sxyprn "channel"; ~⅔ kart, opcjonalny)
|
||||||
|
- title + tagi: `<a class='post_time' title='{New} ...#Tag1 #Tag2... {Watch...} url'>`
|
||||||
|
- thumbnail: `data-src='//...small.jpg'`
|
||||||
|
|
||||||
|
Playback resolwuje extractor `sxyprncom` (osobno; mp4 z trafficdeposit/lulustream).
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import html
|
||||||
|
import logging
|
||||||
import re
|
import re
|
||||||
|
|
||||||
|
from app.connectors.base import (
|
||||||
|
RawPerformer,
|
||||||
|
RawPlaybackSource,
|
||||||
|
RawScene,
|
||||||
|
RawStudio,
|
||||||
|
RawTag,
|
||||||
|
)
|
||||||
from app.connectors.direct_scrapers._search_base import BaseSearchScraper
|
from app.connectors.direct_scrapers._search_base import BaseSearchScraper
|
||||||
|
from app.extractors import browser_get
|
||||||
|
from app.normalize.text import slugify
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
_BASE = "https://sxyprn.com"
|
||||||
|
|
||||||
|
# js-pop anchor karty: /post/<id>.html(?sk=...&so=...&ss=latest na stronie performera)
|
||||||
|
# + aria-label (połamani performerzy — nieużywane).
|
||||||
|
_CARD_ANCHOR_RE = re.compile(
|
||||||
|
r"<a\s+href='/post/(?P<id>[a-f0-9]+)\.html(?:\?[^']*)?'[^>]*class='js-pop'",
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
# Uwaga: duration_small ma atrybut `title='s1->c10'` ze znakiem `>` w środku, więc
|
||||||
|
# NIE używamy `[^>]*` (łamie się na tym `>`) — leniwe `.*?` do wartości HH:MM:SS.
|
||||||
|
_DURATION_RE = re.compile(
|
||||||
|
r"duration_small.*?>\s*(\d{1,2}:\d{2}(?::\d{2})?)\s*<", re.IGNORECASE | re.DOTALL
|
||||||
|
)
|
||||||
|
_SUBCAT_RE = re.compile(r"post_el_small_subcat[^>]*>([^<]+)<", re.IGNORECASE)
|
||||||
|
_THUMB_RE = re.compile(r"data-src='(//[^']+?small\.jpg)'", re.IGNORECASE)
|
||||||
|
# post_time z tytułem — keyowane po ID, bo leży w post_control PO vid_container.
|
||||||
|
_POSTTIME_RE = re.compile(
|
||||||
|
r"post_time'[^>]*href='/post/{id}\.html(?:\?[^']*)?'[^>]*title='([^']*)'",
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
_HASHTAG_RE = re.compile(r"#(\w[\w-]*)")
|
||||||
|
_BRACE_RE = re.compile(r"\{[^}]*\}")
|
||||||
|
_URL_TAIL_RE = re.compile(r"https?://\S+")
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_duration(s: str) -> int | None:
|
||||||
|
parts = s.strip().split(":")
|
||||||
|
try:
|
||||||
|
if len(parts) == 2:
|
||||||
|
return int(parts[0]) * 60 + int(parts[1])
|
||||||
|
if len(parts) == 3:
|
||||||
|
return int(parts[0]) * 3600 + int(parts[1]) * 60 + int(parts[2])
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _clean_title(raw: str) -> tuple[str, list[str]]:
|
||||||
|
"""Zwraca (czysty tytuł, tagi). Usuwa {markery}, hashtagi i końcowy URL."""
|
||||||
|
text = html.unescape(raw)
|
||||||
|
tags = [m.group(1) for m in _HASHTAG_RE.finditer(text)]
|
||||||
|
text = _BRACE_RE.sub(" ", text) # {New}, {Watch At 1080P}, {WATCH...}
|
||||||
|
text = _URL_TAIL_RE.sub(" ", text) # końcowy link do hostera
|
||||||
|
text = _HASHTAG_RE.sub(" ", text) # #Tag
|
||||||
|
text = re.sub(r"\s+", " ", text).strip(" :-")
|
||||||
|
return text, tags
|
||||||
|
|
||||||
|
|
||||||
class SxyPrnScraper(BaseSearchScraper):
|
class SxyPrnScraper(BaseSearchScraper):
|
||||||
sitetag = "sxyprncom"
|
sitetag = "sxyprncom"
|
||||||
_search_url_template = "https://sxyprn.com/?type=videos&query={query}&page={page}"
|
# _search_url_template/_scene_url_re z bazy nieużywane — mamy custom search().
|
||||||
_scene_url_re = re.compile(
|
_card_window = 2600 # od js-pop anchora do post_control z tytułem
|
||||||
r'href="(?P<url>/post/(?P<slug>[a-z0-9]+))\.html"',
|
|
||||||
|
def _performer_path(self, query: str) -> str:
|
||||||
|
"""`Lana Rhoades` → `Lana-Rhoades` (strona performera sxyprn)."""
|
||||||
|
cleaned = re.sub(r"[^A-Za-z0-9 ]+", "", query).strip()
|
||||||
|
return re.sub(r"\s+", "-", cleaned)
|
||||||
|
|
||||||
|
def search(self, query, *, page: int = 1, limit=None):
|
||||||
|
path = self._performer_path(query)
|
||||||
|
if not path:
|
||||||
|
return
|
||||||
|
url = f"{_BASE}/{path}.html" + (f"?page={page}" if page > 1 else "")
|
||||||
|
try:
|
||||||
|
r = browser_get(url, timeout=self._timeout)
|
||||||
|
except Exception as e:
|
||||||
|
log.warning("sxyprn search fetch failed (%s): %s", url, e)
|
||||||
|
return
|
||||||
|
if r.status_code != 200:
|
||||||
|
log.debug("sxyprn search %s status=%d", url, r.status_code)
|
||||||
|
return
|
||||||
|
|
||||||
|
text = r.text
|
||||||
|
query_tokens = {
|
||||||
|
tok for tok in query.lower().split() if len(tok) >= self._query_token_min_len
|
||||||
|
}
|
||||||
|
anchors = list(_CARD_ANCHOR_RE.finditer(text))
|
||||||
|
seen: set[str] = set()
|
||||||
|
yielded = 0
|
||||||
|
for idx, m in enumerate(anchors):
|
||||||
|
post_id = m.group("id")
|
||||||
|
if post_id in seen:
|
||||||
|
continue
|
||||||
|
seen.add(post_id)
|
||||||
|
win_end = anchors[idx + 1].start() if idx + 1 < len(anchors) else m.end() + self._card_window
|
||||||
|
window = text[m.start():win_end]
|
||||||
|
|
||||||
|
tm = re.search(_POSTTIME_RE.pattern.format(id=re.escape(post_id)), text, re.IGNORECASE)
|
||||||
|
if not tm:
|
||||||
|
continue
|
||||||
|
title, tags = _clean_title(tm.group(1))
|
||||||
|
title_l = title.lower()
|
||||||
|
# Strict: scena musi realnie dotyczyć performera (oba tokeny w tytule) —
|
||||||
|
# chroni przed śmieciem na stronie i przed over-attribution.
|
||||||
|
if query_tokens and not all(tok in title_l for tok in query_tokens):
|
||||||
|
continue
|
||||||
|
if not title:
|
||||||
|
continue
|
||||||
|
|
||||||
|
scene_url = f"{_BASE}/post/{post_id}.html"
|
||||||
|
dur_m = _DURATION_RE.search(window)
|
||||||
|
duration_sec = _parse_duration(dur_m.group(1)) if dur_m else None
|
||||||
|
thumb_m = _THUMB_RE.search(window)
|
||||||
|
thumb = thumb_m.group(1) if thumb_m else None
|
||||||
|
if thumb and thumb.startswith("//"):
|
||||||
|
thumb = "https:" + thumb
|
||||||
|
|
||||||
|
studio = None
|
||||||
|
sub_m = _SUBCAT_RE.search(window)
|
||||||
|
if sub_m:
|
||||||
|
name = html.unescape(sub_m.group(1)).strip()
|
||||||
|
if name and name.lower() not in ("all", "trending"):
|
||||||
|
studio = RawStudio(
|
||||||
|
external_id=f"{self.sitetag}:studio:{slugify(name)}",
|
||||||
|
name=name,
|
||||||
|
slug=slugify(name),
|
||||||
)
|
)
|
||||||
|
|
||||||
def _title_from_slug(self, slug: str) -> str:
|
tag_objs: list[RawTag] = []
|
||||||
# sxyprn post ID to nieczytelny hash — placeholder, title backfill przy resolve.
|
seen_tag: set[str] = set()
|
||||||
return f"sxyprn:{slug}"
|
for t in tags:
|
||||||
|
sl = slugify(t)
|
||||||
|
if not sl or sl in seen_tag:
|
||||||
|
continue
|
||||||
|
seen_tag.add(sl)
|
||||||
|
tag_objs.append(RawTag(external_id=f"{self.sitetag}:tag:{sl}", name=t, slug=sl))
|
||||||
|
|
||||||
|
performer = RawPerformer(
|
||||||
|
external_id=f"{self.sitetag}:performer:{slugify(query)}",
|
||||||
|
name=query.strip(),
|
||||||
|
)
|
||||||
|
|
||||||
|
yield RawScene(
|
||||||
|
external_id=f"{self.sitetag}:{post_id}",
|
||||||
|
title=title,
|
||||||
|
duration_sec=duration_sec,
|
||||||
|
url=scene_url,
|
||||||
|
studio=studio,
|
||||||
|
performers=[performer],
|
||||||
|
tags=tag_objs,
|
||||||
|
playback_sources=[
|
||||||
|
RawPlaybackSource(
|
||||||
|
origin=f"tube:{self.sitetag}",
|
||||||
|
page_url=scene_url,
|
||||||
|
duration_sec=duration_sec,
|
||||||
|
thumbnail_url=thumb,
|
||||||
|
)
|
||||||
|
],
|
||||||
|
)
|
||||||
|
yielded += 1
|
||||||
|
if limit and yielded >= limit:
|
||||||
|
return
|
||||||
|
|
|
||||||
205
app/connectors/direct_scrapers/vjav.py
Normal file
205
app/connectors/direct_scrapers/vjav.py
Normal file
|
|
@ -0,0 +1,205 @@
|
||||||
|
"""vjav.com — JAV browse scraper (TXXX network). Sitemap id-walk + JSON metadata API.
|
||||||
|
|
||||||
|
Osobna pula JAV (tytuły azjatyckie + `title_jp`, studia typu JAVHD), origin `tube:vjav`.
|
||||||
|
Świadomie orphan vertical (sekcja JAV) — NIE deduplikuje się z zachodnim katalogiem.
|
||||||
|
|
||||||
|
vjav to SPA (ktk_player + blob) — listing renderuje się JS-owo, więc HTML listingu
|
||||||
|
nie ma linków do scen. Zamiast tego:
|
||||||
|
- **browse**: `sitemap.xml` -> `sitemap_vids_N.xml` (id-e wszystkich wideo). Newest
|
||||||
|
= najwyższe id (auto-increment). Cache posortowanej listy id malejąco, page N =
|
||||||
|
kolejny wycinek PER_PAGE.
|
||||||
|
- **metadata**: bogaty JSON API `GET /api/json/video/1/<floor>/<id>/<id>.json`
|
||||||
|
(floor = id//1000*1000) — title (+title_jp), duration, post_date, channel.title
|
||||||
|
(studio), models{} (performerki), categories{}+tags{}.
|
||||||
|
- **stream**: osobny ekstraktor `app/extractors/tubes/vjav.py` (videofile.php ->
|
||||||
|
get_file HLS). RE 2026-07-10.
|
||||||
|
|
||||||
|
Override `crawl_page()` (API/sitemap flow, wzór jak EpornerApiScraper). HTML-owe
|
||||||
|
_listing_url/_extract/_parse to stuby (ABC ich wymaga, nieużywane).
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import re
|
||||||
|
from datetime import date, datetime
|
||||||
|
|
||||||
|
from app.connectors.base import (
|
||||||
|
RawPerformer,
|
||||||
|
RawPlaybackSource,
|
||||||
|
RawScene,
|
||||||
|
RawStudio,
|
||||||
|
RawTag,
|
||||||
|
)
|
||||||
|
from app.connectors.direct_scrapers._browse_base import BaseBrowseScraper
|
||||||
|
from app.extractors import browser_get
|
||||||
|
from app.normalize.text import slugify
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
_BASE = "https://vjav.com"
|
||||||
|
_PER_PAGE = 40
|
||||||
|
_SITEMAP_INDEX = f"{_BASE}/sitemap.xml"
|
||||||
|
_SITEMAP_RE = re.compile(r"https://vjav\.com/sitemap_vids_\d+\.xml")
|
||||||
|
_VID_RE = re.compile(r"/videos/(\d+)/")
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_duration(value: str | None) -> int | None:
|
||||||
|
"""`1:16:43` / `14:17` -> sekundy."""
|
||||||
|
if not value:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
parts = [int(p) for p in value.strip().split(":")]
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
sec = 0
|
||||||
|
for p in parts:
|
||||||
|
sec = sec * 60 + p
|
||||||
|
return sec or None
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_post_date(value: str | None) -> date | None:
|
||||||
|
if not value:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
return datetime.strptime(value.strip(), "%Y-%m-%d %H:%M:%S").date()
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
class VjavScraper(BaseBrowseScraper):
|
||||||
|
sitetag = "vjav"
|
||||||
|
|
||||||
|
def __init__(self, *args, **kwargs) -> None:
|
||||||
|
super().__init__(*args, **kwargs)
|
||||||
|
# Cache id-ów wideo (posortowane malejąco = newest first), per instancja/run.
|
||||||
|
self._sorted_ids: list[int] | None = None
|
||||||
|
|
||||||
|
# ---- browse: sitemap id-walk ------------------------------------------------
|
||||||
|
def _load_ids(self) -> None:
|
||||||
|
try:
|
||||||
|
res = browser_get(_SITEMAP_INDEX, timeout=self._timeout)
|
||||||
|
idx = res.text if hasattr(res, "text") else res
|
||||||
|
except Exception as e:
|
||||||
|
log.warning("vjav: sitemap index fetch fail: %s", e)
|
||||||
|
self._sorted_ids = []
|
||||||
|
return
|
||||||
|
maps = list(dict.fromkeys(_SITEMAP_RE.findall(idx)))
|
||||||
|
ids: set[int] = set()
|
||||||
|
for sm in maps:
|
||||||
|
try:
|
||||||
|
res = browser_get(sm, timeout=self._timeout)
|
||||||
|
body = res.text if hasattr(res, "text") else res
|
||||||
|
except Exception as e:
|
||||||
|
log.info("vjav: sitemap %s fetch fail: %s", sm, e)
|
||||||
|
continue
|
||||||
|
for m in _VID_RE.finditer(body):
|
||||||
|
ids.add(int(m.group(1)))
|
||||||
|
self._sorted_ids = sorted(ids, reverse=True)
|
||||||
|
log.info("vjav: załadowano %d id z %d sitemap", len(self._sorted_ids), len(maps))
|
||||||
|
|
||||||
|
def _meta_url(self, vid: int) -> str:
|
||||||
|
floor = (vid // 1000) * 1000
|
||||||
|
return f"{_BASE}/api/json/video/1/{floor}/{vid}/{vid}.json"
|
||||||
|
|
||||||
|
# ---- metadata JSON -> RawScene ----------------------------------------------
|
||||||
|
def _parse_meta(self, vid: int, v: dict) -> RawScene | None:
|
||||||
|
title = (v.get("title") or "").strip()
|
||||||
|
slug = (v.get("dir") or "").strip()
|
||||||
|
if not title or not slug:
|
||||||
|
return None
|
||||||
|
page_url = f"{_BASE}/videos/{vid}/{slug}/"
|
||||||
|
duration_sec = _parse_duration(v.get("duration"))
|
||||||
|
release_date = _parse_post_date(v.get("post_date"))
|
||||||
|
thumb = (v.get("thumbsrc") or v.get("thumb") or "").strip() or None
|
||||||
|
|
||||||
|
studio: RawStudio | None = None
|
||||||
|
ch = v.get("channel")
|
||||||
|
if isinstance(ch, dict) and (ch.get("title") or "").strip():
|
||||||
|
st = ch["title"].strip()
|
||||||
|
sg = slugify(st)
|
||||||
|
studio = RawStudio(external_id=f"vjav:studio:{sg}", name=st, slug=sg)
|
||||||
|
|
||||||
|
performers: list[RawPerformer] = []
|
||||||
|
models = v.get("models")
|
||||||
|
if isinstance(models, dict):
|
||||||
|
for mid, md in models.items():
|
||||||
|
nm = (md.get("title") or "").strip() if isinstance(md, dict) else ""
|
||||||
|
if nm:
|
||||||
|
performers.append(RawPerformer(external_id=f"vjav:model:{mid}", name=nm))
|
||||||
|
|
||||||
|
tags: list[RawTag] = []
|
||||||
|
seen_slugs: set[str] = set()
|
||||||
|
for coll in ("categories", "tags"):
|
||||||
|
c = v.get(coll)
|
||||||
|
if not isinstance(c, dict):
|
||||||
|
continue
|
||||||
|
for td in c.values():
|
||||||
|
nm = (td.get("title") or "").strip() if isinstance(td, dict) else ""
|
||||||
|
if not nm:
|
||||||
|
continue
|
||||||
|
sg = slugify(nm)
|
||||||
|
if not sg or sg in seen_slugs or len(sg) > 60:
|
||||||
|
continue
|
||||||
|
seen_slugs.add(sg)
|
||||||
|
tags.append(RawTag(external_id=f"vjav:tag:{sg}", name=nm, slug=sg))
|
||||||
|
|
||||||
|
return RawScene(
|
||||||
|
external_id=f"{self.sitetag}:{page_url}",
|
||||||
|
title=title,
|
||||||
|
release_date=release_date,
|
||||||
|
duration_sec=duration_sec,
|
||||||
|
url=page_url,
|
||||||
|
studio=studio,
|
||||||
|
performers=performers,
|
||||||
|
tags=tags,
|
||||||
|
playback_sources=[
|
||||||
|
RawPlaybackSource(
|
||||||
|
origin=f"tube:{self.sitetag}",
|
||||||
|
page_url=page_url,
|
||||||
|
duration_sec=duration_sec,
|
||||||
|
thumbnail_url=thumb,
|
||||||
|
)
|
||||||
|
],
|
||||||
|
raw={"source": "vjav_api", "id": vid},
|
||||||
|
)
|
||||||
|
|
||||||
|
def crawl_page(self, page: int) -> list[RawScene] | None:
|
||||||
|
if self._sorted_ids is None:
|
||||||
|
self._load_ids()
|
||||||
|
if not self._sorted_ids:
|
||||||
|
# [] gdy sitemap pusty (exhausted); None tylko przy fetch-failu indexu.
|
||||||
|
return [] if self._sorted_ids == [] else None
|
||||||
|
start = (page - 1) * _PER_PAGE
|
||||||
|
chunk = self._sorted_ids[start : start + _PER_PAGE]
|
||||||
|
if not chunk:
|
||||||
|
return []
|
||||||
|
out: list[RawScene] = []
|
||||||
|
for vid in chunk:
|
||||||
|
try:
|
||||||
|
res = browser_get(self._meta_url(vid), timeout=self._timeout, headers={"Referer": _BASE + "/"})
|
||||||
|
body = res.text if hasattr(res, "text") else res
|
||||||
|
v = json.loads(body).get("video")
|
||||||
|
except Exception as e:
|
||||||
|
log.info("vjav: meta fetch/parse fail id=%s: %s", vid, e)
|
||||||
|
continue
|
||||||
|
if not isinstance(v, dict):
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
raw = self._parse_meta(vid, v)
|
||||||
|
except Exception as e:
|
||||||
|
log.warning("vjav: meta parse fail id=%s: %s", vid, e)
|
||||||
|
continue
|
||||||
|
if raw is not None:
|
||||||
|
out.append(raw)
|
||||||
|
return out
|
||||||
|
|
||||||
|
# ---- HTML stuby (nieużywane, ABC wymaga) ------------------------------------
|
||||||
|
def _listing_url(self, page: int) -> str: # pragma: no cover
|
||||||
|
raise NotImplementedError("VjavScraper używa crawl_page (sitemap+API)")
|
||||||
|
|
||||||
|
def _extract_scene_urls(self, listing_html: str) -> list[str]: # pragma: no cover
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
|
def _parse_detail(self, scene_url: str, detail_html: str) -> RawScene | None: # pragma: no cover
|
||||||
|
raise NotImplementedError
|
||||||
|
|
@ -1,19 +1,127 @@
|
||||||
"""watchporn.to — direct HTML scrape.
|
"""watchporn.to — browse scraper (KVS engine). Re-enabled 2026-07-02.
|
||||||
|
|
||||||
Search: `https://watchporn.to/page/<n>/?s=<q>` (WordPress).
|
Był search-scraperem (`?s=`), zamarzł, a potem site przebudowano na KVS z nowym
|
||||||
Scene URL: `https://watchporn.to/videos/<slug>/`.
|
layoutem. DoodStream-CAPTCHA (powód wyłączenia 2026-05-12) zniknął — teraz KVS
|
||||||
|
flashvars `get_file` direct mp4 (extractor `watchporn`, VPS-side, token nie IP-bound).
|
||||||
|
|
||||||
|
Browse `/latest-updates/` → detail page:
|
||||||
|
- title: og:title ("Studio/Creator - Scene Title")
|
||||||
|
- duration + release_date: JSON-LD "duration" (ISO) + "uploadDate"
|
||||||
|
- performerzy: `/models/<slug>/` (pomijamy numeryczne id-slugi), nazwa z tekstu linku
|
||||||
|
- tagi: `/tags/<slug>/`
|
||||||
|
- studio: pierwszy `/categories/<slug>/` (np. EvilAngel, ManyVids)
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import html
|
||||||
import re
|
import re
|
||||||
|
|
||||||
from app.connectors.direct_scrapers._search_base import BaseSearchScraper
|
from app.connectors.base import (
|
||||||
|
RawPerformer,
|
||||||
|
RawPlaybackSource,
|
||||||
class WatchPornScraper(BaseSearchScraper):
|
RawScene,
|
||||||
sitetag = "watchporn"
|
RawStudio,
|
||||||
_search_url_template = "https://watchporn.to/page/{page}/?s={query}"
|
RawTag,
|
||||||
_scene_url_re = re.compile(
|
)
|
||||||
r'href="(?P<url>https://watchporn\.to/videos/(?P<slug>[a-z0-9][a-z0-9\-]+))/"',
|
from app.connectors.direct_scrapers._browse_base import BaseBrowseScraper, meta_content
|
||||||
re.IGNORECASE,
|
from app.connectors.direct_scrapers._playtube import _parse_iso_date, _parse_iso_duration
|
||||||
|
from app.normalize.text import slugify
|
||||||
|
|
||||||
|
_BASE = "https://watchporn.to"
|
||||||
|
_SCENE_URL_RE = re.compile(r'href="(https://watchporn\.to/video/\d+/[a-z0-9\-]+/)"', re.IGNORECASE)
|
||||||
|
_MODEL_RE = re.compile(r'href="https://watchporn\.to/models/([a-z0-9\-]+)/"[^>]*>([^<]+)', re.IGNORECASE)
|
||||||
|
_TAG_RE = re.compile(r'href="https://watchporn\.to/tags/([a-z0-9\-]+)/"[^>]*>([^<]+)', re.IGNORECASE)
|
||||||
|
_CAT_RE = re.compile(r'href="https://watchporn\.to/categories/([a-z0-9\-]+)/"[^>]*>([^<]+)', re.IGNORECASE)
|
||||||
|
_DUR_RE = re.compile(r'"duration"\s*:\s*"([^"]+)"')
|
||||||
|
_UPLOAD_RE = re.compile(r'"uploadDate"\s*:\s*"([^"]+)"')
|
||||||
|
_VIDEO_ID_RE = re.compile(r"/video/(\d+)/")
|
||||||
|
|
||||||
|
|
||||||
|
def _derive_thumb(scene_url: str) -> str | None:
|
||||||
|
"""KVS trzyma poster pod stałym wzorem `contents/videos_screenshots/<id//1000*1000>/
|
||||||
|
<id>/preview.jpg` (zweryfikowane). Fallback gdy detail page nie ma og:image, żeby
|
||||||
|
kafelek na liście miał miniaturkę OD RAZU (bez czekania na auto-enrich w SceneDetail)."""
|
||||||
|
m = _VIDEO_ID_RE.search(scene_url)
|
||||||
|
if not m:
|
||||||
|
return None
|
||||||
|
vid = int(m.group(1))
|
||||||
|
return f"{_BASE}/contents/videos_screenshots/{vid // 1000 * 1000}/{vid}/preview.jpg"
|
||||||
|
|
||||||
|
|
||||||
|
class WatchPornScraper(BaseBrowseScraper):
|
||||||
|
sitetag = "watchporn"
|
||||||
|
|
||||||
|
def _listing_url(self, page: int) -> str:
|
||||||
|
return f"{_BASE}/latest-updates/" if page <= 1 else f"{_BASE}/latest-updates/{page}/"
|
||||||
|
|
||||||
|
def _extract_scene_urls(self, listing_html: str) -> list[str]:
|
||||||
|
seen: set[str] = set()
|
||||||
|
out: list[str] = []
|
||||||
|
for m in _SCENE_URL_RE.finditer(listing_html):
|
||||||
|
url = m.group(1)
|
||||||
|
if url not in seen:
|
||||||
|
seen.add(url)
|
||||||
|
out.append(url)
|
||||||
|
return out
|
||||||
|
|
||||||
|
def _parse_detail(self, scene_url: str, detail_html: str) -> RawScene | None:
|
||||||
|
title = (meta_content(detail_html, property="og:title") or "").strip()
|
||||||
|
if not title:
|
||||||
|
return None
|
||||||
|
|
||||||
|
dm = _DUR_RE.search(detail_html)
|
||||||
|
duration_sec = _parse_iso_duration(dm.group(1)) if dm else None
|
||||||
|
um = _UPLOAD_RE.search(detail_html)
|
||||||
|
release_date = _parse_iso_date(um.group(1)) if um else None
|
||||||
|
|
||||||
|
performers: list[RawPerformer] = []
|
||||||
|
seen_p: set[str] = set()
|
||||||
|
for m in _MODEL_RE.finditer(detail_html):
|
||||||
|
slug = m.group(1)
|
||||||
|
name = html.unescape(m.group(2)).strip()
|
||||||
|
if slug.isdigit() or slug in seen_p or not name:
|
||||||
|
continue
|
||||||
|
seen_p.add(slug)
|
||||||
|
performers.append(RawPerformer(external_id=f"{self.sitetag}:model:{slug}", name=name))
|
||||||
|
|
||||||
|
tags: list[RawTag] = []
|
||||||
|
seen_t: set[str] = set()
|
||||||
|
for m in _TAG_RE.finditer(detail_html):
|
||||||
|
slug = m.group(1)
|
||||||
|
name = html.unescape(m.group(2)).strip()
|
||||||
|
if slug in seen_t or not name:
|
||||||
|
continue
|
||||||
|
seen_t.add(slug)
|
||||||
|
tags.append(RawTag(external_id=f"{self.sitetag}:tag:{slug}", name=name, slug=slug))
|
||||||
|
|
||||||
|
thumbnail_url = (
|
||||||
|
meta_content(detail_html, property="og:image") or _derive_thumb(scene_url)
|
||||||
|
)
|
||||||
|
|
||||||
|
studio: RawStudio | None = None
|
||||||
|
cm = _CAT_RE.search(detail_html)
|
||||||
|
if cm:
|
||||||
|
cname = html.unescape(cm.group(2)).strip()
|
||||||
|
if cname:
|
||||||
|
studio = RawStudio(
|
||||||
|
external_id=f"{self.sitetag}:studio:{slugify(cname)}", name=cname, slug=slugify(cname)
|
||||||
|
)
|
||||||
|
|
||||||
|
return RawScene(
|
||||||
|
external_id=f"{self.sitetag}:{scene_url}",
|
||||||
|
title=title,
|
||||||
|
release_date=release_date,
|
||||||
|
duration_sec=duration_sec,
|
||||||
|
url=scene_url,
|
||||||
|
studio=studio,
|
||||||
|
performers=performers,
|
||||||
|
tags=tags,
|
||||||
|
playback_sources=[
|
||||||
|
RawPlaybackSource(
|
||||||
|
origin=f"tube:{self.sitetag}",
|
||||||
|
page_url=scene_url,
|
||||||
|
duration_sec=duration_sec,
|
||||||
|
thumbnail_url=thumbnail_url,
|
||||||
|
)
|
||||||
|
],
|
||||||
)
|
)
|
||||||
|
|
|
||||||
129
app/connectors/direct_scrapers/xnxx_browse.py
Normal file
129
app/connectors/direct_scrapers/xnxx_browse.py
Normal file
|
|
@ -0,0 +1,129 @@
|
||||||
|
"""xnxx.com — BROWSE scraper (JSON-LD), obok search scrapera.
|
||||||
|
|
||||||
|
Detail page ma JSON-LD VideoObject (name/duration/uploadDate/thumbnail) — i TYLKO to
|
||||||
|
bierzemy. Mimo wspólnego silnika z xvideos, xnxx detail NIE wystawia w SSR linków
|
||||||
|
`/models/` ani `/tags/` (0 wystąpień, ładowane JS-em) → performerów/tagi dorabia
|
||||||
|
canonical-merge + istniejący performer-search. XnxxScraper (search) zostaje; browse
|
||||||
|
dokłada sygnał świeżości. Tytuł z JSON-LD bywa HTML-encoded (`,`/`!`) →
|
||||||
|
html.unescape.
|
||||||
|
|
||||||
|
Listing: xnxx NIE ma czystego SSR `/new/` (404), ale `/best/<YYYY-MM>/<page>` jest
|
||||||
|
SSR (linki /video-<id>/ w surowym HTML). Bierzemy bieżący miesiąc — pokrywa świeży
|
||||||
|
content (sortowanie best-of-month, nie ściśle chronologiczne, ale dla sygnału
|
||||||
|
świeżości wystarcza; ścisłą chronologię i tak daje performer-search). Homepage
|
||||||
|
xnxx jest JS-renderowany (0 linków w surowym HTML), stąd /best/.
|
||||||
|
|
||||||
|
Playback bez zmian (extractor `xnxxcom`). Phash pominięty (xnxx crop-thumbnaile,
|
||||||
|
0% hit do canonical — jak xvideos).
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import html
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import re
|
||||||
|
from datetime import date, datetime
|
||||||
|
|
||||||
|
from app.connectors.base import RawPlaybackSource, RawScene
|
||||||
|
from app.connectors.direct_scrapers._browse_base import BaseBrowseScraper, meta_content
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
_BASE = "https://www.xnxx.com"
|
||||||
|
_SCENE_URL_RE = re.compile(r'href="(/video-[a-z0-9]+/[a-z0-9_\-]+)"', re.IGNORECASE)
|
||||||
|
_JSONLD_RE = re.compile(
|
||||||
|
r'<script[^>]+type=["\']application/ld\+json["\'][^>]*>(.*?)</script>', re.IGNORECASE | re.DOTALL
|
||||||
|
)
|
||||||
|
_SETTITLE_RE = re.compile(r"html5player\.setVideoTitle\('([^']+)'\)")
|
||||||
|
_ISO_DUR_RE = re.compile(r"PT(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?", re.IGNORECASE)
|
||||||
|
|
||||||
|
|
||||||
|
def _dur_to_sec(value: str | None) -> int | None:
|
||||||
|
if not value:
|
||||||
|
return None
|
||||||
|
m = _ISO_DUR_RE.match(str(value).strip())
|
||||||
|
if not m:
|
||||||
|
return None
|
||||||
|
total = int(m.group(1) or 0) * 3600 + int(m.group(2) or 0) * 60 + int(m.group(3) or 0)
|
||||||
|
return total or None
|
||||||
|
|
||||||
|
|
||||||
|
def _iso_date(value: str | None) -> date | None:
|
||||||
|
if not value:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
return datetime.fromisoformat(str(value).replace("Z", "+00:00")).date()
|
||||||
|
except ValueError:
|
||||||
|
m = re.match(r"(\d{4}-\d{2}-\d{2})", str(value))
|
||||||
|
return date.fromisoformat(m.group(1)) if m else None
|
||||||
|
|
||||||
|
|
||||||
|
def _video_object(html: str) -> dict | None:
|
||||||
|
for m in _JSONLD_RE.finditer(html):
|
||||||
|
raw = m.group(1).strip()
|
||||||
|
if not raw:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
data = json.loads(raw)
|
||||||
|
except (json.JSONDecodeError, ValueError):
|
||||||
|
continue
|
||||||
|
items = data if isinstance(data, list) else (data.get("@graph", [data]) if isinstance(data, dict) else [])
|
||||||
|
for obj in items:
|
||||||
|
if isinstance(obj, dict) and obj.get("@type") == "VideoObject":
|
||||||
|
return obj
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
class XnxxBrowseScraper(BaseBrowseScraper):
|
||||||
|
sitetag = "xnxxcom"
|
||||||
|
|
||||||
|
def _listing_url(self, page: int) -> str:
|
||||||
|
month = datetime.now().strftime("%Y-%m")
|
||||||
|
return f"{_BASE}/best/{month}/{page}"
|
||||||
|
|
||||||
|
def _extract_scene_urls(self, listing_html: str) -> list[str]:
|
||||||
|
seen: set[str] = set()
|
||||||
|
out: list[str] = []
|
||||||
|
for m in _SCENE_URL_RE.finditer(listing_html):
|
||||||
|
url = f"{_BASE}{m.group(1)}"
|
||||||
|
if url in seen:
|
||||||
|
continue
|
||||||
|
seen.add(url)
|
||||||
|
out.append(url)
|
||||||
|
return out
|
||||||
|
|
||||||
|
def _parse_detail(self, scene_url: str, detail_html: str) -> RawScene | None:
|
||||||
|
video = _video_object(detail_html) or {}
|
||||||
|
title = (video.get("name") or "").strip()
|
||||||
|
if not title:
|
||||||
|
m = _SETTITLE_RE.search(detail_html)
|
||||||
|
title = m.group(1).strip() if m else (meta_content(detail_html, property="og:title") or "").strip()
|
||||||
|
title = html.unescape(title).strip()
|
||||||
|
if not title:
|
||||||
|
return None
|
||||||
|
|
||||||
|
duration_sec = _dur_to_sec(video.get("duration"))
|
||||||
|
release_date = _iso_date(video.get("uploadDate") or video.get("datePublished"))
|
||||||
|
thumbnail_url = video.get("thumbnailUrl") or meta_content(detail_html, property="og:image")
|
||||||
|
if isinstance(thumbnail_url, list):
|
||||||
|
thumbnail_url = thumbnail_url[0] if thumbnail_url else None
|
||||||
|
|
||||||
|
# performers/tags puste — xnxx detail nie ma ich w SSR (patrz docstring).
|
||||||
|
return RawScene(
|
||||||
|
external_id=f"{self.sitetag}:{scene_url}",
|
||||||
|
title=title,
|
||||||
|
duration_sec=duration_sec,
|
||||||
|
release_date=release_date,
|
||||||
|
url=scene_url,
|
||||||
|
performers=[],
|
||||||
|
tags=[],
|
||||||
|
playback_sources=[
|
||||||
|
RawPlaybackSource(
|
||||||
|
origin=f"tube:{self.sitetag}",
|
||||||
|
page_url=scene_url,
|
||||||
|
duration_sec=duration_sec,
|
||||||
|
thumbnail_url=thumbnail_url,
|
||||||
|
)
|
||||||
|
],
|
||||||
|
raw={"source": "xnxx_browse"},
|
||||||
|
)
|
||||||
156
app/connectors/direct_scrapers/xvideos_browse.py
Normal file
156
app/connectors/direct_scrapers/xvideos_browse.py
Normal file
|
|
@ -0,0 +1,156 @@
|
||||||
|
"""xvideos.com — deep-crawl browse scraper (JSON-LD + page-parse).
|
||||||
|
|
||||||
|
xvideos SSR-uje JSON-LD VideoObject (duration, name, uploadDate) ORAZ na detail-stronie
|
||||||
|
linki `/models/<slug>` (performerzy tej sceny) + `/tags/<slug>` (tagi). Sample 2026-06-03
|
||||||
|
(15 scen): median ~10.5min, 93% ≥3min — dobry full-scene content (nie trailery).
|
||||||
|
|
||||||
|
Mega-katalog (~13M) → deep_crawl z per-tube page-cap (xvideoscom w deep_crawl._PAGE_CAP),
|
||||||
|
żeby nie monopolizował round-robin ani nie zalał bazy. Listing: /new/<page> (newest).
|
||||||
|
Scene: /video.<hash>/<slug>. Playback: page_url + origin tube:xvideoscom (istniejący
|
||||||
|
extractor `xvideoscom` resolvuje stream mobile-side). Phash pominięty (xvideos robi
|
||||||
|
własne crop-thumbnaile — 0% hit do canonical, jak fullmovies/hdporn).
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import re
|
||||||
|
from datetime import date, datetime
|
||||||
|
|
||||||
|
from app.connectors.base import RawPerformer, RawPlaybackSource, RawScene, RawTag
|
||||||
|
from app.connectors.direct_scrapers._browse_base import BaseBrowseScraper, meta_content
|
||||||
|
from app.normalize.text import slugify
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
_BASE = "https://www.xvideos.com"
|
||||||
|
_SCENE_URL_RE = re.compile(r'href="(/video\.[0-9a-z]+/[a-z0-9_]+)"', re.IGNORECASE)
|
||||||
|
_JSONLD_RE = re.compile(
|
||||||
|
r'<script[^>]+type=["\']application/ld\+json["\'][^>]*>(.*?)</script>', re.IGNORECASE | re.DOTALL
|
||||||
|
)
|
||||||
|
# Model anchor: `<a href="/models/<slug>" ...><span ...></span><span class="name">Display Name</span>...`.
|
||||||
|
# Nazwa siedzi w ZAGNIEŻDŻONYM <span class="name">, nie jako bezpośredni tekst anchora —
|
||||||
|
# poprzedni wzorzec `>([^<]{2,60})</a>` wymagał tekstu zaraz po `>` i NIGDY nie pasował do
|
||||||
|
# obecnego markupu xvideos → wszystkie browse-scraped sceny xvideos lądowały z 0 performerami
|
||||||
|
# (bug-report 2026-06-07 "czemu nie ma aktorów, są na stronie"). `(?:(?!</a>).)*?` trzyma match
|
||||||
|
# w obrębie jednego anchora (nie przecieka do nazwy następnego modela gdy span.name brak).
|
||||||
|
_MODEL_RE = re.compile(
|
||||||
|
r'/models/([a-z0-9_-]+)"(?:(?!</a>).)*?<span class="name">\s*([^<]{2,60})</span>',
|
||||||
|
re.IGNORECASE | re.DOTALL,
|
||||||
|
)
|
||||||
|
_TAG_RE = re.compile(r'href="/tags/([a-z0-9_-]+)"', re.IGNORECASE)
|
||||||
|
_SETTITLE_RE = re.compile(r"html5player\.setVideoTitle\('([^']+)'\)")
|
||||||
|
_ISO_DUR_RE = re.compile(r"PT(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?", re.IGNORECASE)
|
||||||
|
|
||||||
|
|
||||||
|
def _dur_to_sec(value: str | None) -> int | None:
|
||||||
|
if not value:
|
||||||
|
return None
|
||||||
|
m = _ISO_DUR_RE.match(str(value).strip())
|
||||||
|
if not m:
|
||||||
|
return None
|
||||||
|
total = int(m.group(1) or 0) * 3600 + int(m.group(2) or 0) * 60 + int(m.group(3) or 0)
|
||||||
|
return total or None
|
||||||
|
|
||||||
|
|
||||||
|
def _iso_date(value: str | None) -> date | None:
|
||||||
|
if not value:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
return datetime.fromisoformat(str(value).replace("Z", "+00:00")).date()
|
||||||
|
except ValueError:
|
||||||
|
m = re.match(r"(\d{4}-\d{2}-\d{2})", str(value))
|
||||||
|
return date.fromisoformat(m.group(1)) if m else None
|
||||||
|
|
||||||
|
|
||||||
|
def _video_object(html: str) -> dict | None:
|
||||||
|
for m in _JSONLD_RE.finditer(html):
|
||||||
|
raw = m.group(1).strip()
|
||||||
|
if not raw:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
data = json.loads(raw)
|
||||||
|
except (json.JSONDecodeError, ValueError):
|
||||||
|
continue
|
||||||
|
items = data if isinstance(data, list) else (data.get("@graph", [data]) if isinstance(data, dict) else [])
|
||||||
|
for obj in items:
|
||||||
|
if isinstance(obj, dict) and obj.get("@type") == "VideoObject":
|
||||||
|
return obj
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
class XVideosBrowseScraper(BaseBrowseScraper):
|
||||||
|
sitetag = "xvideoscom"
|
||||||
|
|
||||||
|
def _listing_url(self, page: int) -> str:
|
||||||
|
return f"{_BASE}/new/{page}"
|
||||||
|
|
||||||
|
def _extract_scene_urls(self, listing_html: str) -> list[str]:
|
||||||
|
seen: set[str] = set()
|
||||||
|
out: list[str] = []
|
||||||
|
for m in _SCENE_URL_RE.finditer(listing_html):
|
||||||
|
url = f"{_BASE}{m.group(1)}"
|
||||||
|
if url in seen:
|
||||||
|
continue
|
||||||
|
seen.add(url)
|
||||||
|
out.append(url)
|
||||||
|
return out
|
||||||
|
|
||||||
|
def _parse_detail(self, scene_url: str, detail_html: str) -> RawScene | None:
|
||||||
|
video = _video_object(detail_html) or {}
|
||||||
|
|
||||||
|
title = (video.get("name") or "").strip()
|
||||||
|
if not title:
|
||||||
|
m = _SETTITLE_RE.search(detail_html)
|
||||||
|
title = m.group(1).strip() if m else (meta_content(detail_html, property="og:title") or "").strip()
|
||||||
|
if not title:
|
||||||
|
return None
|
||||||
|
|
||||||
|
duration_sec = _dur_to_sec(video.get("duration"))
|
||||||
|
release_date = _iso_date(video.get("uploadDate") or video.get("datePublished"))
|
||||||
|
thumbnail_url = video.get("thumbnailUrl") or meta_content(detail_html, property="og:image")
|
||||||
|
if isinstance(thumbnail_url, list):
|
||||||
|
thumbnail_url = thumbnail_url[0] if thumbnail_url else None
|
||||||
|
|
||||||
|
# Performerzy: linki /models/<slug> (scene-specific; nav xvideos używa innego patternu).
|
||||||
|
performers: list[RawPerformer] = []
|
||||||
|
seen_perf: set[str] = set()
|
||||||
|
for m in _MODEL_RE.finditer(detail_html):
|
||||||
|
slug, name = m.group(1), m.group(2).strip()
|
||||||
|
if not name or slug in seen_perf or name.lower() in ("models", "pornstars"):
|
||||||
|
continue
|
||||||
|
seen_perf.add(slug)
|
||||||
|
performers.append(RawPerformer(external_id=f"{self.sitetag}:model:{slug}", name=name))
|
||||||
|
if len(performers) >= 8:
|
||||||
|
break
|
||||||
|
|
||||||
|
# Tagi: /tags/<slug>.
|
||||||
|
tags: list[RawTag] = []
|
||||||
|
seen_tag: set[str] = set()
|
||||||
|
for m in _TAG_RE.finditer(detail_html):
|
||||||
|
slug = m.group(1)
|
||||||
|
if slug in seen_tag or len(slug) > 60:
|
||||||
|
continue
|
||||||
|
seen_tag.add(slug)
|
||||||
|
tags.append(RawTag(external_id=f"{self.sitetag}:tag:{slug}", name=slug.replace("-", " "), slug=slug))
|
||||||
|
if len(tags) >= 15:
|
||||||
|
break
|
||||||
|
|
||||||
|
return RawScene(
|
||||||
|
external_id=f"{self.sitetag}:{scene_url}",
|
||||||
|
title=title,
|
||||||
|
duration_sec=duration_sec,
|
||||||
|
release_date=release_date,
|
||||||
|
url=scene_url,
|
||||||
|
performers=performers,
|
||||||
|
tags=tags,
|
||||||
|
playback_sources=[
|
||||||
|
RawPlaybackSource(
|
||||||
|
origin=f"tube:{self.sitetag}",
|
||||||
|
page_url=scene_url,
|
||||||
|
duration_sec=duration_sec,
|
||||||
|
thumbnail_url=thumbnail_url,
|
||||||
|
)
|
||||||
|
],
|
||||||
|
raw={"source": "xvideos_browse"},
|
||||||
|
)
|
||||||
321
app/connectors/direct_scrapers/yesporn.py
Normal file
321
app/connectors/direct_scrapers/yesporn.py
Normal file
|
|
@ -0,0 +1,321 @@
|
||||||
|
"""yesporn.vip — latest-vids browse scraper.
|
||||||
|
|
||||||
|
Dołączony 2026-05-27. Identyfikowany przez user audit jako "scraper-of-paysites"
|
||||||
|
(DogFart / HardX / TeamSkeet / Vixen / Brazzers content). Wcześniejszy theporndude
|
||||||
|
audit pomylił domeny: `yespornvip.com` (z theporndude rankingu) redirectuje przez
|
||||||
|
pdude.link do `porndudecams.com` affiliate spam — kanoniczna domena ma TLD `.vip`.
|
||||||
|
|
||||||
|
Czemu wart (parity z porndoe):
|
||||||
|
- **JSON-LD VideoObject** w każdym scene page: name, description, uploadDate
|
||||||
|
(ISO `YYYY-MM-DDTHH:MM:SS`), duration (ISO `PT0H39M00S`), thumbnailUrl
|
||||||
|
(BunnyCDN: `yesnn.b-cdn.net/contents/videos_screenshots/...`).
|
||||||
|
- **`<meta property="video:duration" content="2340">`** — durations już w sekundach
|
||||||
|
(fallback gdy ISO-duration parse fail).
|
||||||
|
- **`<meta property="video:release_date">`** — ISO 8601 z timezone, redundant z
|
||||||
|
JSON-LD uploadDate ale czystszy format.
|
||||||
|
- **`<meta property="video:tag">`** (multiple) — kanoniczna lista tagów (np.
|
||||||
|
"Big Ass", "Threesome"). Główne źródło tagów; alternatywnie DOM ma `btn gold`
|
||||||
|
linki ale te miksują performerów/studio z tagami.
|
||||||
|
- **Studio + Performers**: oba w sekcji `<a class="btn gold" href="/channels/<slug>/">`
|
||||||
|
(studio, singular) i `<a class="btn gold" href="/models/<slug>/">` (performerzy,
|
||||||
|
multiple). Slugi mają stable per-type salt (`*-i459s7` dla modeli, `*-7p72tp`
|
||||||
|
dla channels) — zachowują się jak hash z site-version, ale stabilne przez
|
||||||
|
sesje.
|
||||||
|
|
||||||
|
External_id strategia: `yespornvip:<numeric_video_id>` (`/video/69841/...` → `69841`).
|
||||||
|
Slug w URL ma `*-npu57w` suffix który wygląda na stałe-per-page-type, ale id
|
||||||
|
numeryczne jest bezpieczniejsze gdyby site zmienił salt.
|
||||||
|
|
||||||
|
URL patterns:
|
||||||
|
- Listing: `/latest-updates/` (page 1) / `/latest-updates/N/` (page>1)
|
||||||
|
- Scene: `/video/<id>/<slug>/` (id numeryczny, slug = title slug + 6-char salt)
|
||||||
|
- Studio: `/channels/<slug>/`
|
||||||
|
- Performer: `/models/<slug>/`
|
||||||
|
- Search: `/search/<query>/` (nie używane w browse-mode — można dorobić jako
|
||||||
|
osobny tryb dla performer-driven backfill jeśli będzie potrzeba)
|
||||||
|
|
||||||
|
Playback: download endpoint `/view_video_download.php?id=<id>&format=<480|720|1080>`
|
||||||
|
z `data-attach-session="PHPSESSID"` — wymaga session cookie, więc nie direct mp4
|
||||||
|
z server-side. Plus jest `embedUrl: /embed/<id>` w JSON-LD. Extractor →
|
||||||
|
`_vps_blocked_fallback.extract` (zgodne z pre-public bandwidth/anonymity policy):
|
||||||
|
mobile WebView fetcha embed z phone IP, INJECTED_JS scrape'uje `<video>.src`.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import html as html_mod
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import re
|
||||||
|
from datetime import date, datetime
|
||||||
|
|
||||||
|
from app.connectors.base import (
|
||||||
|
RawFingerprint,
|
||||||
|
RawPerformer,
|
||||||
|
RawPlaybackSource,
|
||||||
|
RawScene,
|
||||||
|
RawStudio,
|
||||||
|
RawTag,
|
||||||
|
)
|
||||||
|
from app.connectors.direct_scrapers._browse_base import (
|
||||||
|
BaseBrowseScraper,
|
||||||
|
compute_thumbnail_phash,
|
||||||
|
meta_content,
|
||||||
|
)
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
_BASE = "https://yesporn.vip"
|
||||||
|
|
||||||
|
# Listing — scene URLs w listing HTML: `<a href="https://yesporn.vip/video/<id>/<slug>/">`.
|
||||||
|
# Slug `<slug>` zawiera stable per-type salt (`*-npu57w` dla videos).
|
||||||
|
_SCENE_URL_RE = re.compile(
|
||||||
|
r'href="(https://yesporn\.vip/video/(\d+)/[a-z0-9\-]+/)"',
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
_VIDEO_ID_RE = re.compile(r"/video/(\d+)/", re.IGNORECASE)
|
||||||
|
|
||||||
|
# Studio (singular) i performerzy (multiple) w `<a class="btn gold" href="...">`.
|
||||||
|
# Studio: `/channels/<slug>/`. Performer: `/models/<slug>/`. Tekst linka =
|
||||||
|
# nazwa wyświetlana (może zawierać CSS-y/inne tagi, więc strip tagów po fakcie).
|
||||||
|
_STUDIO_LINK_RE = re.compile(
|
||||||
|
r'<a\s+class="btn\s+gold"\s+href="https://yesporn\.vip/channels/([a-z0-9\-]+)/"[^>]*>(.*?)</a>',
|
||||||
|
re.IGNORECASE | re.DOTALL,
|
||||||
|
)
|
||||||
|
_PERFORMER_LINK_RE = re.compile(
|
||||||
|
r'<a\s+class="btn\s+gold"\s+href="https://yesporn\.vip/models/([a-z0-9\-]+)/"[^>]*>(.*?)</a>',
|
||||||
|
re.IGNORECASE | re.DOTALL,
|
||||||
|
)
|
||||||
|
_HTML_TAG_RE = re.compile(r"<[^>]+>")
|
||||||
|
|
||||||
|
# JSON-LD VideoObject — pełny blok między `<script type="application/ld+json">` tagami.
|
||||||
|
_JSONLD_RE = re.compile(
|
||||||
|
r'<script[^>]+type=["\']application/ld\+json["\'][^>]*>(.*?)</script>',
|
||||||
|
re.IGNORECASE | re.DOTALL,
|
||||||
|
)
|
||||||
|
|
||||||
|
# `<meta property="video:tag" content="Big Ass">` — multiple, jeden tag per meta.
|
||||||
|
_META_TAG_RE = re.compile(
|
||||||
|
r'<meta\s+property=["\']video:tag["\']\s+content=["\']([^"\']+)["\']',
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
|
||||||
|
# ISO 8601 duration `PT0H39M00S` / `PT39M0S` / `PT45S`.
|
||||||
|
_ISO_DUR_RE = re.compile(
|
||||||
|
r"^P?T?(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?$", re.IGNORECASE
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_iso_duration(value: str | None) -> int | None:
|
||||||
|
if not value:
|
||||||
|
return None
|
||||||
|
m = _ISO_DUR_RE.match(value.strip())
|
||||||
|
if not m:
|
||||||
|
return None
|
||||||
|
h = int(m.group(1) or 0)
|
||||||
|
mn = int(m.group(2) or 0)
|
||||||
|
s = int(m.group(3) or 0)
|
||||||
|
total = h * 3600 + mn * 60 + s
|
||||||
|
return total or None
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_iso_date(value: str | None) -> date | None:
|
||||||
|
"""`2026-05-26T19:23:29Z` / `2026-05-26T19:23:29.EDT` → date."""
|
||||||
|
if not value:
|
||||||
|
return None
|
||||||
|
# yesporn emituje `.EDT` jako "timezone" w JSON-LD uploadDate — strip żeby
|
||||||
|
# `fromisoformat` nie crash'ował. video:release_date meta ma czysty `Z`.
|
||||||
|
cleaned = re.sub(r"\.[A-Z]{2,4}$", "", value.strip())
|
||||||
|
try:
|
||||||
|
return datetime.fromisoformat(cleaned.replace("Z", "+00:00")).date()
|
||||||
|
except ValueError:
|
||||||
|
m = re.match(r"(\d{4}-\d{2}-\d{2})", cleaned)
|
||||||
|
if m:
|
||||||
|
try:
|
||||||
|
return date.fromisoformat(m.group(1))
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _iter_jsonld_objects(data: object):
|
||||||
|
"""Spłaszcza JSON-LD: dict / list / @graph → strumień dict-ów."""
|
||||||
|
if isinstance(data, dict):
|
||||||
|
graph = data.get("@graph")
|
||||||
|
if isinstance(graph, list):
|
||||||
|
for item in graph:
|
||||||
|
yield from _iter_jsonld_objects(item)
|
||||||
|
else:
|
||||||
|
yield data
|
||||||
|
elif isinstance(data, list):
|
||||||
|
for item in data:
|
||||||
|
yield from _iter_jsonld_objects(item)
|
||||||
|
|
||||||
|
|
||||||
|
def _extract_video_object(html: str) -> dict | None:
|
||||||
|
for m in _JSONLD_RE.finditer(html):
|
||||||
|
raw = m.group(1).strip()
|
||||||
|
if not raw:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
data = json.loads(raw)
|
||||||
|
except (json.JSONDecodeError, ValueError):
|
||||||
|
continue
|
||||||
|
for obj in _iter_jsonld_objects(data):
|
||||||
|
if obj.get("@type") == "VideoObject":
|
||||||
|
return obj
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _clean_link_text(raw: str) -> str:
|
||||||
|
"""Strip HTML tagów + decode entities + whitespace normalize."""
|
||||||
|
text = _HTML_TAG_RE.sub("", raw)
|
||||||
|
text = html_mod.unescape(text)
|
||||||
|
return " ".join(text.split()).strip()
|
||||||
|
|
||||||
|
|
||||||
|
class YesPornVipScraper(BaseBrowseScraper):
|
||||||
|
sitetag = "yespornvip"
|
||||||
|
|
||||||
|
def _listing_url(self, page: int) -> str:
|
||||||
|
if page <= 1:
|
||||||
|
return f"{_BASE}/latest-updates/"
|
||||||
|
return f"{_BASE}/latest-updates/{page}/"
|
||||||
|
|
||||||
|
def _extract_scene_urls(self, listing_html: str) -> list[str]:
|
||||||
|
seen: set[str] = set()
|
||||||
|
out: list[str] = []
|
||||||
|
for m in _SCENE_URL_RE.finditer(listing_html):
|
||||||
|
url = m.group(1)
|
||||||
|
if url in seen:
|
||||||
|
continue
|
||||||
|
seen.add(url)
|
||||||
|
out.append(url)
|
||||||
|
return out
|
||||||
|
|
||||||
|
def _parse_detail(self, scene_url: str, detail_html: str) -> RawScene | None:
|
||||||
|
video = _extract_video_object(detail_html)
|
||||||
|
if not video:
|
||||||
|
log.info("yesporn: no JSON-LD VideoObject on %s", scene_url)
|
||||||
|
return None
|
||||||
|
|
||||||
|
title = (video.get("name") or "").strip()
|
||||||
|
if not title:
|
||||||
|
return None
|
||||||
|
|
||||||
|
video_id_m = _VIDEO_ID_RE.search(scene_url)
|
||||||
|
video_id = video_id_m.group(1) if video_id_m else None
|
||||||
|
|
||||||
|
description = (video.get("description") or "").strip() or None
|
||||||
|
|
||||||
|
# Duration: preferuj `<meta property="video:duration">` (czyste sekundy),
|
||||||
|
# fallback do JSON-LD ISO format.
|
||||||
|
duration_sec: int | None = None
|
||||||
|
meta_dur = meta_content(detail_html, property="video:duration")
|
||||||
|
if meta_dur and meta_dur.isdigit():
|
||||||
|
duration_sec = int(meta_dur) or None
|
||||||
|
if duration_sec is None:
|
||||||
|
duration_sec = _parse_iso_duration(video.get("duration"))
|
||||||
|
|
||||||
|
# Release date: preferuj `<meta property="video:release_date">` (czystszy
|
||||||
|
# format z timezone), fallback do JSON-LD uploadDate.
|
||||||
|
release_date = _parse_iso_date(
|
||||||
|
meta_content(detail_html, property="video:release_date")
|
||||||
|
or video.get("uploadDate")
|
||||||
|
)
|
||||||
|
|
||||||
|
thumbnail_url = video.get("thumbnailUrl") or None
|
||||||
|
|
||||||
|
# Studio: pierwszy `btn gold` link do `/channels/<slug>/`. Strona renderuje
|
||||||
|
# tylko jednego per scenę (logo studia obok performerów).
|
||||||
|
studio: RawStudio | None = None
|
||||||
|
for m in _STUDIO_LINK_RE.finditer(detail_html):
|
||||||
|
slug = m.group(1).strip()
|
||||||
|
name = _clean_link_text(m.group(2))
|
||||||
|
if not name:
|
||||||
|
continue
|
||||||
|
studio = RawStudio(
|
||||||
|
external_id=f"{self.sitetag}:channel:{slug}",
|
||||||
|
name=name,
|
||||||
|
slug=slug,
|
||||||
|
)
|
||||||
|
break
|
||||||
|
|
||||||
|
# Performers: wszystkie `btn gold` linki do `/models/<slug>/` (multiple).
|
||||||
|
performers: list[RawPerformer] = []
|
||||||
|
seen_perf: set[str] = set()
|
||||||
|
for m in _PERFORMER_LINK_RE.finditer(detail_html):
|
||||||
|
slug = m.group(1).strip()
|
||||||
|
if slug in seen_perf:
|
||||||
|
continue
|
||||||
|
name = _clean_link_text(m.group(2))
|
||||||
|
if not name:
|
||||||
|
continue
|
||||||
|
seen_perf.add(slug)
|
||||||
|
performers.append(
|
||||||
|
RawPerformer(
|
||||||
|
external_id=f"{self.sitetag}:performer:{slug}",
|
||||||
|
name=name,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
# Tagi: `<meta property="video:tag" content="...">` (multiple).
|
||||||
|
# Deny-list: pomiń wszystkie all-lowercase tagi. yesporn.vip SEO-stuffuje
|
||||||
|
# `meta video:tag` tokenami z tytułu i imionami performerów + gibberish
|
||||||
|
# ("bella", "rose", "reverse", "deep", "throat", "ddca"), wszystkie always
|
||||||
|
# lowercase. Legit kategorie są zawsze Title Case ("Big Ass", "Deep
|
||||||
|
# Throat", "Blonde", "Gangbang") lub UPPER ("MILF", "BBW"). Potwierdzone
|
||||||
|
# w 20-scene dry-run 2026-05-27. Trade-off: stracimy hipotetyczne legit
|
||||||
|
# lowercase tagi (np. "interracial" gdyby site je nie capitalize'ował) —
|
||||||
|
# akceptowalne bo tags mają wagę tylko 0.05 w composite scoring resolvera.
|
||||||
|
tags: list[RawTag] = []
|
||||||
|
seen_tag: set[str] = set()
|
||||||
|
for m in _META_TAG_RE.finditer(detail_html):
|
||||||
|
name = html_mod.unescape(m.group(1)).strip()
|
||||||
|
if not name:
|
||||||
|
continue
|
||||||
|
if name == name.lower():
|
||||||
|
continue
|
||||||
|
slug = re.sub(r"[^a-z0-9]+", "-", name.lower()).strip("-")
|
||||||
|
if slug in seen_tag:
|
||||||
|
continue
|
||||||
|
seen_tag.add(slug)
|
||||||
|
tags.append(
|
||||||
|
RawTag(external_id=f"{self.sitetag}:tag:{slug}", name=name, slug=slug)
|
||||||
|
)
|
||||||
|
|
||||||
|
# Phash z thumbnailUrl — BunnyCDN `yesnn.b-cdn.net` hostuje 1.jpg per scene.
|
||||||
|
# Hit-rate vs canonical TPDB/StashDB nieznany do pilot run; graceful: brak
|
||||||
|
# phash → resolver spada do composite scoring (studio + performer + date +
|
||||||
|
# duration + title token-set) — wszystkie dostępne dzięki JSON-LD.
|
||||||
|
fingerprints: list[RawFingerprint] = []
|
||||||
|
if thumbnail_url:
|
||||||
|
ph = compute_thumbnail_phash(thumbnail_url, referer=_BASE + "/")
|
||||||
|
if ph:
|
||||||
|
fingerprints.append(RawFingerprint(kind="phash", value=ph))
|
||||||
|
|
||||||
|
# Playback — page_url do strony sceny. Direct mp4 (`view_video_download.php`)
|
||||||
|
# wymaga PHPSESSID cookie (data-attach-session attribute), więc nie usable
|
||||||
|
# server-side. Extractor `yespornvip` → `_vps_blocked_fallback.extract`:
|
||||||
|
# mobile WebView z phone IP łapie session natively, INJECTED_JS scrape.
|
||||||
|
playback_sources = [
|
||||||
|
RawPlaybackSource(
|
||||||
|
origin=f"tube:{self.sitetag}",
|
||||||
|
page_url=scene_url,
|
||||||
|
duration_sec=duration_sec,
|
||||||
|
thumbnail_url=thumbnail_url,
|
||||||
|
)
|
||||||
|
]
|
||||||
|
|
||||||
|
return RawScene(
|
||||||
|
external_id=f"{self.sitetag}:{video_id or scene_url}",
|
||||||
|
title=title,
|
||||||
|
description=description,
|
||||||
|
release_date=release_date,
|
||||||
|
duration_sec=duration_sec,
|
||||||
|
url=scene_url,
|
||||||
|
studio=studio,
|
||||||
|
performers=performers,
|
||||||
|
tags=tags,
|
||||||
|
fingerprints=fingerprints,
|
||||||
|
playback_sources=playback_sources,
|
||||||
|
)
|
||||||
118
app/connectors/direct_scrapers/youporn_browse.py
Normal file
118
app/connectors/direct_scrapers/youporn_browse.py
Normal file
|
|
@ -0,0 +1,118 @@
|
||||||
|
"""youporn.com — latest BROWSE scraper (JSON-LD + page-parse), obok search scrapera.
|
||||||
|
|
||||||
|
YouPornScraper (search) zostaje w ALL_DIRECT_SCRAPERS; ten browse dokłada świeżość
|
||||||
|
wprost z `/browse/time/?page=<n>` (newest-first, SSR). Detail page ma JSON-LD
|
||||||
|
VideoObject (name/duration/uploadDate/thumbnail) — i TYLKO to bierzemy.
|
||||||
|
|
||||||
|
UWAGA: performerów/tagów z detail-strony NIE wyciągamy. JSON-LD nie ma pola `actor`,
|
||||||
|
a linki `/pornstar/` i `/category/` na stronie są zaśmiecone sidebarem (popularne
|
||||||
|
pornstars/related) bez czystego scene-scoped kontenera — naiwny regex podpinał te
|
||||||
|
same 2 pornstars do KAŻDEJ sceny (mass-misattribution). Browse to tylko sygnał
|
||||||
|
świeżości (próg watchdog 48h); performerów/tagi dorabia canonical-merge + istniejący
|
||||||
|
search scraper (performer-driven). Listing SSR (/watch/<id>/); homepage JS-renderowany.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import re
|
||||||
|
from datetime import date, datetime
|
||||||
|
|
||||||
|
from app.connectors.base import RawPlaybackSource, RawScene
|
||||||
|
from app.connectors.direct_scrapers._browse_base import BaseBrowseScraper, meta_content
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
_BASE = "https://www.youporn.com"
|
||||||
|
_SCENE_URL_RE = re.compile(r'href="(/watch/\d+[^"]*)"', re.IGNORECASE)
|
||||||
|
_JSONLD_RE = re.compile(
|
||||||
|
r'<script[^>]+type=["\']application/ld\+json["\'][^>]*>(.*?)</script>', re.IGNORECASE | re.DOTALL
|
||||||
|
)
|
||||||
|
_ISO_DUR_RE = re.compile(r"PT(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?", re.IGNORECASE)
|
||||||
|
|
||||||
|
|
||||||
|
def _dur_to_sec(value: str | None) -> int | None:
|
||||||
|
if not value:
|
||||||
|
return None
|
||||||
|
m = _ISO_DUR_RE.match(str(value).strip())
|
||||||
|
if not m:
|
||||||
|
return None
|
||||||
|
total = int(m.group(1) or 0) * 3600 + int(m.group(2) or 0) * 60 + int(m.group(3) or 0)
|
||||||
|
return total or None
|
||||||
|
|
||||||
|
|
||||||
|
def _iso_date(value: str | None) -> date | None:
|
||||||
|
if not value:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
return datetime.fromisoformat(str(value).replace("Z", "+00:00")).date()
|
||||||
|
except ValueError:
|
||||||
|
m = re.match(r"(\d{4}-\d{2}-\d{2})", str(value))
|
||||||
|
return date.fromisoformat(m.group(1)) if m else None
|
||||||
|
|
||||||
|
|
||||||
|
def _video_object(html: str) -> dict | None:
|
||||||
|
for m in _JSONLD_RE.finditer(html):
|
||||||
|
raw = m.group(1).strip()
|
||||||
|
if not raw:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
data = json.loads(raw)
|
||||||
|
except (json.JSONDecodeError, ValueError):
|
||||||
|
continue
|
||||||
|
items = data if isinstance(data, list) else (data.get("@graph", [data]) if isinstance(data, dict) else [])
|
||||||
|
for obj in items:
|
||||||
|
if isinstance(obj, dict) and obj.get("@type") == "VideoObject":
|
||||||
|
return obj
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
class YouPornBrowseScraper(BaseBrowseScraper):
|
||||||
|
sitetag = "youporncom"
|
||||||
|
|
||||||
|
def _listing_url(self, page: int) -> str:
|
||||||
|
return f"{_BASE}/browse/time/?page={page}"
|
||||||
|
|
||||||
|
def _extract_scene_urls(self, listing_html: str) -> list[str]:
|
||||||
|
seen: set[str] = set()
|
||||||
|
out: list[str] = []
|
||||||
|
for m in _SCENE_URL_RE.finditer(listing_html):
|
||||||
|
url = f"{_BASE}{m.group(1)}"
|
||||||
|
if url in seen:
|
||||||
|
continue
|
||||||
|
seen.add(url)
|
||||||
|
out.append(url)
|
||||||
|
return out
|
||||||
|
|
||||||
|
def _parse_detail(self, scene_url: str, detail_html: str) -> RawScene | None:
|
||||||
|
video = _video_object(detail_html) or {}
|
||||||
|
title = (video.get("name") or "").strip() or (meta_content(detail_html, property="og:title") or "").strip()
|
||||||
|
if not title:
|
||||||
|
return None
|
||||||
|
|
||||||
|
duration_sec = _dur_to_sec(video.get("duration"))
|
||||||
|
release_date = _iso_date(video.get("uploadDate") or video.get("datePublished"))
|
||||||
|
thumbnail_url = video.get("thumbnailUrl") or meta_content(detail_html, property="og:image")
|
||||||
|
if isinstance(thumbnail_url, list):
|
||||||
|
thumbnail_url = thumbnail_url[0] if thumbnail_url else None
|
||||||
|
|
||||||
|
# performers/tags celowo puste — patrz docstring (sidebar pollution, brak
|
||||||
|
# scene-scoped kontenera). Dorabia canonical-merge + search scraper.
|
||||||
|
return RawScene(
|
||||||
|
external_id=f"{self.sitetag}:{scene_url}",
|
||||||
|
title=title,
|
||||||
|
duration_sec=duration_sec,
|
||||||
|
release_date=release_date,
|
||||||
|
url=scene_url,
|
||||||
|
performers=[],
|
||||||
|
tags=[],
|
||||||
|
playback_sources=[
|
||||||
|
RawPlaybackSource(
|
||||||
|
origin=f"tube:{self.sitetag}",
|
||||||
|
page_url=scene_url,
|
||||||
|
duration_sec=duration_sec,
|
||||||
|
thumbnail_url=thumbnail_url,
|
||||||
|
)
|
||||||
|
],
|
||||||
|
raw={"source": "youporn_browse"},
|
||||||
|
)
|
||||||
|
|
@ -1,119 +0,0 @@
|
||||||
"""ZeroDayXXScraper — direct HTML scrape 0dayxx.com search.
|
|
||||||
|
|
||||||
Search: `https://0dayxx.com/page/<n>/?s=<query>`. Scene URL format:
|
|
||||||
`https://0dayxx.com/0day-porn-video/<slug>/` (lub czasem `/<category>/<slug>/`).
|
|
||||||
"""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import logging
|
|
||||||
import re
|
|
||||||
import urllib.parse
|
|
||||||
from collections.abc import Iterator
|
|
||||||
|
|
||||||
from app.connectors.base import RawPerformer, RawPlaybackSource, RawScene
|
|
||||||
from app.connectors.direct_scrapers.base import BaseDirectTubeScraper
|
|
||||||
from app.extractors import browser_get
|
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
|
|
||||||
_SCENE_URL_RE = re.compile(
|
|
||||||
r'href="(https://0dayxx\.com/(?:0day-porn-video|latest-porn-videos|porn-(?:bf|videos))/([^"/]+))/?"'
|
|
||||||
)
|
|
||||||
_OG_TITLE_RE = re.compile(
|
|
||||||
r'<meta\s+property="og:title"\s+content="([^"]+)"', re.IGNORECASE
|
|
||||||
)
|
|
||||||
_OG_IMAGE_RE = re.compile(
|
|
||||||
r'<meta\s+property="og:image"\s+content="([^"]+)"', re.IGNORECASE
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _fetch_detail(scene_url: str) -> tuple[str | None, str | None]:
|
|
||||||
"""Pobiera 0dayxx detail page i wyciąga (real_title, thumbnail_url).
|
|
||||||
|
|
||||||
0dayxx jest wrapperem (embeduje watchporn.to/inne), więc duration/tagi tu
|
|
||||||
nie są — siedzą na watchporn.to. og:image jednak jest na 0dayxx i daje
|
|
||||||
miniaturkę z poprawnym wymiarem (200x200 — mała, ale lepsza niż żadna).
|
|
||||||
|
|
||||||
Bez tego fetch'u sceny 0dayxx trafiały do dedupu z slug'iem jako title +
|
|
||||||
bez thumbnail_url — czyli z dwoma najsłabszymi sygnałami na raz, co
|
|
||||||
powodowało albo brak match'y albo false-positive merge'y (zgłoszone
|
|
||||||
2026-05-09).
|
|
||||||
"""
|
|
||||||
try:
|
|
||||||
r = browser_get(scene_url, timeout=20)
|
|
||||||
except Exception as e:
|
|
||||||
log.debug("0dayxx detail fetch failed for %s: %s", scene_url, e)
|
|
||||||
return None, None
|
|
||||||
if r.status_code != 200:
|
|
||||||
return None, None
|
|
||||||
title = None
|
|
||||||
thumb = None
|
|
||||||
if (m := _OG_TITLE_RE.search(r.text)):
|
|
||||||
# Strip ` | 0dayxx.com Daily...` suffix (powtórki og:title czasem mają go).
|
|
||||||
title = m.group(1).split("|")[0].strip()
|
|
||||||
if (m := _OG_IMAGE_RE.search(r.text)):
|
|
||||||
thumb = m.group(1).strip()
|
|
||||||
return title, thumb
|
|
||||||
|
|
||||||
|
|
||||||
class ZeroDayXXScraper(BaseDirectTubeScraper):
|
|
||||||
sitetag = "0dayxxcom"
|
|
||||||
|
|
||||||
def search(
|
|
||||||
self,
|
|
||||||
query: str,
|
|
||||||
*,
|
|
||||||
page: int = 1,
|
|
||||||
limit: int | None = None,
|
|
||||||
) -> Iterator[RawScene]:
|
|
||||||
q = urllib.parse.quote_plus(query.strip())
|
|
||||||
url = f"https://0dayxx.com/page/{page}/?s={q}"
|
|
||||||
try:
|
|
||||||
r = browser_get(url, timeout=30)
|
|
||||||
except Exception as e:
|
|
||||||
log.warning("0dayxx search fetch failed: %s", e)
|
|
||||||
return
|
|
||||||
if r.status_code != 200:
|
|
||||||
return
|
|
||||||
|
|
||||||
query_tokens = {tok for tok in query.lower().split() if len(tok) >= 3}
|
|
||||||
|
|
||||||
seen: set[str] = set()
|
|
||||||
yielded = 0
|
|
||||||
for m in _SCENE_URL_RE.finditer(r.text):
|
|
||||||
scene_url = m.group(1) + "/"
|
|
||||||
slug = m.group(2)
|
|
||||||
if scene_url in seen:
|
|
||||||
continue
|
|
||||||
seen.add(scene_url)
|
|
||||||
|
|
||||||
slug_lower = slug.lower()
|
|
||||||
if query_tokens and not any(tok in slug_lower for tok in query_tokens):
|
|
||||||
continue
|
|
||||||
|
|
||||||
real_title, thumb = _fetch_detail(scene_url)
|
|
||||||
title = real_title or slug.replace("-", " ").strip()
|
|
||||||
|
|
||||||
yield RawScene(
|
|
||||||
external_id=f"0dayxxcom:{scene_url}",
|
|
||||||
title=title,
|
|
||||||
url=scene_url,
|
|
||||||
playback_sources=[
|
|
||||||
RawPlaybackSource(
|
|
||||||
origin="tube:0dayxxcom",
|
|
||||||
page_url=scene_url,
|
|
||||||
thumbnail_url=thumb,
|
|
||||||
)
|
|
||||||
],
|
|
||||||
performers=[RawPerformer(name=query.strip())],
|
|
||||||
raw={
|
|
||||||
"source": "direct_scraper:0dayxx",
|
|
||||||
"query": query,
|
|
||||||
"page": page,
|
|
||||||
"url": scene_url,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
yielded += 1
|
|
||||||
if limit is not None and yielded >= limit:
|
|
||||||
return
|
|
||||||
|
|
@ -187,6 +187,18 @@ class DooplayConnector(BaseMovieConnector):
|
||||||
)
|
)
|
||||||
return r.text
|
return r.text
|
||||||
|
|
||||||
|
# Bezpiecznik — dooplay listing potrafi mieć tysiące stron (streamporn.nl ma
|
||||||
|
# >2k filmów). Bez tego ingest wisi godzinami, jest killowany przy restartcie
|
||||||
|
# schedulera, blokując kolejne connectory w queue (bug-report 2026-05-28: "od
|
||||||
|
# wczoraj nie ma nowych filmów" — streamporn wisiał od 5-24, blokował
|
||||||
|
# mangoporn + pandamovies). Listing jest sortowany po dacie uploadu (NIE
|
||||||
|
# release_date filmu — release może być z 2013 a upload z dziś), ale upload
|
||||||
|
# date nie jest w markupie, więc filtrowanie po `since` przez release_date
|
||||||
|
# nie działa. Pragmatyczny cap stron: 3 dla delta (≈150 nowych pozycji/dzień
|
||||||
|
# to znacznie powyżej realnego upload-rate), 50 dla full ingestu (`since=None`).
|
||||||
|
_MAX_PAGES_DELTA = 3
|
||||||
|
_MAX_PAGES_FULL = 50
|
||||||
|
|
||||||
def fetch_movies(
|
def fetch_movies(
|
||||||
self,
|
self,
|
||||||
*,
|
*,
|
||||||
|
|
@ -196,7 +208,8 @@ class DooplayConnector(BaseMovieConnector):
|
||||||
seen = 0
|
seen = 0
|
||||||
page = 1
|
page = 1
|
||||||
seen_urls: set[str] = set()
|
seen_urls: set[str] = set()
|
||||||
while True:
|
max_pages = self._MAX_PAGES_DELTA if since is not None else self._MAX_PAGES_FULL
|
||||||
|
while page <= max_pages:
|
||||||
try:
|
try:
|
||||||
urls = list(self._fetch_listing(page))
|
urls = list(self._fetch_listing(page))
|
||||||
except httpx.HTTPError as e:
|
except httpx.HTTPError as e:
|
||||||
|
|
@ -221,6 +234,10 @@ class DooplayConnector(BaseMovieConnector):
|
||||||
if limit is not None and seen >= limit:
|
if limit is not None and seen >= limit:
|
||||||
return
|
return
|
||||||
page += 1
|
page += 1
|
||||||
|
log.info(
|
||||||
|
"%s: hit max_pages=%d cap (delta=%s), stopping after seen=%d",
|
||||||
|
self.name, max_pages, since is not None, seen,
|
||||||
|
)
|
||||||
|
|
||||||
def _fetch_listing(self, page: int) -> Iterator[str]:
|
def _fetch_listing(self, page: int) -> Iterator[str]:
|
||||||
path = self._listing_path(page)
|
path = self._listing_path(page)
|
||||||
|
|
@ -430,6 +447,14 @@ class StreampornConnector(DooplayConnector):
|
||||||
base_url = "https://streamporn.nl"
|
base_url = "https://streamporn.nl"
|
||||||
|
|
||||||
|
|
||||||
|
class StreampornVipConnector(DooplayConnector):
|
||||||
|
# streamporn.nl padł (stale od 2026-05-23); streamporn.vip to żywy sibling z tym
|
||||||
|
# samym PsyPlay theme i świeższym katalogiem (ocena 2026-07-02). Osobny source_name
|
||||||
|
# → nowe origins `streampornvip:<host>`, nie miesza z martwym .nl.
|
||||||
|
name = "streampornvip"
|
||||||
|
base_url = "https://streamporn.vip"
|
||||||
|
|
||||||
|
|
||||||
class PandamoviesConnector(DooplayConnector):
|
class PandamoviesConnector(DooplayConnector):
|
||||||
name = "pandamovies"
|
name = "pandamovies"
|
||||||
base_url = "https://pandamovies.pw"
|
base_url = "https://pandamovies.pw"
|
||||||
|
|
|
||||||
201
app/connectors/freeomovie.py
Normal file
201
app/connectors/freeomovie.py
Normal file
|
|
@ -0,0 +1,201 @@
|
||||||
|
"""freeomovie.to — movie source (WordPress "bestia" theme, NIE dooplay).
|
||||||
|
|
||||||
|
Dodany 2026-07-02 (ocena). Pełnometrażowe filmy, świeże (auto-poster, dziś), tytuły
|
||||||
|
DVD → title-trigram mirror-attach do canonical (paradisehill/TPDB movies), dokładając
|
||||||
|
playback_sources. Scope: TYLKO `/category/full-movie/` (homepage miesza scen-klipy,
|
||||||
|
które orphanowałyby jako filmy).
|
||||||
|
|
||||||
|
Struktura:
|
||||||
|
- listing `<li class="thumi"><a href title>` → URL filmu + czysty tytuł
|
||||||
|
- detail: `var TABS=[{label,url},...]` (hostery), `"articleSection"` (gatunki + obsada;
|
||||||
|
gatunki PRZED "XXX Movies" → tagi), `"datePublished"` (data POSTA, NIE rok produkcji)
|
||||||
|
|
||||||
|
Świadomie NIE bierzemy:
|
||||||
|
- release_year: datePublished to data uploadu (2026), nie rok filmu → fałszywy rok
|
||||||
|
psułby year-scoring przy matchu do canonical (film z 2010 dostałby 2026). Lepiej None.
|
||||||
|
- performerów/studia z articleSection: format miesza performer/alias/studio bez czystego
|
||||||
|
delimitera → ryzyko junk-performera ("Deeveeous" studio jako performer). Mirror i tak
|
||||||
|
doczepia się do canonical po tytule, a TPDB enrichment dokłada obsadę/studio autorytatywnie.
|
||||||
|
|
||||||
|
Playback: TABS hostery → MoviePlaybackSource per host (voe/luluvid/vidhide resolvują się
|
||||||
|
VPS-side; myvidplay = DoodStream clone, phone-side — wymaga myvidplay.com w DOOD_HOSTS).
|
||||||
|
Pomijamy streamtape (martwy malware) + mxdrop (flaky).
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import html
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import re
|
||||||
|
from collections.abc import Iterator
|
||||||
|
from datetime import datetime
|
||||||
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
|
from app.connectors.base import (
|
||||||
|
BaseMovieConnector,
|
||||||
|
RawMovie,
|
||||||
|
RawPlaybackSource,
|
||||||
|
RawTag,
|
||||||
|
)
|
||||||
|
from app.extractors import browser_get
|
||||||
|
from app.models.source import SourceKind
|
||||||
|
from app.normalize.text import slugify
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
_BASE = "https://www.freeomovie.to"
|
||||||
|
_LIST_ITEM_RE = re.compile(
|
||||||
|
r'<li[^>]*class="thumi"[^>]*>\s*<a[^>]+href="(?P<url>https://www\.freeomovie\.to/[a-z0-9\-]+/)"[^>]*title="(?P<title>[^"]+)"',
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
_TABS_RE = re.compile(r"var\s+TABS\s*=\s*(\[.*?\])\s*;", re.DOTALL)
|
||||||
|
_ARTICLE_SECTION_RE = re.compile(r'"articleSection"\s*:\s*"([^"]+)"')
|
||||||
|
# Poster: <img class="... rmbd ..." src="..."> (fastpic.org itp.). Bez tego film-orphan
|
||||||
|
# (tytuł bez matchu do canonical) miał poster_url=None → pusty kafelek (report 8f8c10c0).
|
||||||
|
_POSTER_RE = re.compile(
|
||||||
|
r'<img[^>]+class="[^"]*\brmbd\b[^"]*"[^>]+src="(https?://[^"]+)"',
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Hostery pomijane (martwe/flaky). player4me = JS-SPA bez znanego resolvera (28 martwych
|
||||||
|
# źródeł freeomovie:video, report 8f8c10c0).
|
||||||
|
_SKIP_HOSTS = ("streamtape", "mxdrop", "mixdrop", "streamsb", "player4me")
|
||||||
|
|
||||||
|
|
||||||
|
def _host_label(embed_url: str) -> str | None:
|
||||||
|
host = (urlparse(embed_url).hostname or "").lower()
|
||||||
|
if not host:
|
||||||
|
return None
|
||||||
|
parts = host.replace("www.", "").split(".")
|
||||||
|
# Rejestrowalna domena (SLD), nie subdomena: video.player4me.xyz → 'player4me' (nie
|
||||||
|
# 'video'), myvidplay.com → 'myvidplay'. Inaczej origin był śmieciowy (freeomovie:video).
|
||||||
|
return parts[-2] if len(parts) >= 2 else parts[0]
|
||||||
|
|
||||||
|
|
||||||
|
class FreeoMovieConnector(BaseMovieConnector):
|
||||||
|
kind = SourceKind.scraper
|
||||||
|
name = "freeomovie"
|
||||||
|
base_url = _BASE
|
||||||
|
|
||||||
|
_MAX_PAGES_DELTA = 3
|
||||||
|
_MAX_PAGES_FULL = 40
|
||||||
|
|
||||||
|
def __init__(self, *, timeout: float = 30.0) -> None:
|
||||||
|
self._timeout = timeout
|
||||||
|
|
||||||
|
def close(self) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def _fetch(self, url: str) -> str:
|
||||||
|
if not url.startswith("http"):
|
||||||
|
url = _BASE + url
|
||||||
|
r = browser_get(
|
||||||
|
url,
|
||||||
|
headers={
|
||||||
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/125.0",
|
||||||
|
"Accept": "text/html,application/xhtml+xml",
|
||||||
|
"Referer": _BASE + "/",
|
||||||
|
},
|
||||||
|
timeout=self._timeout,
|
||||||
|
follow_redirects=True,
|
||||||
|
)
|
||||||
|
if r.status_code >= 400:
|
||||||
|
raise RuntimeError(f"{r.status_code} for {url}")
|
||||||
|
return r.text
|
||||||
|
|
||||||
|
def fetch_movies(
|
||||||
|
self, *, since: datetime | None = None, limit: int | None = None
|
||||||
|
) -> Iterator[RawMovie]:
|
||||||
|
seen = 0
|
||||||
|
seen_urls: set[str] = set()
|
||||||
|
max_pages = self._MAX_PAGES_DELTA if since is not None else self._MAX_PAGES_FULL
|
||||||
|
for page in range(1, max_pages + 1):
|
||||||
|
path = "/category/full-movie/" if page == 1 else f"/category/full-movie/page/{page}/"
|
||||||
|
try:
|
||||||
|
listing = self._fetch(path)
|
||||||
|
except Exception as e:
|
||||||
|
log.warning("freeomovie listing page=%d failed: %s", page, e)
|
||||||
|
return
|
||||||
|
items = [(m.group("url"), html.unescape(m.group("title")).strip())
|
||||||
|
for m in _LIST_ITEM_RE.finditer(listing)]
|
||||||
|
if not items:
|
||||||
|
log.info("freeomovie: empty page=%d, stop", page)
|
||||||
|
return
|
||||||
|
for url, title in items:
|
||||||
|
if url in seen_urls:
|
||||||
|
continue
|
||||||
|
seen_urls.add(url)
|
||||||
|
try:
|
||||||
|
movie = self._parse_detail(url, title)
|
||||||
|
except Exception as e:
|
||||||
|
log.warning("freeomovie detail %s failed: %s", url, e)
|
||||||
|
continue
|
||||||
|
if movie is None:
|
||||||
|
continue
|
||||||
|
yield movie
|
||||||
|
seen += 1
|
||||||
|
if limit is not None and seen >= limit:
|
||||||
|
return
|
||||||
|
|
||||||
|
def _parse_detail(self, url: str, title: str) -> RawMovie | None:
|
||||||
|
detail = self._fetch(url)
|
||||||
|
if not title:
|
||||||
|
return None
|
||||||
|
|
||||||
|
# Tagi: gatunki z articleSection PRZED "XXX Movies" (reszta = obsada/studio, skip).
|
||||||
|
tags: list[RawTag] = []
|
||||||
|
seen_tag: set[str] = set()
|
||||||
|
sm = _ARTICLE_SECTION_RE.search(detail)
|
||||||
|
if sm:
|
||||||
|
for part in sm.group(1).split(","):
|
||||||
|
name = part.strip()
|
||||||
|
if not name or name.lower() == "xxx movies":
|
||||||
|
if name.lower() == "xxx movies":
|
||||||
|
break # dalej idzie obsada/studio — nie tagi
|
||||||
|
continue
|
||||||
|
sl = slugify(name)
|
||||||
|
if not sl or sl in seen_tag:
|
||||||
|
continue
|
||||||
|
seen_tag.add(sl)
|
||||||
|
tags.append(RawTag(external_id=f"{self.name}:tag:{sl}", name=name, slug=sl))
|
||||||
|
|
||||||
|
# Playback: TABS array hosterów.
|
||||||
|
playback: list[RawPlaybackSource] = []
|
||||||
|
seen_host: set[str] = set()
|
||||||
|
tm = _TABS_RE.search(detail)
|
||||||
|
if tm:
|
||||||
|
try:
|
||||||
|
arr = json.loads(tm.group(1))
|
||||||
|
except (json.JSONDecodeError, ValueError):
|
||||||
|
arr = []
|
||||||
|
for entry in arr:
|
||||||
|
embed = (entry.get("url") or "").strip() if isinstance(entry, dict) else ""
|
||||||
|
if not embed.startswith("http"):
|
||||||
|
continue
|
||||||
|
host = _host_label(embed)
|
||||||
|
if not host or host in _SKIP_HOSTS or host in seen_host:
|
||||||
|
continue
|
||||||
|
seen_host.add(host)
|
||||||
|
playback.append(
|
||||||
|
RawPlaybackSource(
|
||||||
|
origin=f"{self.name}:{host}",
|
||||||
|
page_url=embed,
|
||||||
|
embed_url=embed,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if not playback:
|
||||||
|
return None # bez playbacku film jest bezużyteczny (nie tworzymy orphana)
|
||||||
|
|
||||||
|
pm = _POSTER_RE.search(detail)
|
||||||
|
poster_url = html.unescape(pm.group(1)) if pm else None
|
||||||
|
|
||||||
|
slug = urlparse(url).path.strip("/").split("/")[-1]
|
||||||
|
return RawMovie(
|
||||||
|
external_id=slug,
|
||||||
|
title=title,
|
||||||
|
url=url,
|
||||||
|
poster_url=poster_url,
|
||||||
|
tags=tags,
|
||||||
|
playback_sources=playback,
|
||||||
|
raw={"source": "freeomovie", "url": url},
|
||||||
|
)
|
||||||
|
|
@ -22,7 +22,7 @@ from __future__ import annotations
|
||||||
import logging
|
import logging
|
||||||
import re
|
import re
|
||||||
from collections.abc import Iterator
|
from collections.abc import Iterator
|
||||||
from datetime import UTC, date, datetime
|
from datetime import UTC, date, datetime, timedelta
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
|
|
@ -72,6 +72,46 @@ _CHAPTER_RE = re.compile(
|
||||||
r'<a\s+href="#"\s+class="js-list-item"\s+data-index="(\d+)">([^<]+)</a>',
|
r'<a\s+href="#"\s+class="js-list-item"\s+data-index="(\d+)">([^<]+)</a>',
|
||||||
re.IGNORECASE,
|
re.IGNORECASE,
|
||||||
)
|
)
|
||||||
|
# videoList JS array w detail page — może mieć multiple parts (Video.js playlist):
|
||||||
|
# var videoList = [{"sources":[{"src":"...part1.mp4","type":"video/mp4"}]}, ...]
|
||||||
|
# Bez parsowania tego mobile WebView gra tylko pierwszy part, kolejne pomija.
|
||||||
|
# Bug-reports `c5693926`/`418270e4` 2026-05-21 ("ładuje tylko 1 z 4 części").
|
||||||
|
_VIDEO_LIST_RE = re.compile(r"var\s+videoList\s*=\s*(\[.*?\])\s*;", re.IGNORECASE | re.DOTALL)
|
||||||
|
_VIDEO_SRC_RE = re.compile(r'"src"\s*:\s*"([^"]+\.mp4[^"]*)"', re.IGNORECASE)
|
||||||
|
|
||||||
|
|
||||||
|
def extract_video_parts(html: str) -> list[tuple[str, str]]:
|
||||||
|
"""Wyciąga listę MP4 parts z paradisehill detail HTML.
|
||||||
|
|
||||||
|
Returns: [(mp4_url, label), ...] np. `[(.../part1.mp4, "Part 1"), ...]`.
|
||||||
|
Pusta lista gdy `videoList` nieobecny lub bez sources (login-only filmy).
|
||||||
|
"""
|
||||||
|
m = _VIDEO_LIST_RE.search(html)
|
||||||
|
if not m:
|
||||||
|
return []
|
||||||
|
parts: list[tuple[str, str]] = []
|
||||||
|
for i, src_m in enumerate(_VIDEO_SRC_RE.finditer(m.group(1)), start=1):
|
||||||
|
url = src_m.group(1).replace("\\/", "/")
|
||||||
|
parts.append((url, f"Part {i}"))
|
||||||
|
return parts
|
||||||
|
|
||||||
|
|
||||||
|
def fetch_and_extract_parts(page_url: str, *, timeout: float = 20.0) -> list[tuple[str, str]]:
|
||||||
|
"""Resolve-time helper: pobierz page, wyciągnij videoList parts.
|
||||||
|
Używane przez `app.api.playback.resolve_movie_playback` dla origin='paradisehill'.
|
||||||
|
"""
|
||||||
|
with httpx.Client(
|
||||||
|
timeout=timeout,
|
||||||
|
follow_redirects=True,
|
||||||
|
headers={
|
||||||
|
"User-Agent": USER_AGENT,
|
||||||
|
"Cookie": "is18=1",
|
||||||
|
"Accept-Language": "en-US,en;q=0.9",
|
||||||
|
},
|
||||||
|
) as client:
|
||||||
|
r = client.get(page_url)
|
||||||
|
r.raise_for_status()
|
||||||
|
return extract_video_parts(r.text)
|
||||||
# Listing page item:
|
# Listing page item:
|
||||||
_LIST_ITEM_RE = re.compile(
|
_LIST_ITEM_RE = re.compile(
|
||||||
r'<div\s+class="item\s+list-film-item"[^>]*>\s*'
|
r'<div\s+class="item\s+list-film-item"[^>]*>\s*'
|
||||||
|
|
@ -139,15 +179,18 @@ class ParadisehillConnector(BaseMovieConnector):
|
||||||
if movie is None:
|
if movie is None:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# `since` filter — datePublished poniżej threshold = stop crawla,
|
# `since` filter — datePublished (= data uploadu na paradisehill) poniżej
|
||||||
# bo listing jest chronologiczny. since z `_last_successful_finished_at`
|
# progu = stop crawla (listing chronologiczny).
|
||||||
# jest TZ-aware (UTC); combine() daje naive — przywróć UTC tzinfo żeby
|
#
|
||||||
# porównanie nie crashowało.
|
# UWAGA: release_date to DATA (bez godziny). Wcześniej combine()→00:00
|
||||||
|
# porównywane z TIMESTAMPEM `since` ucinało crawl na PIERWSZYM filmie z dnia
|
||||||
|
# == since (midnight < since o dowolnej porze dnia) → uploady tego samego dnia
|
||||||
|
# systematycznie ginęły, bo movie-ingest jest dzienny (seen=0-2/run mimo
|
||||||
|
# świeżych filmów na froncie strony — bug-report 2026-06-01 "Movies stoją").
|
||||||
|
# Fix: porównuj po DACIE z 1-dniowym grace; ponowny fetch świeżych jest tani
|
||||||
|
# (external_records upsert pomija niezmieniony hash).
|
||||||
if since is not None and movie.release_date is not None:
|
if since is not None and movie.release_date is not None:
|
||||||
rd_dt = datetime.combine(
|
if movie.release_date < (since - timedelta(days=1)).date():
|
||||||
movie.release_date, datetime.min.time(), tzinfo=UTC
|
|
||||||
)
|
|
||||||
if rd_dt < since:
|
|
||||||
log.info(
|
log.info(
|
||||||
"paradisehill: hit since boundary at %s (%s), stop",
|
"paradisehill: hit since boundary at %s (%s), stop",
|
||||||
mid, movie.release_date,
|
mid, movie.release_date,
|
||||||
|
|
@ -230,15 +273,32 @@ def _parse_detail(hex_id: str, html: str) -> RawMovie | None:
|
||||||
|
|
||||||
# Genre — pierwszy itemprop="genre" w samym block-inside (nie w recommendations).
|
# Genre — pierwszy itemprop="genre" w samym block-inside (nie w recommendations).
|
||||||
# Recommended films też mają itemprop="genre" więc match limity do block-inside.
|
# Recommended films też mają itemprop="genre" więc match limity do block-inside.
|
||||||
|
# Wcześniejszy regex wymagał `</div></div><div class="similar"` — ale paradisehill
|
||||||
|
# czasami ma `</div></noindex>...<div class="similar"` (banner skin z 2026-05-19),
|
||||||
|
# przez co block_match failował → fallback do html[:8000] → 0 tagów. Bug-report
|
||||||
|
# `3c999b27` 2026-05-21 ("Brak kategorii"). Robust: szukaj similar jako stop boundary,
|
||||||
|
# bez wymagania zamknięcia konkretnymi `</div>`.
|
||||||
tags: list[RawTag] = []
|
tags: list[RawTag] = []
|
||||||
block_match = re.search(
|
block_start = re.search(
|
||||||
r'<div\s+class="block-inside"[^>]*itemtype="http://schema\.org/Movie"[^>]*>'
|
r'<div\s+class="block-inside"[^>]*itemtype="http://schema\.org/Movie"[^>]*>',
|
||||||
r'(.*?)</div>\s*</div>\s*<div\s+class="similar',
|
|
||||||
html,
|
html,
|
||||||
re.DOTALL,
|
|
||||||
)
|
)
|
||||||
block = block_match.group(1) if block_match else html[:8000]
|
if block_start:
|
||||||
for m_genre in re.finditer(r'itemprop="genre"[^>]*>([^<]+)</', block, re.IGNORECASE):
|
rest = html[block_start.end():]
|
||||||
|
# Stop boundary: pierwszy <div class="similar...">. Wszystko przedtem to
|
||||||
|
# właściwa zawartość filmu (genre/cast/itd.); reszta to recommendations
|
||||||
|
# i komentarze ktore mają własne itemprop="genre".
|
||||||
|
stop = re.search(r'<div\s+class="similar', rest)
|
||||||
|
block = rest[: stop.start()] if stop else rest[:12000]
|
||||||
|
else:
|
||||||
|
block = html[:8000]
|
||||||
|
# Paradisehill miesza dwa szablony per-page:
|
||||||
|
# v1: `itemprop="genre">Female Domination</span>`
|
||||||
|
# v2: `itemprop="genre"><a href="/category/...">All Sex</a></span>` (od 2026-05)
|
||||||
|
# Optional `<a>` wrapper między `itemprop` a tekstem — bez tego v2 dawał empty.
|
||||||
|
for m_genre in re.finditer(
|
||||||
|
r'itemprop="genre"[^>]*>\s*(?:<a[^>]*>)?\s*([^<]+)', block, re.IGNORECASE,
|
||||||
|
):
|
||||||
name = _decode_html(m_genre.group(1).strip())
|
name = _decode_html(m_genre.group(1).strip())
|
||||||
if name and len(tags) < 10:
|
if name and len(tags) < 10:
|
||||||
tags.append(RawTag(name=name, slug=_slugify(name)))
|
tags.append(RawTag(name=name, slug=_slugify(name)))
|
||||||
|
|
|
||||||
|
|
@ -49,6 +49,7 @@ def _is_retryable_http_error(exc: BaseException) -> bool:
|
||||||
from app.config import get_settings
|
from app.config import get_settings
|
||||||
from app.connectors.base import (
|
from app.connectors.base import (
|
||||||
BaseConnector,
|
BaseConnector,
|
||||||
|
RawMovie,
|
||||||
RawPerformer,
|
RawPerformer,
|
||||||
RawScene,
|
RawScene,
|
||||||
RawStudio,
|
RawStudio,
|
||||||
|
|
@ -195,6 +196,37 @@ class TPDBConnector(BaseConnector):
|
||||||
first = data[0]
|
first = data[0]
|
||||||
return str(first.get("id")) if first.get("id") else None
|
return str(first.get("id")) if first.get("id") else None
|
||||||
|
|
||||||
|
def search_movies(self, query: str, *, per_page: int = 10) -> list[dict[str, Any]]:
|
||||||
|
"""GET /movies?q=<query> → surowe payloady filmów (list endpoint jest już
|
||||||
|
hydrated: tags + performers + site embedded, bez detail-fetcha).
|
||||||
|
|
||||||
|
Używane do enrichmentu movies: wyszukujemy kandydatów po tytule, scorujemy
|
||||||
|
i mapujemy najlepszy przez `_parse_movie`. Pusty list na błąd/brak."""
|
||||||
|
if not query.strip():
|
||||||
|
return []
|
||||||
|
with self._client() as client:
|
||||||
|
try:
|
||||||
|
payload = self._get(client, "/movies", {"q": query, "per_page": per_page})
|
||||||
|
except httpx.HTTPStatusError as e:
|
||||||
|
log.warning("tpdb /movies q=%s failed: %s", query, e)
|
||||||
|
return []
|
||||||
|
return payload.get("data") or []
|
||||||
|
|
||||||
|
def fetch_movie(self, movie_uuid: str) -> dict[str, Any] | None:
|
||||||
|
"""GET /movies/<uuid> → pojedynczy film (detail). None gdy 404/błąd.
|
||||||
|
|
||||||
|
NB: list endpoint jest już hydrated, więc detail rzadko potrzebny — trzymamy
|
||||||
|
dla spójności / gdyby TPDB kiedyś przeniósł część pól tylko do detalu."""
|
||||||
|
with self._client() as client:
|
||||||
|
try:
|
||||||
|
payload = self._get(client, f"/movies/{movie_uuid}", {})
|
||||||
|
except httpx.HTTPStatusError as e:
|
||||||
|
if e.response.status_code == 404:
|
||||||
|
return None
|
||||||
|
log.warning("tpdb /movies/%s failed: %s", movie_uuid, e)
|
||||||
|
return None
|
||||||
|
return payload.get("data")
|
||||||
|
|
||||||
def _paginate_scenes(
|
def _paginate_scenes(
|
||||||
self,
|
self,
|
||||||
params: dict[str, Any],
|
params: dict[str, Any],
|
||||||
|
|
@ -327,3 +359,60 @@ def _parse_scene(raw: dict[str, Any]) -> RawScene | None:
|
||||||
fingerprints=[], # TPDB nie publikuje pHashy w głównym endpoint
|
fingerprints=[], # TPDB nie publikuje pHashy w głównym endpoint
|
||||||
raw=raw,
|
raw=raw,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_movie(raw: dict[str, Any]) -> RawMovie | None:
|
||||||
|
"""TPDB movie payload → RawMovie. Reużywa _parse_studio/_parse_performer/_parse_tag
|
||||||
|
(movies mają identyczny kształt site/performers/tags co sceny)."""
|
||||||
|
external_id = raw.get("id")
|
||||||
|
title = raw.get("title")
|
||||||
|
if not external_id or not title:
|
||||||
|
return None
|
||||||
|
|
||||||
|
performers: list[RawPerformer] = []
|
||||||
|
for p in raw.get("performers") or []:
|
||||||
|
parsed = _parse_performer(p)
|
||||||
|
if parsed is not None:
|
||||||
|
performers.append(parsed)
|
||||||
|
tags: list[RawTag] = []
|
||||||
|
for t in raw.get("tags") or []:
|
||||||
|
parsed_t = _parse_tag(t)
|
||||||
|
if parsed_t is not None:
|
||||||
|
tags.append(parsed_t)
|
||||||
|
|
||||||
|
directors = raw.get("directors") or []
|
||||||
|
director = ", ".join(d.get("name") for d in directors if isinstance(d, dict) and d.get("name")) or None
|
||||||
|
|
||||||
|
d = _parse_date(raw.get("date"))
|
||||||
|
# duration bywa 1s-placeholderem na movies — poniżej 60s traktujemy jak brak.
|
||||||
|
dur = raw.get("duration")
|
||||||
|
duration_sec = int(dur) if dur and int(dur) > 60 else None
|
||||||
|
|
||||||
|
def _img(field: str) -> str | None:
|
||||||
|
v = raw.get(field)
|
||||||
|
if isinstance(v, str) and v:
|
||||||
|
return v
|
||||||
|
if isinstance(v, dict):
|
||||||
|
return v.get("large") or v.get("full") or v.get("medium") or None
|
||||||
|
return None
|
||||||
|
|
||||||
|
poster = _img("poster") or _img("posters")
|
||||||
|
backdrop = _img("background") or _img("backdrop")
|
||||||
|
|
||||||
|
return RawMovie(
|
||||||
|
external_id=str(external_id),
|
||||||
|
title=title,
|
||||||
|
description=raw.get("description"),
|
||||||
|
release_year=d.year if d else None,
|
||||||
|
release_date=d,
|
||||||
|
duration_sec=duration_sec,
|
||||||
|
director=director,
|
||||||
|
rating=float(raw["rating"]) if raw.get("rating") not in (None, "") else None,
|
||||||
|
poster_url=poster,
|
||||||
|
backdrop_url=backdrop,
|
||||||
|
url=raw.get("url"),
|
||||||
|
studio=_parse_studio(raw.get("site")),
|
||||||
|
performers=performers,
|
||||||
|
tags=tags,
|
||||||
|
raw=raw,
|
||||||
|
)
|
||||||
|
|
|
||||||
1
app/enrich/__init__.py
Normal file
1
app/enrich/__init__.py
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
"""Post-ingest enrichment of canonical entities from authoritative metadata sources."""
|
||||||
355
app/enrich/tpdb_movies.py
Normal file
355
app/enrich/tpdb_movies.py
Normal file
|
|
@ -0,0 +1,355 @@
|
||||||
|
"""TPDB movie enrichment + dedup.
|
||||||
|
|
||||||
|
TPDB `/movies` jest naszym kanonicznym źródłem metadanych filmów: obsada, kategorie
|
||||||
|
(tagi), studio, reżyser, rok — plus stabilny UUID do dedupu. paradisehill (primary
|
||||||
|
movie source) prawie nie ma obsady, więc TPDB wypełnia największą lukę.
|
||||||
|
|
||||||
|
Zasada: TPDB TYLKO wzbogaca ISTNIEJĄCE filmy (z paradisehill/dooplay) i dedupuje,
|
||||||
|
NIGDY nie tworzy nowych — TPDB nie ma playbacku, więc nowy film byłby niegrywalny.
|
||||||
|
|
||||||
|
Flow per film:
|
||||||
|
1. skip, jeśli film ma już TPDB movie ref (movie_external_refs jest tylko-movie,
|
||||||
|
więc dowolny ref z tpdb source = już wzbogacony),
|
||||||
|
2. search TPDB /movies?q=<tytuł>, wybierz najlepszego kandydata (token-set na tytule
|
||||||
|
+ guard roku ±2, próg `min_title`),
|
||||||
|
3. DEDUP: jeśli ten TPDB UUID jest już przypięty do INNEGO naszego filmu → to ten
|
||||||
|
sam film (mirror) → merge_movies (keep = starszy created_at),
|
||||||
|
4. wzbogać ocalały film: obsada (resolve_performer → MoviePerformer), tagi
|
||||||
|
(resolve_tag → MovieTag source=tpdb), studio (fill studio_id), reżyser/rok/
|
||||||
|
poster/rating (fill-only), przypnij MovieExternalRef(tpdb, uuid).
|
||||||
|
|
||||||
|
Match jest zachowawczy (próg 0.90 + guard roku), bo TPDB search zwraca dużo (np.
|
||||||
|
"Pirates" → gay Knightbreeders przed feature) — bez scoringu wzięlibyśmy zły film.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import uuid
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
|
||||||
|
from rapidfuzz import fuzz
|
||||||
|
from sqlalchemy import func, select
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app.connectors.tpdb import TPDBConnector, _parse_movie
|
||||||
|
from app.models.movie import Movie, MovieExternalRef, MoviePerformer
|
||||||
|
from app.models.movie_playback_source import MoviePlaybackSource
|
||||||
|
from app.models.performer import Performer
|
||||||
|
from app.models.studio import Studio
|
||||||
|
from app.normalize.movies import normalize_movie
|
||||||
|
from app.normalize.text import normalize
|
||||||
|
from app.resolve.movie_merge import merge_movies
|
||||||
|
from app.resolve.movie_resolver import _sync_performers, _sync_tags
|
||||||
|
from app.resolve.performer_resolver import resolve_performer
|
||||||
|
from app.resolve.studio_resolver import resolve_studio
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
_DASH = str.maketrans({"–": "-", "—": "-", "‑": "-"})
|
||||||
|
|
||||||
|
|
||||||
|
def _cand_year(raw: dict) -> int | None:
|
||||||
|
d = raw.get("date")
|
||||||
|
if d and len(str(d)) >= 4:
|
||||||
|
try:
|
||||||
|
return int(str(d)[:4])
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _movie_studio_name(session: Session, movie: Movie) -> str | None:
|
||||||
|
if not movie.studio_id:
|
||||||
|
return None
|
||||||
|
st = session.get(Studio, movie.studio_id)
|
||||||
|
return st.name if st is not None else None
|
||||||
|
|
||||||
|
|
||||||
|
def _movie_performer_names(session: Session, movie: Movie) -> set[str]:
|
||||||
|
"""Znormalizowane nazwy performerów już przypiętych do filmu (do rozróżniania
|
||||||
|
kandydatów TPDB o tym samym tytule)."""
|
||||||
|
rows = session.execute(
|
||||||
|
select(Performer.name_normalized)
|
||||||
|
.join(MoviePerformer, MoviePerformer.performer_id == Performer.id)
|
||||||
|
.where(MoviePerformer.movie_id == movie.id)
|
||||||
|
).all()
|
||||||
|
return {r[0] for r in rows if r[0]}
|
||||||
|
|
||||||
|
|
||||||
|
def _cand_studio_name(raw: dict) -> str:
|
||||||
|
site = raw.get("site") or {}
|
||||||
|
name = site.get("name") if isinstance(site, dict) else site
|
||||||
|
return normalize(name or "")
|
||||||
|
|
||||||
|
|
||||||
|
def _cand_performer_names(raw: dict) -> set[str]:
|
||||||
|
return {
|
||||||
|
normalize(p.get("name"))
|
||||||
|
for p in (raw.get("performers") or [])
|
||||||
|
if isinstance(p, dict) and p.get("name")
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _best_match(
|
||||||
|
session: Session, connector: TPDBConnector, movie: Movie, *, min_title: float
|
||||||
|
) -> dict | None:
|
||||||
|
"""Najlepszy TPDB movie payload dla naszego filmu, albo None.
|
||||||
|
|
||||||
|
Tytuł to gate (`token_sort_ratio` >= min_title): odporny na inwersję i kolejność,
|
||||||
|
penalizuje różnicę długości (krótki generyczny tytuł nie łapie dłuższego nadzbioru).
|
||||||
|
ALE generyczne tytuły ("Monster Tits", "Pirates") mają w TPDB WIELE różnych filmów
|
||||||
|
o tym samym tytule (różne studia/obsady) i identycznym score 1.0. Sam tytuł ich NIE
|
||||||
|
rozróżni, a `per_page=10` często w ogóle nie zwracał właściwego (poprawny film bywa
|
||||||
|
poza top-10). Dlatego:
|
||||||
|
- `per_page=40` (właściwy film generycznego tytułu bywa dalej w wynikach),
|
||||||
|
- wśród kandydatów przechodzących title-gate wybieramy po KOMPOZYCIE:
|
||||||
|
tytuł + zbieżność studia + pokrycie obsady (nasze studio/obsada z primary
|
||||||
|
źródła jak paradisehill rozróżniają który to film),
|
||||||
|
- GUARD anty-misattribution: jeśli mamy sygnał (studio na naszym filmie) i jest
|
||||||
|
>1 kandydat o ~identycznym tytule, a zwycięzca NIE dzieli studia, to no_match
|
||||||
|
(lepiej nie wzbogacić niż przypiąć zły film o tej samej nazwie).
|
||||||
|
Precyzja > recall. Guard roku ±2 (inny rok = inna edycja)."""
|
||||||
|
query = (movie.title or "").translate(_DASH).strip()[:60]
|
||||||
|
if not query:
|
||||||
|
return None
|
||||||
|
my_norm = normalize(movie.title)
|
||||||
|
my_studio = _movie_studio_name(session, movie)
|
||||||
|
my_studio_norm = normalize(my_studio) if my_studio else ""
|
||||||
|
my_perfs = _movie_performer_names(session, movie)
|
||||||
|
|
||||||
|
best: dict | None = None
|
||||||
|
best_key: tuple[float, float, float, float] | None = None
|
||||||
|
best_studio_sim = 0.0
|
||||||
|
best_perf_overlap = 0.0
|
||||||
|
n_exact_title = 0
|
||||||
|
for raw in connector.search_movies(query, per_page=40):
|
||||||
|
cand_title = raw.get("title")
|
||||||
|
if not cand_title:
|
||||||
|
continue
|
||||||
|
tscore = fuzz.token_sort_ratio(my_norm, normalize(cand_title)) / 100.0
|
||||||
|
if tscore < min_title:
|
||||||
|
continue
|
||||||
|
cy = _cand_year(raw)
|
||||||
|
if movie.release_year and cy and abs(movie.release_year - cy) > 2:
|
||||||
|
continue # guard: inny rok → prawdopodobnie inny film (Taxi 2 ≠ Taxi Violeur 2)
|
||||||
|
if tscore >= 0.97:
|
||||||
|
n_exact_title += 1
|
||||||
|
cand_studio = _cand_studio_name(raw)
|
||||||
|
studio_sim = (
|
||||||
|
fuzz.token_set_ratio(my_studio_norm, cand_studio) / 100.0
|
||||||
|
if my_studio_norm and cand_studio
|
||||||
|
else 0.0
|
||||||
|
)
|
||||||
|
cand_perfs = _cand_performer_names(raw)
|
||||||
|
perf_overlap = (len(my_perfs & cand_perfs) / len(my_perfs)) if my_perfs else 0.0
|
||||||
|
# Kompozyt: tytuł jako baza, studio i obsada jako rozróżniacze (waga 0.6 każdy).
|
||||||
|
composite = tscore + 0.6 * studio_sim + 0.6 * perf_overlap
|
||||||
|
key = (composite, tscore, studio_sim, perf_overlap)
|
||||||
|
if best_key is None or key > best_key:
|
||||||
|
best_key = key
|
||||||
|
best = raw
|
||||||
|
best_studio_sim = studio_sim
|
||||||
|
best_perf_overlap = perf_overlap
|
||||||
|
|
||||||
|
if best is None:
|
||||||
|
return None
|
||||||
|
# Anty-misattribution: mamy czym rozróżnić (studio lub obsada na naszym filmie), jest
|
||||||
|
# >1 kandydat o ~identycznym tytule, a zwycięzca nie dzieli NIC (ani studia, ani
|
||||||
|
# obsady) → to prawie na pewno inny film o tej samej nazwie. Nie wzbogacaj.
|
||||||
|
my_has_signal = bool(my_studio_norm) or bool(my_perfs)
|
||||||
|
if (
|
||||||
|
my_has_signal
|
||||||
|
and n_exact_title >= 2
|
||||||
|
and best_studio_sim < 0.5
|
||||||
|
and best_perf_overlap < 0.5
|
||||||
|
):
|
||||||
|
return None
|
||||||
|
return best
|
||||||
|
|
||||||
|
|
||||||
|
def _fill_scalar_fields(movie: Movie, norm) -> None:
|
||||||
|
"""Fill-only — nie nadpisujemy pól ustawionych przez primary (paradisehill/mirror)."""
|
||||||
|
if norm.director and not movie.director:
|
||||||
|
movie.director = norm.director
|
||||||
|
if norm.release_year and not movie.release_year:
|
||||||
|
movie.release_year = norm.release_year
|
||||||
|
if norm.release_date and not movie.release_date:
|
||||||
|
movie.release_date = norm.release_date
|
||||||
|
if norm.duration_sec and not movie.duration_sec:
|
||||||
|
movie.duration_sec = norm.duration_sec
|
||||||
|
if norm.description and not movie.description:
|
||||||
|
movie.description = norm.description
|
||||||
|
if norm.poster_url and not movie.poster_url:
|
||||||
|
movie.poster_url = norm.poster_url
|
||||||
|
if norm.backdrop_url and not movie.backdrop_url:
|
||||||
|
movie.backdrop_url = norm.backdrop_url
|
||||||
|
if norm.rating is not None and movie.rating is None:
|
||||||
|
movie.rating = norm.rating
|
||||||
|
|
||||||
|
|
||||||
|
def enrich_movie(
|
||||||
|
session: Session,
|
||||||
|
movie: Movie,
|
||||||
|
*,
|
||||||
|
connector: TPDBConnector,
|
||||||
|
source_id: uuid.UUID,
|
||||||
|
min_title: float = 0.90,
|
||||||
|
) -> str:
|
||||||
|
"""Zwraca: 'skip' | 'no_match' | 'enriched' | 'merged'."""
|
||||||
|
already = session.execute(
|
||||||
|
select(MovieExternalRef.external_id).where(
|
||||||
|
MovieExternalRef.source_id == source_id,
|
||||||
|
MovieExternalRef.movie_id == movie.id,
|
||||||
|
)
|
||||||
|
).first()
|
||||||
|
if already is not None:
|
||||||
|
return "skip"
|
||||||
|
|
||||||
|
raw = _best_match(session, connector, movie, min_title=min_title)
|
||||||
|
if raw is None:
|
||||||
|
return "no_match"
|
||||||
|
ext_id = str(raw["id"])
|
||||||
|
|
||||||
|
outcome = "enriched"
|
||||||
|
# DEDUP: ten TPDB UUID już przypięty do innego naszego filmu → mirror tego samego.
|
||||||
|
other = session.execute(
|
||||||
|
select(MovieExternalRef).where(
|
||||||
|
MovieExternalRef.source_id == source_id,
|
||||||
|
MovieExternalRef.external_id == ext_id,
|
||||||
|
)
|
||||||
|
).scalar_one_or_none()
|
||||||
|
if other is not None and other.movie_id != movie.id:
|
||||||
|
m2 = session.get(Movie, other.movie_id)
|
||||||
|
if m2 is not None:
|
||||||
|
now = datetime.now(UTC)
|
||||||
|
keep, drop = (
|
||||||
|
(movie, m2)
|
||||||
|
if (movie.created_at or now) <= (m2.created_at or now)
|
||||||
|
else (m2, movie)
|
||||||
|
)
|
||||||
|
movie = merge_movies(
|
||||||
|
session, keep_id=keep.id, drop_id=drop.id, resolved_by="tpdb:dedup"
|
||||||
|
)
|
||||||
|
outcome = "merged"
|
||||||
|
|
||||||
|
rm = _parse_movie(raw)
|
||||||
|
if rm is None:
|
||||||
|
return "no_match"
|
||||||
|
norm = normalize_movie(rm)
|
||||||
|
|
||||||
|
if norm.studio is not None:
|
||||||
|
studio = resolve_studio(session, norm=norm.studio, source_id=source_id)
|
||||||
|
if studio is not None and not movie.studio_id:
|
||||||
|
movie.studio_id = studio.id
|
||||||
|
|
||||||
|
# Dedup wchodzących performerów po external_id (TPDB potrafi wylistować tego samego
|
||||||
|
# kanonicznego performera 2×: pod aliasem i kanonicznie) — bez tego resolve_performer
|
||||||
|
# próbuje wstawić performer_external_refs 2× → UniqueViolation (jak w scene_resolver).
|
||||||
|
resolved: list[tuple[uuid.UUID, str | None]] = []
|
||||||
|
seen_perf_keys: set[str] = set()
|
||||||
|
for p_norm in norm.performers:
|
||||||
|
key = p_norm.external_id or normalize(p_norm.name)
|
||||||
|
if key in seen_perf_keys:
|
||||||
|
continue
|
||||||
|
seen_perf_keys.add(key)
|
||||||
|
performer = resolve_performer(session, norm=p_norm, source_id=source_id)
|
||||||
|
resolved.append((performer.id, p_norm.as_alias_in_scene))
|
||||||
|
_sync_performers(session, movie_id=movie.id, resolved=resolved)
|
||||||
|
_sync_tags(session, movie_id=movie.id, norm=norm, source_id=source_id)
|
||||||
|
_fill_scalar_fields(movie, norm)
|
||||||
|
|
||||||
|
ref = session.execute(
|
||||||
|
select(MovieExternalRef).where(
|
||||||
|
MovieExternalRef.source_id == source_id,
|
||||||
|
MovieExternalRef.external_id == ext_id,
|
||||||
|
)
|
||||||
|
).scalar_one_or_none()
|
||||||
|
if ref is None:
|
||||||
|
session.add(
|
||||||
|
MovieExternalRef(
|
||||||
|
source_id=source_id,
|
||||||
|
external_id=ext_id,
|
||||||
|
movie_id=movie.id,
|
||||||
|
confidence=1.0,
|
||||||
|
url=rm.url,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
elif ref.movie_id != movie.id:
|
||||||
|
ref.movie_id = movie.id
|
||||||
|
|
||||||
|
return outcome
|
||||||
|
|
||||||
|
|
||||||
|
def _candidate_movies(session: Session, *, source_id: uuid.UUID, limit: int) -> list[Movie]:
|
||||||
|
"""Filmy jeszcze nie wzbogacone TPDB, priorytet: mają żywy playback (są grywalne)
|
||||||
|
i brak obsady LUB brak studia (największa luka). Reszta później."""
|
||||||
|
has_tpdb = (
|
||||||
|
select(MovieExternalRef.movie_id)
|
||||||
|
.where(MovieExternalRef.source_id == source_id)
|
||||||
|
.scalar_subquery()
|
||||||
|
)
|
||||||
|
has_playback = (
|
||||||
|
select(MoviePlaybackSource.movie_id)
|
||||||
|
.where(MoviePlaybackSource.dead_at.is_(None))
|
||||||
|
.scalar_subquery()
|
||||||
|
)
|
||||||
|
perf_count = (
|
||||||
|
select(func.count())
|
||||||
|
.select_from(MoviePerformer)
|
||||||
|
.where(MoviePerformer.movie_id == Movie.id)
|
||||||
|
.correlate(Movie)
|
||||||
|
.scalar_subquery()
|
||||||
|
)
|
||||||
|
stmt = (
|
||||||
|
select(Movie)
|
||||||
|
.where(
|
||||||
|
Movie.id.not_in(has_tpdb),
|
||||||
|
Movie.id.in_(has_playback),
|
||||||
|
)
|
||||||
|
.where((perf_count == 0) | (Movie.studio_id.is_(None)))
|
||||||
|
.order_by(Movie.created_at.desc())
|
||||||
|
.limit(limit)
|
||||||
|
)
|
||||||
|
return list(session.execute(stmt).scalars().all())
|
||||||
|
|
||||||
|
|
||||||
|
def run_tpdb_movie_enrich(
|
||||||
|
session_factory,
|
||||||
|
*,
|
||||||
|
limit: int = 200,
|
||||||
|
min_title: float = 0.90,
|
||||||
|
) -> dict[str, int]:
|
||||||
|
"""Batch: wzbogać do `limit` filmów. Commit per-film (jeden błąd nie cofa reszty).
|
||||||
|
`session_factory` = kontekstowy scope (app.db.session_scope)."""
|
||||||
|
from app.ingest import get_or_create_source
|
||||||
|
from app.models.source import SourceKind
|
||||||
|
|
||||||
|
connector = TPDBConnector()
|
||||||
|
counters = {"seen": 0, "enriched": 0, "merged": 0, "no_match": 0, "skip": 0, "errors": 0}
|
||||||
|
|
||||||
|
with session_factory() as session:
|
||||||
|
src = get_or_create_source(session, kind=SourceKind.tpdb, name="tpdb")
|
||||||
|
source_id = src.id
|
||||||
|
session.commit()
|
||||||
|
movies = _candidate_movies(session, source_id=source_id, limit=limit)
|
||||||
|
movie_ids = [m.id for m in movies]
|
||||||
|
|
||||||
|
log.info("tpdb-movie-enrich: %d candidate movies", len(movie_ids))
|
||||||
|
for mid in movie_ids:
|
||||||
|
counters["seen"] += 1
|
||||||
|
try:
|
||||||
|
with session_factory() as session:
|
||||||
|
movie = session.get(Movie, mid)
|
||||||
|
if movie is None:
|
||||||
|
continue
|
||||||
|
outcome = enrich_movie(
|
||||||
|
session, movie, connector=connector, source_id=source_id, min_title=min_title
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
counters[outcome] = counters.get(outcome, 0) + 1
|
||||||
|
except Exception as e: # pragma: no cover - defensywnie, jeden film nie wywala batcha
|
||||||
|
counters["errors"] += 1
|
||||||
|
log.warning("tpdb-movie-enrich failed for %s: %s", mid, e)
|
||||||
|
|
||||||
|
log.info("tpdb-movie-enrich done: %s", counters)
|
||||||
|
return counters
|
||||||
|
|
@ -29,13 +29,22 @@ from app.extractors.tubes import (
|
||||||
_ytdlp,
|
_ytdlp,
|
||||||
eporner,
|
eporner,
|
||||||
freshporno,
|
freshporno,
|
||||||
|
fullmovies,
|
||||||
|
hdporngg,
|
||||||
|
hqfap,
|
||||||
hqporner,
|
hqporner,
|
||||||
|
javflix,
|
||||||
|
neporn,
|
||||||
latestpornvideo,
|
latestpornvideo,
|
||||||
paradisehill,
|
paradisehill,
|
||||||
porn00,
|
porn00,
|
||||||
pornhat,
|
porntrex,
|
||||||
pornxp,
|
supjav,
|
||||||
sxyprn,
|
sxyprn,
|
||||||
|
vjav,
|
||||||
|
watchporn,
|
||||||
|
xhamster,
|
||||||
|
yespornvip,
|
||||||
)
|
)
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
@ -50,70 +59,137 @@ log = logging.getLogger(__name__)
|
||||||
# embed-iframe extractor (page → /e/<id> iframe → P.A.C.K.E.R. unpack). Custom kod
|
# embed-iframe extractor (page → /e/<id> iframe → P.A.C.K.E.R. unpack). Custom kod
|
||||||
# tylko tam gdzie tube ma niestandardowy schemat (eporner XHR, sxyprn URL transform).
|
# tylko tam gdzie tube ma niestandardowy schemat (eporner XHR, sxyprn URL transform).
|
||||||
_REGISTRY: dict[str, Callable[[str], list[StreamSource] | None]] = {
|
_REGISTRY: dict[str, Callable[[str], list[StreamSource] | None]] = {
|
||||||
# Custom (zoptymalizowane / niestandardowy player)
|
# hqporner — dedicated extractor zwraca multi-quality `<source>` mp4 URLs
|
||||||
# hqporner — CDN URL (bigcdn.cc, video.flyflv.com z `ip=` parametrem) IP-bound do
|
# (bigcdn.cc / hqwo.cc / flyflv) z `force_proxy=True`. CDN URLs IP-bound do
|
||||||
# requestera. VPS resolve daje 200 ale mobile direct = 404/403. Switch na WebView
|
# VPS, więc playback.py routuje przez proxy — mobile dostaje quality picker
|
||||||
# fallback: mobile pobiera embed iframe (mydaddy.cc/hqwo.cc) z phone IP, FluidPlayer
|
# + natywny ExoPlayer, bez WebView.
|
||||||
# JS decoduje mp4 URL z mobile session. Plus INJECTED_JS skanuje `<source>.src`.
|
# Bug-report e8ddd8d4: WebView fallback (`_vps_blocked_fallback`) ładował
|
||||||
# ~32k scen (drugi po porntrex największy single saving). Verified 2026-05-18.
|
# hqporner.com scene page w WebView, ale ta strona ma ad-iframes (adtng,
|
||||||
"hqpornercom": _vps_blocked_fallback.extract,
|
# goaserv, mavrtracktor) + pop-under-triggery → user klikał i widział
|
||||||
|
# reklamę zamiast video. INJECTED_JS w PlayerScreen.tsx nie chwytał
|
||||||
|
# popupów dośc szybko. Powrót do natywnego = `<source>` mp4 picker omija
|
||||||
|
# tę ścieżkę całkowicie.
|
||||||
|
"hqpornercom": hqporner.extract,
|
||||||
"epornercom": eporner.extract,
|
"epornercom": eporner.extract,
|
||||||
"sxyprncom": sxyprn.extract,
|
"sxyprncom": sxyprn.extract,
|
||||||
# Mainstream tubes — yt-dlp
|
# Mainstream tubes — yt-dlp
|
||||||
# NB: 2026-05-18 cross-IP test potwierdził że xvideos/xnxx/pornhub/youporn/redtube
|
# NB: 2026-05-18 cross-IP test potwierdził że xvideos/xnxx/pornhub/youporn/redtube
|
||||||
# CDN URLs są **time-bound** (nie IP-bound) — mobile_direct_ok auto-detect w
|
# CDN URLs są **time-bound** (nie IP-bound) — mobile_direct_ok auto-detect w
|
||||||
# playback.py daje mobile direct fetch, zero VPS bandwidth.
|
# playback.py daje mobile direct fetch, zero VPS bandwidth.
|
||||||
"pornhubcom": _ytdlp.extract,
|
# pornhub + redtube — USUNIĘTE CAŁKOWICIE 2026-06-22 (user request). Scrapery były
|
||||||
"redtubecom": _ytdlp.extract,
|
# disabled od 2026-05-12 (0.4% canonical match — głównie skrócone amatorskie clipy),
|
||||||
|
# zamrożone sceny/źródła skasowane z DB. Brak ekstraktorów → zero resolve.
|
||||||
"xvideoscom": _ytdlp.extract,
|
"xvideoscom": _ytdlp.extract,
|
||||||
"xnxxcom": _ytdlp.extract,
|
"xnxxcom": _ytdlp.extract,
|
||||||
"youporncom": _ytdlp.extract,
|
"youporncom": _ytdlp.extract,
|
||||||
# porntrex KVS get_file — `kt_ips=<vps_ip>` cookie + single-use token (410 po reuse).
|
# porntrex KVS — 2026-05-22 VPS znów dociera (HTTP 200). Dedykowany extractor:
|
||||||
# CDN IP-bound do VPS, mobile direct = 403. Switch na _vps_blocked_fallback:
|
# flashvars `video_url` → `get_file` 302 → CDN time-bound signed URL
|
||||||
# mobile WebView z phone IP → KVS player JS dekoduje video.src → INJECTED_JS scrape.
|
# (`expires`+`md5`, NIE IP-bound) → mobile gra direct, zero VPS bandwidth.
|
||||||
# 137k scen oszczędzone z VPS bandwidth (largest single saving).
|
"porntrexcom": porntrex.extract,
|
||||||
"porntrexcom": _vps_blocked_fallback.extract,
|
# fpoxxx — KVS, plain get_file + license. 2026-06-01 (task #20): get_file 302 →
|
||||||
# VPS-blocked tubes — KVS / Cloudflare blokuje Hetzner IP, ale działają z residential
|
# `videos3.fpo.xxx/remote_control.php?acctoken=<base64>` — zdekodowany acctoken
|
||||||
# IP (potwierdzone Chrome DevTools MCP 2026-05-15). Mobile WebView + INJECTED_JS
|
# zawiera WBITY IP serwera-resolvera → definitywnie IP-bound. WebView only.
|
||||||
# (PlayerScreen.tsx:805) skanuje <video>.src + XHR — łapie URL po decode-ie player JS.
|
|
||||||
"xhamstercom": _vps_blocked_fallback.extract,
|
|
||||||
"porndittcom": _vps_blocked_fallback.extract,
|
|
||||||
"fpoxxx": _vps_blocked_fallback.extract,
|
"fpoxxx": _vps_blocked_fallback.extract,
|
||||||
"sxylandcom": _vps_blocked_fallback.extract,
|
# sxyland — embeduje playmogo.com/e/<id> (= klon DoodStream: doodcdn.io + pass_md5
|
||||||
|
# + niewidzialny CF Turnstile; Chrome-DevTools verify 2026-06-08, bug-report 827a50a1).
|
||||||
|
# Strona sxyland NIE jest Turnstile-gated (VPS curl wyciąga iframe URL z HTML), więc
|
||||||
|
# _embed_iframe wyłuskuje embed playmogo i oddaje jako type='hoster' → mobile
|
||||||
|
# doodstream.ts resolvuje phone-side (phone IP przechodzi invisible Turnstile) → direct
|
||||||
|
# mp4 → autoplay. Wcześniej _vps_blocked_fallback ładował CAŁĄ stronę sxyland w WebView
|
||||||
|
# (ads + klik-to-play + brak autoplay = dokładnie objaw z reportu 827a50a1).
|
||||||
|
"sxylandcom": _embed_iframe.extract,
|
||||||
# Aggregator tubes — generic embed-iframe → hoster unpacker
|
# Aggregator tubes — generic embed-iframe → hoster unpacker
|
||||||
"latestpornvideocom": latestpornvideo.extract,
|
"latestpornvideocom": latestpornvideo.extract,
|
||||||
"xmoviesforyoucom": _embed_iframe.extract,
|
"xmoviesforyoucom": _embed_iframe.extract,
|
||||||
"watchporn": _embed_iframe.extract,
|
# watchporn — 2026-07-02 przebudowany na KVS (DoodStream-CAPTCHA zniknął).
|
||||||
|
# flashvars get_file direct mp4, same-session 302 resolve, token nie IP-bound.
|
||||||
|
"watchporn": watchporn.extract,
|
||||||
"siskavideo": _embed_iframe.extract,
|
"siskavideo": _embed_iframe.extract,
|
||||||
"porn4dayspw": _embed_iframe.extract,
|
"porn4dayspw": _embed_iframe.extract,
|
||||||
"porndishcom": _embed_iframe.extract,
|
"porndishcom": _embed_iframe.extract,
|
||||||
# xxxfreewatch — DELISTED 2026-05-18. 790 solo-orphan scen, 0% match, CF-walled z VPS.
|
# xxxfreewatch — DELISTED 2026-05-18. 790 solo-orphan scen, 0% match, CF-walled z VPS.
|
||||||
"latestleaksco": _embed_iframe.extract,
|
"latestleaksco": _embed_iframe.extract,
|
||||||
"mypornerleakcom": _embed_iframe.extract,
|
"mypornerleakcom": _embed_iframe.extract,
|
||||||
# PornHat — dedicated extractor: tylko `<source>` z player area (skip sidebar
|
# xhamster — 2026-06-08 PRZEPIĘTE z _vps_blocked_fallback na natywny server-side HLS.
|
||||||
# trailer URLs `_preview*.mp4`), dedupe po filename. Get_file 302 → CDN, proxy
|
# Re-test (DevTools + cross-IP): VPS pobiera scene page bez CF challenge, master m3u8
|
||||||
# follow_redirects=True wymagane (fix w stream_proxy.py).
|
# w SSR HTML, manifest+segmenty time-bound (portable, nie IP-bound). Mobile gra HLS
|
||||||
"pornhatcom": pornhat.extract,
|
# direct, multi-quality, zero VPS proxy/WebView/reklam. Patrz tubes/xhamster.py.
|
||||||
# Freshporno KVS — `cv=` HMAC signed token IP-bound. Server-side resolve dało
|
# ~155k solo-scen upgrade z WebView-z-reklamami na natywne. Wcześniej WebView fallback
|
||||||
# 200 z VPS, ale laptop dostał 302+SSL error → token validate'uje requester IP.
|
# ładował ad-heavy stronę z phone IP (działało, ale gorszy UX + preroll VAST).
|
||||||
# Switch na WebView fallback: mobile pobiera embed page, KVS player decoduje
|
"xhamstercom": xhamster.extract,
|
||||||
# video_url w-page, ExoPlayer dostaje URL z phone session. ~15k scen.
|
# Freshporno KVS (function/0 + license). 2026-06-04 DevTools + cross-IP re-test
|
||||||
"freshpornoorg": _vps_blocked_fallback.extract,
|
# NAPRAWIA błąd z #20: finalny cdn4.freshporno.org/remote_control.php jest PORTABLE
|
||||||
# porn00 / pornxp — force_proxy=True wprost (IP-bound CDN). Switch na WebView
|
# (token time-bound nie IP-bound — VPS odtworzył token z residential → 206) ale
|
||||||
# fallback. Niski volume (84 scen), trivial saving ale konsystencja flow.
|
# wymaga browser-TLS (curl_cffi chrome/ExoPlayer → 206; plain curl → 000). W #20
|
||||||
"porn00org": _vps_blocked_fallback.extract,
|
# testowałem plain-curl-em poza sesją → 000 → błędnie „nieosiągalny" → WebView.
|
||||||
|
# Teraz backend-resolve jak yespornvip/pornditt (_kvs używa curl_cffi chrome).
|
||||||
|
# Native, multi-quality, zero proxy/WebView. (zweryfikowane na emulatorze przed deploy)
|
||||||
|
"freshpornoorg": freshporno.extract,
|
||||||
|
# porn00 — KVS (plain get_file + license). 2026-06-04 DevTools + cross-IP re-test
|
||||||
|
# NAPRAWIA błąd z #20: finalny fe.porn00.org/...?token=&expires= jest PORTABLE
|
||||||
|
# (token time-bound nie IP-bound — Bright Data residential proxy z innego IP → 206)
|
||||||
|
# ale wymaga browser-TLS (curl_cffi chrome → 206; plain curl → 403). W #20
|
||||||
|
# testowałem finalny URL plain-curl-em → 403 → błędnie „IP-bound" → WebView.
|
||||||
|
# Teraz backend-resolve przez _kvs (curl_cffi chrome), native multi-quality,
|
||||||
|
# ZERO proxy (wcześniej force_proxy łamał no-proxy). Same mechanizm co freshporno.
|
||||||
|
"porn00org": porn00.extract,
|
||||||
|
# pornxp — `<source> //sr.porn-xp.com/<token>/.../720.mp4` (redirect → xpxp.eu).
|
||||||
|
# 2026-06-01 (task #20): 403 cross-IP → token w path IP-bound. WebView only.
|
||||||
"pornxpph": _vps_blocked_fallback.extract,
|
"pornxpph": _vps_blocked_fallback.extract,
|
||||||
|
# yesporn.vip — KVS engine. VPS znów dociera (HTTP 200, odblokowane jak porntrex),
|
||||||
|
# więc resolvujemy SERVER-SIDE: dekoduj flashvars `video_url`/alt/alt2 (function/0/ +
|
||||||
|
# license_code, algo kt_player) → follow get_file 302 → portable cdn5 url (time-bound,
|
||||||
|
# NIE IP/cookie-bound, zweryfikowane cross-IP 2026-05-31). Mobile gra direct natywnie,
|
||||||
|
# multi-quality, ZERO WebView/reklam/preroll. Wcześniej WebView fallback pokazywał
|
||||||
|
# ad-heavy stronę a scrape łapał preroll-reklamę (bkcdn) zamiast wideo.
|
||||||
|
"yespornvip": yespornvip.extract,
|
||||||
# Direct-scraping tubes (mają też search scraper w connectors/direct_scrapers/)
|
# Direct-scraping tubes (mają też search scraper w connectors/direct_scrapers/)
|
||||||
# — używają identycznego embed-iframe pattern dla streamingu.
|
# — używają identycznego embed-iframe pattern dla streamingu.
|
||||||
# hdporn92com — DELISTED 2026-05-18. Scene pages to SEO shell bez player iframe,
|
# hdporn92com — DELISTED 2026-05-18. Scene pages to SEO shell bez player iframe,
|
||||||
# JS hijackuje kliki na popunder. Wszystkie playback_sources mass-marked dead.
|
# JS hijackuje kliki na popunder. Wszystkie playback_sources mass-marked dead.
|
||||||
# 0dayxx wraps watchporn.to embed. watchporn.to/get_file/ token IP-bound (302→410
|
# 0dayxx + pornditt + pornhat — USUNIĘTE CAŁKOWICIE 2026-06-22 (user request): orphan
|
||||||
# cross-IP). Switch na WebView fallback. ~5k scen.
|
# factories (0–0.2% canonical match), zastępujemy lepszymi źródłami. Dane skasowane.
|
||||||
"0dayxxcom": _vps_blocked_fallback.extract,
|
|
||||||
# CF-protected tube — curl_cffi w fetch_tube_html bypassa JA3, embed-iframe pattern.
|
# CF-protected tube — curl_cffi w fetch_tube_html bypassa JA3, embed-iframe pattern.
|
||||||
"perverzijacom": _embed_iframe.extract,
|
"perverzijacom": _embed_iframe.extract,
|
||||||
# Special: WebView-only (Yii2 session-bound player).
|
# Special: WebView-only (Yii2 session-bound player).
|
||||||
"paradisehillcc": paradisehill.extract,
|
"paradisehillcc": paradisehill.extract,
|
||||||
|
# PornDoe — dołączony 2026-05-21 (theporndude audit). Stream URL nie inline w
|
||||||
|
# SSR HTML (player JS init po Play click), więc WebView fallback: mobile pobiera
|
||||||
|
# /watch/<id> z phone IP, player JS dekoduje video.src, INJECTED_JS scrape.
|
||||||
|
# 0 VPS bandwidth — zgodne z pre-public bandwidth/anonimowość priorytet.
|
||||||
|
"porndoecom": _vps_blocked_fallback.extract,
|
||||||
|
# fullmovies.xxx + hdporn.gg — BRAKOWAŁO extractora (try_extract→None→"no stream";
|
||||||
|
# fullmovies.xxx + hdporn.gg — ta sama platforma (`<source>/get_file/8512/`).
|
||||||
|
# 2026-06-04 (DevTools + cross-IP, naprawia „loading forever" + bug 19866e9e):
|
||||||
|
# get_file binduje fpvcdn do IP FETCHERA + jest stateless + ważny ≥90s, więc
|
||||||
|
# oddajemy get_file NIEZRESOLWOWANY (mobile_direct) — telefon follow-uje 302 →
|
||||||
|
# fpvcdn z IP telefonu → gra. POMIJAMY 4K (time-out 30s na fpvcdn = przyczyna
|
||||||
|
# „loading forever"; 720/480p gra ~1s). Native, multi-quality, ZERO proxy/WebView.
|
||||||
|
# (#19866e9e wcześniej źle: założyłem „get_file 403 IP-bound" testem plain-curl.)
|
||||||
|
"fullmoviesxxx": fullmovies.extract,
|
||||||
|
"hdporngg": hdporngg.extract,
|
||||||
|
# hqfap (PlayTube CMS) — JSON-LD contentUrl = direct mp4. USUNIĘTY 2026-06-25 (cała
|
||||||
|
# biblioteka serwowała `/upload/videos/video_down.mp4` stub), PRZYWRÓCONY 2026-07-07:
|
||||||
|
# strona wróciła na CDN vstor.top z realnymi plikami (portable cross-IP, zweryfikowane),
|
||||||
|
# user request. 4k69 zostaje usunięty (nie sprawdzany ponownie).
|
||||||
|
"hqfapcom": hqfap.extract,
|
||||||
|
# javflix (JAV, WordPress) — przyciski serwerów to `<a class="myLink" href="<embed>">`
|
||||||
|
# (streamtape/voe/doodstream/emturbovid). Generyczny _embed_iframe łapie je anchor-hoster
|
||||||
|
# patternem → type='hoster', telefon resolwuje (voe backend, dood/filemoon phone-side).
|
||||||
|
# Wrapper javflix.extract odsiewa placeholder players.mp4.
|
||||||
|
"javflix": javflix.extract,
|
||||||
|
# vjav (JAV, TXXX network) — videofile.php -> get_file HLS (patrz tubes/vjav.py).
|
||||||
|
"vjav": vjav.extract,
|
||||||
|
# supjav (JAV, embed-aggregator) — data-link reverse-hex -> lk1 -> hoster (type=hoster).
|
||||||
|
"supjav": supjav.extract,
|
||||||
|
# neporn — KVS function/0 + license (jak freshporno). Server-side _kvs resolve →
|
||||||
|
# data001.neporn.com/remote_control.php portable (cross-IP 206, 2026-06-10).
|
||||||
|
"neporncom": neporn.extract,
|
||||||
|
# superporn — `<source>` mp4 (cdnst*.superporn.com) token IP-bound do fetchera
|
||||||
|
# (403 cross-IP, test 2026-06-10), a sama strona CF-blocked z VPS. Resolve MUSI
|
||||||
|
# być phone-side: WebView ładuje stronę z residential IP telefonu, INJECTED_JS
|
||||||
|
# bierze video.src. Ingest HTML idzie osobno przez Bright Data proxy (scraper).
|
||||||
|
"superporncom": _vps_blocked_fallback.extract,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -144,6 +220,15 @@ def supported_sitetags() -> tuple[str, ...]:
|
||||||
return tuple(_REGISTRY.keys())
|
return tuple(_REGISTRY.keys())
|
||||||
|
|
||||||
|
|
||||||
|
def is_vps_blocked_fallback(sitetag: str) -> bool:
|
||||||
|
"""True gdy sitetag resolvuje się TYLKO przez WebView fallback (IP-bound CDN /
|
||||||
|
ad-heavy / CAPTCHA — np. fpoxxx, pornxpph). Takie źródła dają gorszy
|
||||||
|
UX (reklamy, czarny ekran) niż natywny KVS/direct resolve, więc UI powinien je
|
||||||
|
rankować NIŻEJ gdy scena ma też natywne źródło (bug-report 2026-06-07: scena
|
||||||
|
pokazywała fpoxxx-WebView przed działającym freshporno bo sort był alfabetyczny)."""
|
||||||
|
return _REGISTRY.get(sitetag) is _vps_blocked_fallback.extract
|
||||||
|
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"try_extract",
|
"try_extract",
|
||||||
"supported_sitetags",
|
"supported_sitetags",
|
||||||
|
|
|
||||||
|
|
@ -56,10 +56,18 @@ def browser_get(
|
||||||
timeout: float = 60.0,
|
timeout: float = 60.0,
|
||||||
follow_redirects: bool = True,
|
follow_redirects: bool = True,
|
||||||
impersonate: str = _DEFAULT_IMPERSONATE,
|
impersonate: str = _DEFAULT_IMPERSONATE,
|
||||||
|
proxy: str | None = None,
|
||||||
) -> FetchResult:
|
) -> FetchResult:
|
||||||
"""GET z Chrome TLS fingerprint (curl_cffi). Spada do httpx gdy curl_cffi brak."""
|
"""GET z Chrome TLS fingerprint (curl_cffi). Spada do httpx gdy curl_cffi brak.
|
||||||
|
|
||||||
|
`proxy` (http://user:pass@host:port) — routuje request przez proxy. Używane
|
||||||
|
tylko do ingestu HTML tubów blokujących VPS IP (np. superporn przez Bright Data
|
||||||
|
ISP proxy). NIE dla streamów."""
|
||||||
if not _HAS_CURL_CFFI:
|
if not _HAS_CURL_CFFI:
|
||||||
with httpx.Client(timeout=timeout, follow_redirects=follow_redirects) as http:
|
proxies = {"http://": proxy, "https://": proxy} if proxy else None
|
||||||
|
with httpx.Client(
|
||||||
|
timeout=timeout, follow_redirects=follow_redirects, proxies=proxies
|
||||||
|
) as http:
|
||||||
r = http.get(url, headers=dict(headers or {}))
|
r = http.get(url, headers=dict(headers or {}))
|
||||||
return FetchResult(status_code=r.status_code, text=r.text, url=str(r.url))
|
return FetchResult(status_code=r.status_code, text=r.text, url=str(r.url))
|
||||||
|
|
||||||
|
|
@ -69,6 +77,8 @@ def browser_get(
|
||||||
timeout=timeout,
|
timeout=timeout,
|
||||||
impersonate=impersonate,
|
impersonate=impersonate,
|
||||||
allow_redirects=follow_redirects,
|
allow_redirects=follow_redirects,
|
||||||
|
proxies={"http": proxy, "https": proxy} if proxy else None,
|
||||||
|
verify=not proxy, # Bright Data MITM CA — curl_cffi nie ma go w bundlu
|
||||||
)
|
)
|
||||||
return FetchResult(status_code=r.status_code, text=r.text, url=str(r.url))
|
return FetchResult(status_code=r.status_code, text=r.text, url=str(r.url))
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -170,12 +170,28 @@ def extract_stream_from_hoster(
|
||||||
# Per-hoster dedicated extractors (specific URL shapes / decode patterns).
|
# Per-hoster dedicated extractors (specific URL shapes / decode patterns).
|
||||||
# Mixdrop: P.A.C.K.E.R. → MDCore.wurl protocol-relative `//host/v2/<id>.mp4?s=...`
|
# Mixdrop: P.A.C.K.E.R. → MDCore.wurl protocol-relative `//host/v2/<id>.mp4?s=...`
|
||||||
# — generic packer fallback regex `https?://...\.mp4` mija ten URL (no scheme).
|
# — generic packer fallback regex `https?://...\.mp4` mija ten URL (no scheme).
|
||||||
if re.search(r"(?:mixdrop|m1xdrop|mxdrop)\.[a-z]+/", iframe_url, re.IGNORECASE):
|
# `miixdrop` (double-i) to current canonical domain — wszystkie legacy
|
||||||
|
# `mixdrop.{ag,sb,my,co,...}` + `m1xdrop.bz` 301-ują tam. Bez `miixdrop`
|
||||||
|
# w dispatch URLs already-on-new-domain (upstream tubes które zaktualizowały
|
||||||
|
# embed src) trafiałyby do generic logic, gdzie regex `https?://...\.mp4`
|
||||||
|
# mija protocol-relative `//a-delivery22.mxcontent.net/...`.
|
||||||
|
if re.search(r"(?:mixdrop|miixdrop|m1xdrop|mxdrop)\.[a-z]+/", iframe_url, re.IGNORECASE):
|
||||||
from app.extractors.hosters import mixdrop
|
from app.extractors.hosters import mixdrop
|
||||||
sources = mixdrop.extract(iframe_url, timeout=timeout)
|
sources = mixdrop.extract(iframe_url, timeout=timeout)
|
||||||
if sources:
|
if sources:
|
||||||
return sources[0].link
|
return sources[0].link
|
||||||
# Fall through to generic logic gdyby dedicated zwrócił None.
|
# Fall through to generic logic gdyby dedicated zwrócił None.
|
||||||
|
# xtremestream (perverzija): videojs+hls.js player. Embed `/player/index.php?data=<HASH>`
|
||||||
|
# — m3u8 master playlist serwowany przez `/player/xs1.php?data=<HASH>` (ten sam HASH,
|
||||||
|
# inny endpoint; potwierdzone w inline JS `m3u8_loader_url`). Brak packera/file: w
|
||||||
|
# HTML → generic logic zwracała None → WebView fallback. Trywialna podmiana endpointu.
|
||||||
|
_xtr = re.search(
|
||||||
|
r"(https?://[^/]*xtremestream\.[a-z]+/player/)index\.php(\?data=[0-9a-f]+)",
|
||||||
|
iframe_url,
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
if _xtr:
|
||||||
|
return f"{_xtr.group(1)}xs1.php{_xtr.group(2)}"
|
||||||
# Streamtape: 4 `document.getElementById(...).innerHTML = prefix + (...).substring(N)`
|
# Streamtape: 4 `document.getElementById(...).innerHTML = prefix + (...).substring(N)`
|
||||||
# assignmenty, z czego 2 są DECOY z połamanym hostname. Dedicated decode picks
|
# assignmenty, z czego 2 są DECOY z połamanym hostname. Dedicated decode picks
|
||||||
# correct one + builds `/get_video?id=...&token=...` URL.
|
# correct one + builds `/get_video?id=...&token=...` URL.
|
||||||
|
|
@ -210,6 +226,32 @@ def extract_stream_from_hoster(
|
||||||
if sources:
|
if sources:
|
||||||
return sources[0].link
|
return sources[0].link
|
||||||
return None
|
return None
|
||||||
|
# filemoon "Byse" SPA — server-side resolve jest bezcelowy: stream URL z API
|
||||||
|
# jest IP-bound do requestera, więc VPS dostałby URL działający tylko z VPS.
|
||||||
|
# Zwracamy None od razu → _embed_iframe Stage 2 → type='hoster' → mobile
|
||||||
|
# filemoonHoster.ts robi POST /playback + AES-256-GCM z IP użytkownika.
|
||||||
|
if re.search(
|
||||||
|
r"//(?:[a-z0-9-]+\.)?(?:filemoon|kerapoxy|lvturbo|emturbovid|"
|
||||||
|
r"bysezoxexe|bysezejataos|moonseries)\.[a-z]{2,4}/",
|
||||||
|
iframe_url,
|
||||||
|
re.IGNORECASE,
|
||||||
|
):
|
||||||
|
log.debug("hoster %s: filemoon SPA → type=hoster (mobile-side resolve)", iframe_url)
|
||||||
|
return None
|
||||||
|
# DoodStream + klony (playmogo/doply/myvidplay/dood.*/doodstream) — protokół pass_md5
|
||||||
|
# + niewidzialny Cloudflare Turnstile wiązany z IP requestera. Server-side resolve
|
||||||
|
# bezcelowy (VPS dostaje Turnstile gate / IP-bound token). Zwracamy None → caller
|
||||||
|
# oddaje embed jako type='hoster' → mobile doodstream.ts robi pass_md5 z IP usera
|
||||||
|
# (residential przechodzi Turnstile). Bez tego leciało do yt-dlp → "Unsupported URL"
|
||||||
|
# spam (playmogo zalewał logi movie-resolve) + marnowanie ~sekund na martwy resolve.
|
||||||
|
if re.search(
|
||||||
|
r"//(?:[a-z0-9-]+\.)?(?:playmogo|doply|myvidplay|doodstream|dood|d0+d|"
|
||||||
|
r"dooood|do0od|do7go|ds2play|doodcdn)\.[a-z]{2,6}/",
|
||||||
|
iframe_url,
|
||||||
|
re.IGNORECASE,
|
||||||
|
):
|
||||||
|
log.debug("hoster %s: doodstream clone → type=hoster (mobile-side resolve)", iframe_url)
|
||||||
|
return None
|
||||||
headers = {
|
headers = {
|
||||||
"User-Agent": _DEFAULT_UA,
|
"User-Agent": _DEFAULT_UA,
|
||||||
"Accept": "text/html,application/xhtml+xml",
|
"Accept": "text/html,application/xhtml+xml",
|
||||||
|
|
|
||||||
|
|
@ -21,8 +21,8 @@ from __future__ import annotations
|
||||||
import logging
|
import logging
|
||||||
import re
|
import re
|
||||||
|
|
||||||
from app.extractors._fetch import browser_get
|
from app.extractors._fetch import _DEFAULT_UA, browser_get
|
||||||
from app.extractors._models import StreamSource
|
from app.extractors._models import HosterDead, StreamSource
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
@ -32,16 +32,34 @@ _PACKER_RE = re.compile(
|
||||||
re.DOTALL,
|
re.DOTALL,
|
||||||
)
|
)
|
||||||
_MP4_URL_RE = re.compile(r'MDCore\.wurl\s*=\s*"([^"]+\.mp4[^"]*)"')
|
_MP4_URL_RE = re.compile(r'MDCore\.wurl\s*=\s*"([^"]+\.mp4[^"]*)"')
|
||||||
|
# Dead-video page (200 OK but no packer, only the "sorry" shell). Wcześniej nasz
|
||||||
|
# extractor zwracał None bez sygnału "dead" → playback.py nie ustawiał dead_at,
|
||||||
|
# mobile dostawał pusty wynik → czarny ekran zamiast skip-to-next-source.
|
||||||
|
_DEAD_RE = re.compile(
|
||||||
|
r"can't find the video|WE ARE SORRY",
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def extract(page_url: str, *, timeout: float = 30.0) -> list[StreamSource] | None:
|
def extract(page_url: str, *, timeout: float = 30.0) -> list[StreamSource] | None:
|
||||||
res = browser_get(page_url, timeout=timeout)
|
# UA + Accept są wymagane — bez nich mixdrop dla VALID video zwraca minimalny
|
||||||
|
# body bez P.A.C.K.E.R. (sam stream_proxy._refetch_mixdrop_url też tak robi).
|
||||||
|
# Brak headerów powodował że extract() na żywym mixdrop ID dostawał shell bez
|
||||||
|
# packera → no match → None → mobile dostawał czarny ekran.
|
||||||
|
headers = {
|
||||||
|
"User-Agent": _DEFAULT_UA,
|
||||||
|
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||||
|
"Accept-Language": "en-US,en;q=0.9",
|
||||||
|
}
|
||||||
|
res = browser_get(page_url, headers=headers, timeout=timeout)
|
||||||
if res.status_code != 200 or not res.text:
|
if res.status_code != 200 or not res.text:
|
||||||
log.info("mixdrop: fetch fail status=%s url=%s", res.status_code, page_url)
|
log.info("mixdrop: fetch fail status=%s url=%s", res.status_code, page_url)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
m = _PACKER_RE.search(res.text)
|
m = _PACKER_RE.search(res.text)
|
||||||
if not m:
|
if not m:
|
||||||
|
if _DEAD_RE.search(res.text):
|
||||||
|
raise HosterDead(f"mixdrop {page_url}: video not found")
|
||||||
log.info("mixdrop: no P.A.C.K.E.R. block in %s (page changed?)", page_url)
|
log.info("mixdrop: no P.A.C.K.E.R. block in %s (page changed?)", page_url)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -124,7 +124,7 @@ _JS_SERVER_URL_RE = re.compile(
|
||||||
_ANCHOR_HOSTER_RE = re.compile(
|
_ANCHOR_HOSTER_RE = re.compile(
|
||||||
r'<a\s+[^>]*href=["\'](?P<url>https?://(?:'
|
r'<a\s+[^>]*href=["\'](?P<url>https?://(?:'
|
||||||
r'playmogo|luluvid|doodporn|doodstream|dood\.[a-z]+|streamtape|streamta\.pe|'
|
r'playmogo|luluvid|doodporn|doodstream|dood\.[a-z]+|streamtape|streamta\.pe|'
|
||||||
r'filemoon|streamwish|sdefx|veev|turbovidhls|gounlimited|iceyfile|hlswish|'
|
r'filemoon|emturbovid|streamwish|sdefx|veev|turbovidhls|gounlimited|iceyfile|hlswish|'
|
||||||
r'mixdrop|voe|vidoza|mediafire|asnwish|obeywish|streamruby|hqq\.[a-z]+|'
|
r'mixdrop|voe|vidoza|mediafire|asnwish|obeywish|streamruby|hqq\.[a-z]+|'
|
||||||
r'feurl|streamhide|krakenfiles|earnvids|jollytuna|peekvids|playerwish'
|
r'feurl|streamhide|krakenfiles|earnvids|jollytuna|peekvids|playerwish'
|
||||||
r')\.[a-z]{2,8}/[^"\']+)["\']',
|
r')\.[a-z]{2,8}/[^"\']+)["\']',
|
||||||
|
|
@ -177,13 +177,12 @@ DEAD_HOSTER_RE = re.compile(
|
||||||
r'|streamtape\.[a-z]+|streamta\.pe|streamtap\.com|streamcrypt\.net' # malware
|
r'|streamtape\.[a-z]+|streamta\.pe|streamtap\.com|streamcrypt\.net' # malware
|
||||||
r'|scloud\.ninja|stape\.fun|tapecontent\.net|streamtapeadblock\.[a-z]+' # streamtape mirrors
|
r'|scloud\.ninja|stape\.fun|tapecontent\.net|streamtapeadblock\.[a-z]+' # streamtape mirrors
|
||||||
r'|openload\.co|openload\.io|oload\.[a-z]+' # openload (offline od 2019)
|
r'|openload\.co|openload\.io|oload\.[a-z]+' # openload (offline od 2019)
|
||||||
# filemoon.* — wszystkie mirrory (filemoon.to/sx/nl/in/ru/co + aliasy
|
# filemoon — NIE jest na blacklist. ~2026-05 rebrand na SPA "Byse Frontend"
|
||||||
# kerapoxy.cc, lvturbo.com) serwują od ~2026-05 ten sam SPA "Byse Frontend"
|
# zabił stary P.A.C.K.E.R.-JWPlayer embed (stąd wcześniejsze błędne uznanie
|
||||||
# placeholder bez player JS. Globalny shutdown. Siska/perverzija/xmoviesforyou
|
# za "globalny shutdown"), ale video żyje za prywatnym JSON API. RE 2026-05-22:
|
||||||
# mają filemoon jako default embed → wszystkie sceny przez ten path = dead
|
# POST /api/videos/<code>/embed/playback {"fingerprint":{}} → AES-256-GCM →
|
||||||
# iframe (bug-report 16966e77 2026-05-16 "Niby 404 ale graficzne"). Blacklist
|
# m3u8. URL jest IP-bound, więc resolver MUSI iść z urządzenia użytkownika:
|
||||||
# eliminuje próby + wymusza fallback na alt hostera / TubePageError None.
|
# filemoon przelatuje jako type='hoster' → mobile/src/lib/filemoonHoster.ts.
|
||||||
r'|filemoon\.[a-z]{2,4}|kerapoxy\.cc|lvturbo\.com|emturbovid\.com' # dead 2026-05
|
|
||||||
r')'
|
r')'
|
||||||
r'(?:[:/]|$)', # port, path, lub end-of-string
|
r'(?:[:/]|$)', # port, path, lub end-of-string
|
||||||
re.IGNORECASE,
|
re.IGNORECASE,
|
||||||
|
|
@ -216,6 +215,12 @@ _IP_BOUND_CDN_RE = re.compile(
|
||||||
r"premilkyway\.com" # latestpornvideo
|
r"premilkyway\.com" # latestpornvideo
|
||||||
r"|tnmr\.org" # mypornerleak (legacy CDN)
|
r"|tnmr\.org" # mypornerleak (legacy CDN)
|
||||||
r"|acek-cdn\.com" # mypornerleak (current CDN, shared KVS infra)
|
r"|acek-cdn\.com" # mypornerleak (current CDN, shared KVS infra)
|
||||||
|
# xtremestream.xyz (perverzija) — `player/xs1.php?data=` to NIE direct mp4 tylko
|
||||||
|
# IP-bound player endpoint (403 cross-IP z VPS). Stage 1 zwracał go jako type=mp4
|
||||||
|
# → natywny player ładował HTML-player w nieskończoność (reports 06-24 "perverzija
|
||||||
|
# nie działa"/"loading w nieskończoność"). Skip → hoster fallback: WebView ładuje
|
||||||
|
# index.php playera z residential IP telefonu, xs1.php gra w jego sesji.
|
||||||
|
r"|xtremestream\.[a-z]{2,8}"
|
||||||
# URL signature shared across these CDNs: `/hls2/<XX>/<scene_id>/.../master.m3u8?t=<token>&s=<ts>&e=<exp>&srv=<srv>&asn=`
|
# URL signature shared across these CDNs: `/hls2/<XX>/<scene_id>/.../master.m3u8?t=<token>&s=<ts>&e=<exp>&srv=<srv>&asn=`
|
||||||
# — `asn` query param = Autonomous System Number bind. Generic match jako safety net.
|
# — `asn` query param = Autonomous System Number bind. Generic match jako safety net.
|
||||||
r")\b",
|
r")\b",
|
||||||
|
|
@ -278,6 +283,15 @@ def _extract_direct_stream_urls(page_html: str, page_url: str) -> list[StreamSou
|
||||||
# i embed pages z .mp4 suffix (playmogo/dood /d/ — to HTML, nie video).
|
# i embed pages z .mp4 suffix (playmogo/dood /d/ — to HTML, nie video).
|
||||||
if _NOT_DIRECT_STREAM_RE.search(url):
|
if _NOT_DIRECT_STREAM_RE.search(url):
|
||||||
continue
|
continue
|
||||||
|
# KVS false-positives (report c156e4b7/8db71220, watchporn przez _embed_iframe):
|
||||||
|
# preview_/videos_screenshots to krótkie trailery (404), a surowy get_file/ bez
|
||||||
|
# podpisanego query-tokena daje 403 — kt_player dokłada token z license_code (patrz
|
||||||
|
# natywny watchporn.extract). Żadne z nich to pełne wideo → pomijamy.
|
||||||
|
low = url.lower()
|
||||||
|
if "videos_screenshots/" in low or "/preview_" in low:
|
||||||
|
continue
|
||||||
|
if "/get_file/" in low and "?" not in url:
|
||||||
|
continue
|
||||||
# Quality wykrycie z nazwy pliku
|
# Quality wykrycie z nazwy pliku
|
||||||
q_int = 0
|
q_int = 0
|
||||||
q_label = "mp4"
|
q_label = "mp4"
|
||||||
|
|
|
||||||
186
app/extractors/tubes/_kvs.py
Normal file
186
app/extractors/tubes/_kvs.py
Normal file
|
|
@ -0,0 +1,186 @@
|
||||||
|
"""Współdzielony resolver dla KVS (kt_player) tube'ów z `function/0/` + `license_code`.
|
||||||
|
|
||||||
|
Wzorzec (yespornvip, pornditt, ... — wszystkie KVS na tym samym silniku):
|
||||||
|
flashvars: `video_url` / `video_alt_url` / `video_alt_url2` = `function/0/https://<host>/
|
||||||
|
get_file/<srv>/<HASH>/.../<id>.mp4/` (480/720/1080p) + `license_code: '$...'`.
|
||||||
|
kt_player dekoduje HASH (permutacja pierwszych 32 znaków algorytmem license_code —
|
||||||
|
algo zgodny z yt-dlp KVS `_kvs_get_real_url`, zweryfikowany 2026-05-31 że odtwarza
|
||||||
|
output kt_player).
|
||||||
|
|
||||||
|
Zdekodowany `get_file` 302-redirectuje do CDN (cdn5.yesporn.vip / twa.tgprn.com / ...)
|
||||||
|
który serwuje wideo (206). Finalny URL jest **time-bound signed, NIE IP/cookie-bound** —
|
||||||
|
portable cross-IP. get_file token jest session-bound, więc 302 MUSI być rozwiązany w tej
|
||||||
|
samej sesji curl_cffi co fetch strony. Oddajemy finalny portable CDN url per jakość →
|
||||||
|
mobile gra direct natywnie, multi-quality, zero WebView/reklam/proxy.
|
||||||
|
|
||||||
|
Dlaczego server-side (nie WebView): WebView ładuje ad-heavy stronę + VAST preroll, a
|
||||||
|
scrape łapie reklamę zamiast contentu. Direct get_file→CDN omija player/reklamę.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import re
|
||||||
|
import time
|
||||||
|
import urllib.parse as _up
|
||||||
|
|
||||||
|
from app.extractors._fetch import _DEFAULT_IMPERSONATE, _DEFAULT_UA, _HAS_CURL_CFFI
|
||||||
|
from app.extractors._models import StreamSource
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# Łapie OBA formaty flashvars:
|
||||||
|
# - `function/0/https://.../get_file/<HASH>/...` (yespornvip/pornditt/freshporno —
|
||||||
|
# zakodowany, dekodowany przez real_url)
|
||||||
|
# - `https://.../get_file/...` (porn00 — już zdekodowany plain get_file; real_url
|
||||||
|
# przepuszcza go bez zmian)
|
||||||
|
# W obu przypadkach finał to get_file → 302 → CDN (follow in-session).
|
||||||
|
_URL_RE = re.compile(
|
||||||
|
r"(video(?:_alt)?_url\d*)\s*:\s*[\"']"
|
||||||
|
r"(function/0/[^\"']+|https?://[^\"']*?/get_file/[^\"']*)[\"']",
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
_TEXT_RE = re.compile(
|
||||||
|
r"(video(?:_alt)?_url\d*)_text\s*:\s*[\"']([^\"']*)[\"']",
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
_LICENSE_RE = re.compile(r"license_code\s*:\s*[\"'](\$[^\"']+)[\"']", re.IGNORECASE)
|
||||||
|
|
||||||
|
_HASH_LENGTH = 32
|
||||||
|
|
||||||
|
# get_file 302-follow timeout. Zdrowy CDN odpowiada <1s; gdy wideo zostało usunięte
|
||||||
|
# z CDN (strona istnieje, ale get_file stalluje) request wisi do timeoutu. Trzymamy
|
||||||
|
# go NISKO i osobno od page-fetch timeoutu — bug 6ec1960e 2026-06-02: yesporn.vip
|
||||||
|
# przeszedł na cdn4/remote_control.php i dla martwych scen wszystkie 3 jakości
|
||||||
|
# wisiały po 60s = 180s → mobile "resolving w nieskończoność".
|
||||||
|
_GETFILE_TIMEOUT = 10.0
|
||||||
|
|
||||||
|
|
||||||
|
def _license_token(license_code: str) -> list[int]:
|
||||||
|
license_code = license_code.replace("$", "")
|
||||||
|
license_values = [int(c) for c in license_code]
|
||||||
|
modlicense = license_code.replace("0", "1")
|
||||||
|
center = len(modlicense) // 2
|
||||||
|
modlicense = str(4 * abs(int(modlicense[:center + 1]) - int(modlicense[center:])))[:center + 1]
|
||||||
|
return [
|
||||||
|
(license_values[index + offset] + current) % 10
|
||||||
|
for index, current in enumerate(int(c) for c in modlicense)
|
||||||
|
for offset in range(4)
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def real_url(video_url: str, license_code: str) -> str:
|
||||||
|
"""Dekoduje `function/0/...get_file/N/<HASH>/...` permutując pierwsze 32 znaki HASH."""
|
||||||
|
if not video_url.startswith("function/0/"):
|
||||||
|
return video_url
|
||||||
|
parsed = _up.urlparse(video_url[len("function/0/"):])
|
||||||
|
lt = _license_token(license_code)
|
||||||
|
parts = parsed.path.split("/")
|
||||||
|
h = parts[3][:_HASH_LENGTH]
|
||||||
|
idx = list(range(_HASH_LENGTH))
|
||||||
|
acc = 0
|
||||||
|
for src in reversed(range(_HASH_LENGTH)):
|
||||||
|
acc += lt[src]
|
||||||
|
dest = (src + acc) % _HASH_LENGTH
|
||||||
|
idx[src], idx[dest] = idx[dest], idx[src]
|
||||||
|
parts[3] = "".join(h[i] for i in idx) + parts[3][_HASH_LENGTH:]
|
||||||
|
return _up.urlunparse(parsed._replace(path="/".join(parts)))
|
||||||
|
|
||||||
|
|
||||||
|
def _quality_rank(label: str | None) -> int:
|
||||||
|
if not label:
|
||||||
|
return -1
|
||||||
|
m = re.search(r"(\d{3,4})\s*p", label, re.IGNORECASE)
|
||||||
|
return int(m.group(1)) if m else -1
|
||||||
|
|
||||||
|
|
||||||
|
def _resolve_get_file(session, base_url: str, get_file_url: str, timeout: float) -> str | None:
|
||||||
|
sep = "&" if "?" in get_file_url else "?"
|
||||||
|
url = f"{get_file_url}{sep}rnd={int(time.time() * 1000)}"
|
||||||
|
try:
|
||||||
|
r = session.get(
|
||||||
|
url, timeout=timeout, allow_redirects=True, stream=True,
|
||||||
|
headers={"Referer": base_url + "/", "Range": "bytes=0-1"},
|
||||||
|
)
|
||||||
|
final = str(r.url)
|
||||||
|
status = r.status_code
|
||||||
|
r.close()
|
||||||
|
except Exception as e:
|
||||||
|
log.info("kvs: get_file resolve failed (%s): %s", get_file_url[:60], e)
|
||||||
|
return None
|
||||||
|
if status >= 400 or "/get_file/" in final:
|
||||||
|
log.info("kvs: get_file resolve bad status=%s final=%s", status, final[:70])
|
||||||
|
return None
|
||||||
|
return final
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_kvs(page_url: str, *, base_url: str, timeout: float = 60.0) -> list[StreamSource] | None:
|
||||||
|
"""Fetch KVS page → decode function/0 get_file (per jakość) → follow 302 → portable CDN.
|
||||||
|
|
||||||
|
Zwraca StreamSource'y posortowane malejąco po jakości, lub None gdy nic nie wyszło.
|
||||||
|
base_url: scheme+host hosta (np. 'https://yesporn.vip') — do Referera i logów.
|
||||||
|
"""
|
||||||
|
if not _HAS_CURL_CFFI:
|
||||||
|
log.info("kvs: curl_cffi unavailable — cannot resolve %s", page_url)
|
||||||
|
return None
|
||||||
|
from curl_cffi import requests as _cf_requests
|
||||||
|
session = _cf_requests.Session(impersonate=_DEFAULT_IMPERSONATE)
|
||||||
|
try:
|
||||||
|
resp = session.get(
|
||||||
|
page_url,
|
||||||
|
headers={"User-Agent": _DEFAULT_UA, "Accept": "text/html,application/xhtml+xml"},
|
||||||
|
timeout=timeout, allow_redirects=True,
|
||||||
|
)
|
||||||
|
html = resp.text if resp.status_code < 400 else ""
|
||||||
|
except Exception as e:
|
||||||
|
log.info("kvs: page fetch failed %s: %s", page_url, e)
|
||||||
|
return None
|
||||||
|
if not html:
|
||||||
|
log.info("kvs: empty page %s", page_url)
|
||||||
|
return None
|
||||||
|
|
||||||
|
licm = _LICENSE_RE.search(html)
|
||||||
|
if not licm:
|
||||||
|
log.info("kvs: no license_code on %s", page_url)
|
||||||
|
return None
|
||||||
|
license_code = licm.group(1)
|
||||||
|
|
||||||
|
quality_by_var: dict[str, str] = {}
|
||||||
|
for m in _TEXT_RE.finditer(html):
|
||||||
|
quality_by_var[m.group(1).lower()] = m.group(2).strip()
|
||||||
|
|
||||||
|
# get_file timeout NISKI (osobny od page-fetch) + early-break: gdy 2 pierwsze
|
||||||
|
# jakości nie rozwiążą się (przy 0 dotychczasowych wyników) scena jest martwa na
|
||||||
|
# CDN — nie ma sensu czekać na trzecią (oszczędza kolejne _GETFILE_TIMEOUT).
|
||||||
|
gf_timeout = min(timeout, _GETFILE_TIMEOUT)
|
||||||
|
seen_dec: set[str] = set()
|
||||||
|
result: list[StreamSource] = []
|
||||||
|
fails = 0
|
||||||
|
for m in _URL_RE.finditer(html):
|
||||||
|
var_name = m.group(1).lower()
|
||||||
|
decoded = real_url(m.group(2), license_code)
|
||||||
|
if decoded in seen_dec:
|
||||||
|
continue
|
||||||
|
seen_dec.add(decoded)
|
||||||
|
final = _resolve_get_file(session, base_url, decoded, gf_timeout)
|
||||||
|
if not final:
|
||||||
|
fails += 1
|
||||||
|
if fails >= 2 and not result:
|
||||||
|
log.info("kvs: %d get_file fails, 0 results — martwa scena? %s", fails, page_url)
|
||||||
|
break
|
||||||
|
continue
|
||||||
|
result.append(
|
||||||
|
StreamSource(
|
||||||
|
link=final,
|
||||||
|
type="mp4",
|
||||||
|
quality=quality_by_var.get(var_name) or None,
|
||||||
|
referer=base_url + "/",
|
||||||
|
raw={"mobile_direct_ok": True},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
if not result:
|
||||||
|
log.info("kvs: no resolvable get_file on %s", page_url)
|
||||||
|
return None
|
||||||
|
|
||||||
|
result.sort(key=lambda s: _quality_rank(s.quality), reverse=True)
|
||||||
|
return result
|
||||||
76
app/extractors/tubes/_source_getfile.py
Normal file
76
app/extractors/tubes/_source_getfile.py
Normal file
|
|
@ -0,0 +1,76 @@
|
||||||
|
"""Współdzielony resolver dla tubów z `<source src=.../get_file/...mp4>` + IP-bound CDN
|
||||||
|
(hdporn.gg, fullmovies.xxx — ta sama platforma, `/get_file/8512/`).
|
||||||
|
|
||||||
|
2026-06-04 (DevTools + cross-IP investigation — naprawia „hdporngg loading forever"):
|
||||||
|
get_file 302-redirectuje do `fpvcdn.com` z **IP fetchera wbitym w URL** (`ip=<kto-fetchnął>`),
|
||||||
|
więc finalny CDN jest IP-bound do tego kto resolvuje. Dlatego oddajemy get_file URL
|
||||||
|
**NIEZRESOLWOWANY** (mobile_direct) — ExoPlayer na telefonie sam follow-uje 302, fpvcdn
|
||||||
|
bindje się do IP telefonu, gra. (Resolve na VPS → bind do IP VPS → mobile 403.)
|
||||||
|
|
||||||
|
Zweryfikowane: get_file jest STATELESS (świeża sesja działa) + ważny ≥90s, więc telefon
|
||||||
|
ma czas (resolve→picker→tap = sekundy). Źródło **2160p/4K konsekwentnie time-outuje na
|
||||||
|
fpvcdn (~30s)** → POMIJAMY je (to była przyczyna „loading forever" — player ładował 4K
|
||||||
|
pierwsze); 720p/480p/1080p resolvują w ~1s.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import re
|
||||||
|
|
||||||
|
from app.extractors._fetch import _DEFAULT_IMPERSONATE, _DEFAULT_UA, _HAS_CURL_CFFI
|
||||||
|
from app.extractors._models import StreamSource
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
_SOURCE_RE = re.compile(
|
||||||
|
r"<source\s+src=['\"]([^'\"]+/get_file/[^'\"]+\.mp4[^'\"]*)['\"]"
|
||||||
|
r"[^>]*?(?:title|label)=['\"]?([^'\">]*)",
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
# fpvcdn nie serwuje 4K (30s timeout) — skip żeby player nie wisiał na nim.
|
||||||
|
_SKIP_QUALITY_RE = re.compile(r"2160|1440|4k", re.IGNORECASE)
|
||||||
|
|
||||||
|
|
||||||
|
def resolve(page_url: str, base_url: str, *, timeout: float = 30.0) -> list[StreamSource] | None:
|
||||||
|
if not _HAS_CURL_CFFI:
|
||||||
|
log.info("source_getfile: curl_cffi unavailable — %s", page_url)
|
||||||
|
return None
|
||||||
|
from curl_cffi import requests as cf
|
||||||
|
try:
|
||||||
|
html = cf.get(
|
||||||
|
page_url, impersonate=_DEFAULT_IMPERSONATE,
|
||||||
|
headers={"User-Agent": _DEFAULT_UA, "Accept": "text/html,application/xhtml+xml"},
|
||||||
|
timeout=timeout,
|
||||||
|
).text
|
||||||
|
except Exception as e:
|
||||||
|
log.info("source_getfile: page fetch failed %s: %s", page_url, e)
|
||||||
|
return None
|
||||||
|
|
||||||
|
seen: set[str] = set()
|
||||||
|
out: list[StreamSource] = []
|
||||||
|
for m in _SOURCE_RE.finditer(html):
|
||||||
|
url = m.group(1).strip()
|
||||||
|
quality = (m.group(2) or "").strip()
|
||||||
|
if url.startswith("//"):
|
||||||
|
url = "https:" + url
|
||||||
|
if url in seen:
|
||||||
|
continue
|
||||||
|
seen.add(url)
|
||||||
|
if _SKIP_QUALITY_RE.search(quality):
|
||||||
|
log.info("source_getfile: skip broken-CDN quality %r on %s", quality, page_url)
|
||||||
|
continue
|
||||||
|
out.append(StreamSource(
|
||||||
|
link=url, type="mp4", quality=quality or None,
|
||||||
|
referer=base_url + "/", raw={"mobile_direct_ok": True},
|
||||||
|
))
|
||||||
|
|
||||||
|
if not out:
|
||||||
|
log.info("source_getfile: no playable <source>/get_file on %s", page_url)
|
||||||
|
return None
|
||||||
|
|
||||||
|
def _rank(s: StreamSource) -> int:
|
||||||
|
mm = re.search(r"(\d{3,4})", s.quality or "")
|
||||||
|
return int(mm.group(1)) if mm else -1
|
||||||
|
|
||||||
|
out.sort(key=_rank, reverse=True)
|
||||||
|
return out
|
||||||
|
|
@ -1,63 +1,24 @@
|
||||||
"""freshporno.org — KVS engine, BEZ `<source>` tagów.
|
"""freshporno.org — KVS (kt_player) direct stream extractor. Patrz app/extractors/tubes/_kvs.py.
|
||||||
|
|
||||||
Page używa kt_player (KVS Flash + JS legacy player) — URLs są wewnątrz JavaScript
|
Flashvars `video_url`/`video_alt_url`/`video_alt_url2` = `function/0/...get_file/...` +
|
||||||
flashvars JSON (`video_url: 'function/0/<URL>'`) i w `<a href="...?download=true">`
|
`license_code` (silnik identyczny z yespornvip/pornditt). Resolve server-side: decode +
|
||||||
linkach z labelem "MP4 720p" / "MP4 480p".
|
follow 302 → `cdn4.freshporno.org/remote_control.php?time=&cv=...` (206 video/mp4).
|
||||||
|
|
||||||
Bierzemy anchor pattern bo ma WSZYSTKIE quality z explicit labelem (vs flashvars
|
2026-06-04 (DevTools + cross-IP re-test, naprawia błędny wniosek z #20): finalny CDN url
|
||||||
ma tylko main+alt, max 2 jakości). `<a href="...get_file/...mp4/?download=true...">MP4 <q>p, ...`
|
jest **portable cross-IP** (token time-bound, NIE IP-bound — VPS odtworzył token
|
||||||
|
wygenerowany z residential IP → 206) ale wymaga **browser-podobnego TLS** (curl_cffi
|
||||||
Sidebar suggested videos używają `data-preview="...get_file/.../<id>_preview.mp4"` —
|
chrome / ExoPlayer/okhttp → 206; plain curl → connection 000). W #20 testowałem finalny
|
||||||
inny pattern (nie `<a href>`), więc anchor regex je naturalnie pomija.
|
URL plain-curl-em poza sesją → 000 → błędnie uznałem „nieosiągalny z residential" i
|
||||||
|
zostawiłem na WebView. _kvs.resolve_kvs używa curl_cffi chrome impersonation, więc dociera
|
||||||
CDN token IP-bound do VPS — mobile dostanie 403 na direct, fallback proxy działa.
|
do cdn4; mobile (ExoPlayer) gra direct. Native, multi-quality, zero WebView/proxy/reklam.
|
||||||
get_file 302 → `cdn4.freshporno.org/remote_control.php?...&file=<path>` direct mp4
|
|
||||||
(nie HLS). Type='mp4'.
|
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import logging
|
|
||||||
import re
|
|
||||||
|
|
||||||
from app.extractors._fetch import fetch_tube_html
|
|
||||||
from app.extractors._models import StreamSource
|
from app.extractors._models import StreamSource
|
||||||
|
from app.extractors.tubes import _kvs
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
_BASE = "https://freshporno.org"
|
||||||
|
|
||||||
# `<a href="<URL>?download=true...">MP4 <quality>p, <size>` — main + alt streams.
|
|
||||||
_ANCHOR_QUALITY_RE = re.compile(
|
|
||||||
r'<a\s+[^>]*href="(?P<url>https?://[^"]+/get_file/[^"]+\.mp4/)\?download=true[^"]*"'
|
|
||||||
r'[^>]*>\s*MP4\s+(?P<q>\d{3,4}p)',
|
|
||||||
re.IGNORECASE,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def extract(page_url: str, *, timeout: float = 60.0) -> list[StreamSource] | None:
|
def extract(page_url: str, *, timeout: float = 60.0) -> list[StreamSource] | None:
|
||||||
html = fetch_tube_html(page_url, timeout=timeout)
|
return _kvs.resolve_kvs(page_url, base_url=_BASE, timeout=timeout)
|
||||||
|
|
||||||
seen_keys: set[str] = set()
|
|
||||||
result: list[StreamSource] = []
|
|
||||||
for m in _ANCHOR_QUALITY_RE.finditer(html):
|
|
||||||
url = m.group("url")
|
|
||||||
quality = m.group("q")
|
|
||||||
# Dedupe po basename (path bez query string).
|
|
||||||
basename = url.rstrip("/").split("/")[-1]
|
|
||||||
if basename in seen_keys:
|
|
||||||
continue
|
|
||||||
seen_keys.add(basename)
|
|
||||||
result.append(StreamSource(link=url, type="mp4", quality=quality))
|
|
||||||
|
|
||||||
if not result:
|
|
||||||
log.info("freshporno: no MP4 anchor matches on %s", page_url)
|
|
||||||
return None
|
|
||||||
|
|
||||||
def _quality_key(s: StreamSource) -> int:
|
|
||||||
if not s.quality:
|
|
||||||
return -1
|
|
||||||
try:
|
|
||||||
return int(s.quality.rstrip("p"))
|
|
||||||
except ValueError:
|
|
||||||
return -1
|
|
||||||
|
|
||||||
result.sort(key=_quality_key, reverse=True)
|
|
||||||
return result
|
|
||||||
|
|
|
||||||
|
|
@ -1,68 +1,19 @@
|
||||||
"""fullmovies.xxx — direct mp4 sources extractor.
|
"""fullmovies.xxx — `<source>/get_file` tube z IP-bound fpvcdn. Patrz _source_getfile.py.
|
||||||
|
|
||||||
Detail page ma `<video class="video-js">` z multiple `<source>` (per quality):
|
Ta sama platforma co hdporn.gg (`/get_file/8512/`). 2026-06-04 (DevTools + cross-IP):
|
||||||
`<source src='https://www.fullmovies.xxx/get_file/<token>/<dir>/<id>/<id>_2160m.mp4/' type='video/mp4' label="2160p" selected="true">`
|
stary extractor zakładał „get_file IP-bound → 403 → force_proxy". Faktycznie get_file
|
||||||
`<source src='.../<id>_720m.mp4/' type='video/mp4' label="720p">`
|
binduje fpvcdn do IP **fetchera** (mobile), więc oddajemy get_file NIEZRESOLWOWANY
|
||||||
`<source src='.../<id>_480m.mp4/' type='video/mp4' label="480p">`
|
(mobile_direct, ZERO proxy) — telefon follow-uje 302 → fpvcdn z IP telefonu, gra. Pomijamy
|
||||||
|
4K/2160p (konsekwentnie time-outuje na fpvcdn ~30s; reszta gra ~1s). Bug 19866e9e: scena z
|
||||||
URL pattern: `https://www.fullmovies.xxx/get_file/<signed_token>/<dir>/<id>/<id>_<q>m.mp4/`
|
TYLKO fullmovies+hdporngg nie grała (extractor nie był zarejestrowany).
|
||||||
- Trailing slash — server odsyła 302 na CDN.
|
|
||||||
- `<signed_token>` IP-bound do requester (jak HQPorner /get_file/). Mobile direct = 403.
|
|
||||||
- force_proxy=True wymusza wszystko przez goon proxy (proxy follows redirect na CDN).
|
|
||||||
|
|
||||||
Quality labels: 2160p / 1080p / 720p / 480p / 360p.
|
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import logging
|
|
||||||
import re
|
|
||||||
|
|
||||||
from app.extractors._fetch import fetch_tube_html
|
|
||||||
from app.extractors._models import StreamSource
|
from app.extractors._models import StreamSource
|
||||||
|
from app.extractors.tubes import _source_getfile
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
_BASE = "https://www.fullmovies.xxx"
|
||||||
|
|
||||||
# Single-quoted attribute (apostrophes inside src=). Quality from `label="<q>"`.
|
|
||||||
_SOURCE_RE = re.compile(
|
|
||||||
r"""<source\s+src=['"](?P<url>https?://[^'"]+\.mp4/?)['"]"""
|
|
||||||
r"""\s+type=['"]video/mp4['"]"""
|
|
||||||
r"""\s+label=['"](?P<q>[^'"]+)['"]""",
|
|
||||||
re.IGNORECASE,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def extract(page_url: str, *, timeout: float = 60.0) -> list[StreamSource] | None:
|
def extract(page_url: str, *, timeout: float = 30.0) -> list[StreamSource] | None:
|
||||||
html = fetch_tube_html(page_url, timeout=timeout)
|
return _source_getfile.resolve(page_url, _BASE, timeout=timeout)
|
||||||
seen: set[str] = set()
|
|
||||||
result: list[StreamSource] = []
|
|
||||||
# fullmovies /get_file/ URL ma signed token IP-bound do requester. Bez force_proxy
|
|
||||||
# mobile dostaje 403. Proxy follows 302 na CDN.
|
|
||||||
proxy_flag = {"force_proxy": True}
|
|
||||||
for m in _SOURCE_RE.finditer(html):
|
|
||||||
url = m.group("url")
|
|
||||||
if url in seen:
|
|
||||||
continue
|
|
||||||
seen.add(url)
|
|
||||||
result.append(
|
|
||||||
StreamSource(
|
|
||||||
link=url,
|
|
||||||
type="mp4",
|
|
||||||
quality=m.group("q"),
|
|
||||||
referer=f"{page_url}",
|
|
||||||
raw=proxy_flag,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
if not result:
|
|
||||||
log.info("fullmovies: no <source> tags on %s", page_url)
|
|
||||||
return None
|
|
||||||
|
|
||||||
# Sort by quality desc (2160p > 1080p > 720p > 480p > 360p)
|
|
||||||
def _q(s: StreamSource) -> int:
|
|
||||||
try:
|
|
||||||
return int((s.quality or "0").rstrip("p"))
|
|
||||||
except ValueError:
|
|
||||||
return 0
|
|
||||||
|
|
||||||
result.sort(key=_q, reverse=True)
|
|
||||||
return result
|
|
||||||
|
|
|
||||||
17
app/extractors/tubes/hdporngg.py
Normal file
17
app/extractors/tubes/hdporngg.py
Normal file
|
|
@ -0,0 +1,17 @@
|
||||||
|
"""hdporn.gg — `<source>/get_file` tube z IP-bound fpvcdn. Patrz _source_getfile.py.
|
||||||
|
|
||||||
|
Bug: „hdporngg loading który trwa nie wiadomo ile". Przyczyna (DevTools 2026-06-04):
|
||||||
|
player ładował 2160p/4K jako pierwsze, a 4K source na fpvcdn time-outuje (~30s); 720/480p
|
||||||
|
grają w ~1s. Oddajemy niezresolwowany get_file (mobile sam follow-uje 302 → fpvcdn z IP
|
||||||
|
telefonu, bo CDN jest IP-bound do fetchera) i POMIJAMY 4K. Native, multi-quality, bez WebView.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from app.extractors._models import StreamSource
|
||||||
|
from app.extractors.tubes import _source_getfile
|
||||||
|
|
||||||
|
_BASE = "https://www.hdporn.gg"
|
||||||
|
|
||||||
|
|
||||||
|
def extract(page_url: str, *, timeout: float = 30.0) -> list[StreamSource] | None:
|
||||||
|
return _source_getfile.resolve(page_url, _BASE, timeout=timeout)
|
||||||
84
app/extractors/tubes/hqfap.py
Normal file
84
app/extractors/tubes/hqfap.py
Normal file
|
|
@ -0,0 +1,84 @@
|
||||||
|
"""hqfap.com — direct stream extractor.
|
||||||
|
|
||||||
|
Scene page (SSR, za Cloudflare → curl_cffi w fetch_tube_html) ma JSON-LD
|
||||||
|
VideoObject z `contentUrl` = direct mp4. Generacje hostingu w katalogu:
|
||||||
|
|
||||||
|
- obecne sceny (2026-07-07): `d*.vstor.top/whlvid/<epoch>/<token>/.../<id>_<q>p.mp4`
|
||||||
|
— token time-bound, PORTABLE cross-IP (206 z residential ISP i VPS Hetzner,
|
||||||
|
weryfikacja 2026-07-07 po powrocie strony),
|
||||||
|
- starsze: `v4.cdnde.com/...?video=<b64>&time=<epoch>&ip=<addr>` (`ip` nieegzekwowany)
|
||||||
|
oraz `vd*.okcdn.ru/?expires=...&srcIp=...` (ok.ru) — również portable cross-IP.
|
||||||
|
|
||||||
|
Wszystkie generacje grają direct z telefonu → StreamSource ma `mobile_direct_ok`
|
||||||
|
(vstor.top nie łapie się w `_TIME_BOUND_CDN_RE` w playback.py, więc flagujemy jawnie),
|
||||||
|
zero proxy/WebView.
|
||||||
|
|
||||||
|
Historia: wyłączony 2026-06-22 i USUNIĘTY 2026-06-25 gdy CAŁA biblioteka CDN
|
||||||
|
serwowała stały `/upload/videos/video_down.mp4` stub. Strona wróciła na nowy CDN
|
||||||
|
(vstor.top) z realnymi plikami → przywrócony 2026-07-07 (user request). Guard na
|
||||||
|
stub zostaje defensywnie.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import re
|
||||||
|
|
||||||
|
from app.extractors._fetch import fetch_tube_html
|
||||||
|
from app.extractors._models import StreamSource
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
_JSONLD_RE = re.compile(
|
||||||
|
r'<script[^>]+type=["\']application/ld\+json["\'][^>]*>(.*?)</script>',
|
||||||
|
re.IGNORECASE | re.DOTALL,
|
||||||
|
)
|
||||||
|
# Fallback gdy JSON-LD nie parsuje się jako JSON (trailing comma itp.).
|
||||||
|
_CONTENT_URL_RE = re.compile(r'"contentUrl"\s*:\s*"([^"]+)"')
|
||||||
|
_QUALITY_RE = re.compile(r"_(\d{3,4})p\.mp4", re.IGNORECASE)
|
||||||
|
|
||||||
|
|
||||||
|
def extract(page_url: str, *, timeout: float = 60.0) -> list[StreamSource] | None:
|
||||||
|
html = fetch_tube_html(page_url, timeout=timeout)
|
||||||
|
|
||||||
|
content_url: str | None = None
|
||||||
|
for m in _JSONLD_RE.finditer(html):
|
||||||
|
raw = m.group(1).strip()
|
||||||
|
if not raw:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
data = json.loads(raw)
|
||||||
|
except (json.JSONDecodeError, ValueError):
|
||||||
|
continue
|
||||||
|
items = data if isinstance(data, list) else [data]
|
||||||
|
for obj in items:
|
||||||
|
if isinstance(obj, dict) and obj.get("@type") == "VideoObject":
|
||||||
|
content_url = (obj.get("contentUrl") or "").strip() or None
|
||||||
|
break
|
||||||
|
if content_url:
|
||||||
|
break
|
||||||
|
if not content_url:
|
||||||
|
rm = _CONTENT_URL_RE.search(html)
|
||||||
|
content_url = rm.group(1).strip() if rm else None
|
||||||
|
if not content_url or not content_url.startswith("http"):
|
||||||
|
log.warning("hqfap: no contentUrl in JSON-LD for %s", page_url)
|
||||||
|
return None
|
||||||
|
|
||||||
|
# Guard: `/upload/videos/video_down.mp4` (+ mirror *.workers.dev) to stały ~3MB
|
||||||
|
# "server down" placeholder (powód usunięcia 2026-06-25). Jeśli wróci — traktuj
|
||||||
|
# jak brak źródła (lepiej żadne niż stub).
|
||||||
|
if "/upload/videos/video_down.mp4" in content_url:
|
||||||
|
log.info("hqfap: stub video_down.mp4 (placeholder, no real video) on %s", page_url)
|
||||||
|
return None
|
||||||
|
|
||||||
|
qm = _QUALITY_RE.search(content_url)
|
||||||
|
quality = f"{qm.group(1)}p" if qm else None
|
||||||
|
return [
|
||||||
|
StreamSource(
|
||||||
|
link=content_url,
|
||||||
|
quality=quality,
|
||||||
|
type="mp4",
|
||||||
|
referer="https://hqfap.com/",
|
||||||
|
raw={"mobile_direct_ok": True},
|
||||||
|
)
|
||||||
|
]
|
||||||
|
|
@ -32,10 +32,25 @@ from app.extractors.hoster import extract_stream_from_hoster
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
_IFRAME_RE = re.compile(
|
# Wyciągamy zawartość `<div id="playerWrapper">…</div>` osobno, potem szukamy
|
||||||
r'<div[^>]+id=["\']?playerWrapper["\']?[^>]*>.*?<iframe[^>]+src=["\']([^"\']+)',
|
# `<iframe>` TYLKO wewnątrz. Wcześniej regex `playerWrapper>.*?<iframe` z DOTALL
|
||||||
|
# przelatywał przez pusty/JS-loaded wrapper i łapał kolejny `<iframe>` w
|
||||||
|
# dokumencie — a hqporner ma 2-3 ad-iframes (adtng/goaserv/mavrtracktor) wokół
|
||||||
|
# playera, więc trafialiśmy w reklamę zamiast w mydaddy.cc/hqwo.cc.
|
||||||
|
_PLAYER_WRAPPER_RE = re.compile(
|
||||||
|
r'<div[^>]+id=["\']?playerWrapper["\']?[^>]*>(.*?)</div>',
|
||||||
re.IGNORECASE | re.DOTALL,
|
re.IGNORECASE | re.DOTALL,
|
||||||
)
|
)
|
||||||
|
_PLAYER_IFRAME_RE = re.compile(r'<iframe[^>]+src=["\']([^"\']+)', re.IGNORECASE)
|
||||||
|
|
||||||
|
# Whitelist hostów embed iframe'a. Hqporner rotuje między mydaddy.cc i hqwo.cc
|
||||||
|
# (zmiany typowo co kilka miesięcy). Wszystko inne (adtng, goaserv, mavrtracktor,
|
||||||
|
# smartpop, popcash, reebr) → reklama. Brak match = fail safe (return None),
|
||||||
|
# nie próbujemy go odpalić jako hostera bo to ad-redirect → pop-under.
|
||||||
|
_VIDEO_IFRAME_HOST_RE = re.compile(
|
||||||
|
r"//(?:[a-z0-9-]+\.)?(?:mydaddy|hqwo|hqporner|flyflv)\.[a-z]{2,4}/",
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
|
||||||
# Match `<source src="...mp4" title="...">` z opcjonalnym title. Po unescape
|
# Match `<source src="...mp4" title="...">` z opcjonalnym title. Po unescape
|
||||||
# (`\"` → `"`) ten regex łapie zarówno raw HTML (mydaddy.cc) jak i JS-embedded
|
# (`\"` → `"`) ten regex łapie zarówno raw HTML (mydaddy.cc) jak i JS-embedded
|
||||||
|
|
@ -45,19 +60,36 @@ _SOURCE_RE = re.compile(
|
||||||
re.IGNORECASE,
|
re.IGNORECASE,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Whitelist CDN-ów mp4. Real video URLs lecą z bigcdn.cc (s12./s68./...),
|
||||||
|
# hqwo.cc/pubs, flyflv. Wszystko spoza listy w `<source>` tagu = pre-roll /
|
||||||
|
# interstitial / ad injection (hipoteza z bug-reportu: hqporner zaczął
|
||||||
|
# wrzucać ad mp4 URLs do `<source>` w 2026).
|
||||||
|
_VIDEO_CDN_HOST_RE = re.compile(
|
||||||
|
r"//(?:[a-z0-9-]+\.)?(?:bigcdn|hqwo|flyflv|hqwallcdn)\.[a-z]{2,4}/",
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def extract(page_url: str, *, timeout: float = 60.0) -> list[StreamSource] | None:
|
def extract(page_url: str, *, timeout: float = 60.0) -> list[StreamSource] | None:
|
||||||
page_html = fetch_tube_html(page_url, timeout=timeout)
|
page_html = fetch_tube_html(page_url, timeout=timeout)
|
||||||
m = _IFRAME_RE.search(page_html)
|
wrapper_m = _PLAYER_WRAPPER_RE.search(page_html)
|
||||||
if not m:
|
if not wrapper_m:
|
||||||
log.warning("hqporner: no iframe in %s", page_url)
|
log.warning("hqporner: no playerWrapper div in %s", page_url)
|
||||||
return None
|
return None
|
||||||
iframe_src = m.group(1).strip()
|
iframe_m = _PLAYER_IFRAME_RE.search(wrapper_m.group(1))
|
||||||
|
if not iframe_m:
|
||||||
|
log.warning("hqporner: no iframe inside playerWrapper for %s", page_url)
|
||||||
|
return None
|
||||||
|
iframe_src = iframe_m.group(1).strip()
|
||||||
if iframe_src.startswith("//"):
|
if iframe_src.startswith("//"):
|
||||||
iframe_src = "https:" + iframe_src
|
iframe_src = "https:" + iframe_src
|
||||||
elif iframe_src.startswith("/"):
|
elif iframe_src.startswith("/"):
|
||||||
iframe_src = f"https://hqporner.com{iframe_src}"
|
iframe_src = f"https://hqporner.com{iframe_src}"
|
||||||
|
|
||||||
|
if not _VIDEO_IFRAME_HOST_RE.search(iframe_src):
|
||||||
|
log.warning("hqporner: iframe host not whitelisted (likely ad): %s", iframe_src)
|
||||||
|
return None
|
||||||
|
|
||||||
headers = {
|
headers = {
|
||||||
"User-Agent": _DEFAULT_UA,
|
"User-Agent": _DEFAULT_UA,
|
||||||
"Accept": "text/html,application/xhtml+xml",
|
"Accept": "text/html,application/xhtml+xml",
|
||||||
|
|
@ -94,8 +126,23 @@ def extract(page_url: str, *, timeout: float = 60.0) -> list[StreamSource] | Non
|
||||||
if url in seen_urls:
|
if url in seen_urls:
|
||||||
continue
|
continue
|
||||||
seen_urls.add(url)
|
seen_urls.add(url)
|
||||||
|
# Drop `<source>` URLs spoza znanych CDN-ów. Jeśli hqporner wstrzyknie
|
||||||
|
# `<source src="//ads.example.com/preroll.mp4">` (hipoteza z bug-reportu)
|
||||||
|
# — bez whitelist'a quality picker w mobile mógłby wystrzelić mu URL
|
||||||
|
# reklamy zamiast 1080p mp4.
|
||||||
|
if not _VIDEO_CDN_HOST_RE.search(url):
|
||||||
|
log.info("hqporner: skip non-CDN source URL: %s", url)
|
||||||
|
continue
|
||||||
title = (sm.group(2) or "").strip()
|
title = (sm.group(2) or "").strip()
|
||||||
sources.append(StreamSource(link=url, quality=title or None, type="mp4", referer=iframe_referer))
|
# `force_proxy=True` (2026-05-20): CDN-y bigcdn.cc/flyflv IP-bound + flyflv ma
|
||||||
|
# `ip=46.62.219.154` w URL path. Mobile direct = 404/403 → fallback proxy
|
||||||
|
# generuje flicker. Force_proxy wymusza mobile użycie proxied od razu.
|
||||||
|
# Bug-report e8ddd8d4: "kliknięcie otwiera reklamę" gdy _vps_blocked_fallback
|
||||||
|
# (hqporner page ads). Force_proxy + native mp4 = quality picker + natywny.
|
||||||
|
sources.append(StreamSource(
|
||||||
|
link=url, quality=title or None, type="mp4", referer=iframe_referer,
|
||||||
|
raw={"force_proxy": True},
|
||||||
|
))
|
||||||
|
|
||||||
if sources:
|
if sources:
|
||||||
return sources
|
return sources
|
||||||
|
|
@ -110,5 +157,7 @@ def extract(page_url: str, *, timeout: float = 60.0) -> list[StreamSource] | Non
|
||||||
|
|
||||||
# Fallback 2: oddaj iframe URL jako hoster type — mobile otworzy w WebView,
|
# Fallback 2: oddaj iframe URL jako hoster type — mobile otworzy w WebView,
|
||||||
# FluidPlayer JS sam wyciągnie URL po user click / przejściu adblock check.
|
# FluidPlayer JS sam wyciągnie URL po user click / przejściu adblock check.
|
||||||
|
# Iframe_src ma już zwalidowany host whitelist (mydaddy.cc/hqwo.cc), więc
|
||||||
|
# WebView nie wpadnie w ad-domain redirect.
|
||||||
log.info("hqporner: using hoster fallback for %s", iframe_src)
|
log.info("hqporner: using hoster fallback for %s", iframe_src)
|
||||||
return [StreamSource(link=iframe_src, type="hoster")]
|
return [StreamSource(link=iframe_src, type="hoster")]
|
||||||
|
|
|
||||||
19
app/extractors/tubes/javflix.py
Normal file
19
app/extractors/tubes/javflix.py
Normal file
|
|
@ -0,0 +1,19 @@
|
||||||
|
"""javflix.cc extractor — cienki wrapper na generyczny _embed_iframe.
|
||||||
|
|
||||||
|
javflix trzyma hostery w `<a class="myLink" href="<embed>">` (streamtape/voe/doodstream/
|
||||||
|
emturbovid), które _embed_iframe łapie anchor-hoster patternem. Wrapper odsiewa tylko
|
||||||
|
placeholder `players.mp4` (pusty iframe zanim JS podmieni src) — bez tego trafiał jako
|
||||||
|
martwe pierwsze źródło type='mp4'.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from app.extractors._models import StreamSource
|
||||||
|
from app.extractors.tubes import _embed_iframe
|
||||||
|
|
||||||
|
|
||||||
|
def extract(page_url: str, *, timeout: float = 60.0) -> list[StreamSource] | None:
|
||||||
|
srcs = _embed_iframe.extract(page_url, timeout=timeout)
|
||||||
|
if not srcs:
|
||||||
|
return None
|
||||||
|
srcs = [s for s in srcs if "players.mp4" not in s.link]
|
||||||
|
return srcs or None
|
||||||
19
app/extractors/tubes/neporn.py
Normal file
19
app/extractors/tubes/neporn.py
Normal file
|
|
@ -0,0 +1,19 @@
|
||||||
|
"""neporn.com — KVS (kt_player) direct stream extractor. Patrz app/extractors/tubes/_kvs.py.
|
||||||
|
|
||||||
|
Flashvars `video_url` = `function/0/...get_file/...` + `license_code` (silnik jak
|
||||||
|
freshporno/porn00/yespornvip). Resolve server-side: decode + follow 302 →
|
||||||
|
`data001.neporn.com/remote_control.php?time=&cv=...`.
|
||||||
|
|
||||||
|
Cross-IP test 2026-06-10: finalny URL portable (token time-bound, NIE IP-bound —
|
||||||
|
VPS resolve → lokalny ISP 206 video/mp4). Mobile gra direct, zero proxy/WebView.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from app.extractors._models import StreamSource
|
||||||
|
from app.extractors.tubes import _kvs
|
||||||
|
|
||||||
|
_BASE = "https://neporn.com"
|
||||||
|
|
||||||
|
|
||||||
|
def extract(page_url: str, *, timeout: float = 60.0) -> list[StreamSource] | None:
|
||||||
|
return _kvs.resolve_kvs(page_url, base_url=_BASE, timeout=timeout)
|
||||||
|
|
@ -1,50 +1,24 @@
|
||||||
"""porn00.org — KVS engine extractor.
|
"""porn00.org — KVS (kt_player) direct stream extractor. Patrz app/extractors/tubes/_kvs.py.
|
||||||
|
|
||||||
Detail page wbudowuje stream URLs w JS flashvars block:
|
flashvars `video_url`/`video_alt_url` to PLAIN get_file (już zdekodowany, bez function/0):
|
||||||
- `video_url: 'https://.../get_file/.../<id>.mp4/?v-acctoken=...'` (default, 360p)
|
`https://www.porn00.org/get_file/3/<hash>/.../<id>.mp4` + `license_code`. _kvs._URL_RE
|
||||||
- `video_alt_url: 'https://.../get_file/.../<id>_720p.mp4/?v-acctoken=...'` (alt, 720p)
|
łapie też ten format; real_url przepuszcza plain bez dekodowania; follow 302 →
|
||||||
|
`fe.porn00.org/videos/.../<id>.mp4?token=&expires=` (206 video/mp4).
|
||||||
|
|
||||||
CDN token (`v-acctoken=...`) jest IP-bound do VPS, mobile direct fetch → 403.
|
2026-06-04 (DevTools + cross-IP re-test, naprawia błąd z #20): finalny CDN jest
|
||||||
playback.py wraps URL przez stream_proxy z `Referer: <page_url>` — działa.
|
**portable cross-IP** (token time-bound nie IP-bound — zweryfikowane przez Bright Data
|
||||||
|
residential proxy: get_file z jednego IP, fetch finalnego z innego IP → 206). W #20
|
||||||
Get_file → 302 → direct mp4 (jak freshporno). Type='mp4'.
|
testowałem finalny URL plain-curl-em → 403 → błędnie „IP-bound" → WebView. Wymaga
|
||||||
|
browser-TLS (curl_cffi chrome → 206; plain curl → 403). _kvs używa curl_cffi chrome,
|
||||||
|
mobile (ExoPlayer) gra direct. Native, multi-quality, ZERO proxy (wcześniej force_proxy).
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import logging
|
|
||||||
import re
|
|
||||||
|
|
||||||
from app.extractors._fetch import fetch_tube_html
|
|
||||||
from app.extractors._models import StreamSource
|
from app.extractors._models import StreamSource
|
||||||
|
from app.extractors.tubes import _kvs
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
_BASE = "https://www.porn00.org"
|
||||||
|
|
||||||
_VIDEO_URL_RE = re.compile(
|
|
||||||
r"""video_url:\s*['"]([^'"]+\.mp4[^'"]*)['"]""", re.IGNORECASE,
|
|
||||||
)
|
|
||||||
_VIDEO_ALT_URL_RE = re.compile(
|
|
||||||
r"""video_alt_url:\s*['"]([^'"]+\.mp4[^'"]*)['"]""", re.IGNORECASE,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def extract(page_url: str, *, timeout: float = 60.0) -> list[StreamSource] | None:
|
def extract(page_url: str, *, timeout: float = 60.0) -> list[StreamSource] | None:
|
||||||
html = fetch_tube_html(page_url, timeout=timeout)
|
return _kvs.resolve_kvs(page_url, base_url=_BASE, timeout=timeout)
|
||||||
result: list[StreamSource] = []
|
|
||||||
|
|
||||||
# Preferujemy alt (720p) przed default (360p).
|
|
||||||
# CDN token `v-acctoken` jest IP-bound do VPS. Mobile direct fetch ZAWSZE → 403,
|
|
||||||
# więc oznacz force_proxy żeby player od razu używał proxified URL bez prób direct.
|
|
||||||
# Bez tego: każdy playback = "mrugnięcie" (direct fail → fallback na proxy).
|
|
||||||
proxy_flag = {"force_proxy": True}
|
|
||||||
if (m := _VIDEO_ALT_URL_RE.search(html)):
|
|
||||||
result.append(StreamSource(link=m.group(1), type="mp4", quality="720p", raw=proxy_flag))
|
|
||||||
if (m := _VIDEO_URL_RE.search(html)):
|
|
||||||
url = m.group(1)
|
|
||||||
if not result or result[0].link != url:
|
|
||||||
result.append(StreamSource(link=url, type="mp4", quality="360p", raw=proxy_flag))
|
|
||||||
|
|
||||||
if not result:
|
|
||||||
log.info("porn00: no video_url flashvars on %s", page_url)
|
|
||||||
return None
|
|
||||||
|
|
||||||
return result
|
|
||||||
|
|
|
||||||
|
|
@ -1,86 +0,0 @@
|
||||||
"""pornhat.com — KVS engine. get_file 302 → HLS m3u8 manifest.
|
|
||||||
|
|
||||||
**2026-05-18 bandwidth optimization**: pornhat CDN tokens (`cdn.privatehost.com`) są
|
|
||||||
**time-bound, nie IP-bound** (`?sign=<HMAC>&exp_time=<unix>`). Zweryfikowane Chrome
|
|
||||||
DevTools MCP — VPS-resolved URL działa z każdego IP, bez Referer header. Zamiast
|
|
||||||
zwracać `pornhat.com/get_file/` URL (mobile dostaje go i robi 302 chain przez VPS
|
|
||||||
proxy), robimy server-side resolve i zwracamy końcowy manifest URL z signed token.
|
|
||||||
|
|
||||||
Mobile ExoPlayer otrzymuje:
|
|
||||||
`https://nvms12.cdn.privatehost.com/hls/contents/.../?sign=...&exp_time=...`
|
|
||||||
i pobiera manifest + segments direct z CDN. **Zero VPS bandwidth** (poza ~5KB
|
|
||||||
initial resolve fetch).
|
|
||||||
|
|
||||||
`mobile_direct_ok=True` w `raw` mówi playback.py że dla type=m3u8 ten URL jest OK
|
|
||||||
dla `direct_url=raw_url` (zazwyczaj m3u8 by szły przez proxy).
|
|
||||||
|
|
||||||
Token wygasa za ~30-120 min od resolve (depends na lra param). User pause+resume
|
|
||||||
po >2h może dostać 403 → mobile fallback na proxified URL re-resolve'a.
|
|
||||||
"""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import logging
|
|
||||||
|
|
||||||
import httpx
|
|
||||||
|
|
||||||
from app.extractors._models import StreamSource
|
|
||||||
from app.extractors.tubes._kvs_source import extract_kvs_sources
|
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
|
|
||||||
def _resolve_get_file_redirect(get_file_url: str, *, timeout: float = 15.0) -> str | None:
|
|
||||||
"""Follow 302 chain pornhat.com/get_file/ → cdn.privatehost.com/hls/...
|
|
||||||
|
|
||||||
Returns final manifest URL z signed token, lub None gdy fail.
|
|
||||||
"""
|
|
||||||
try:
|
|
||||||
with httpx.Client(
|
|
||||||
timeout=timeout,
|
|
||||||
follow_redirects=True,
|
|
||||||
headers={
|
|
||||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
|
|
||||||
"Referer": "https://www.pornhat.com/",
|
|
||||||
},
|
|
||||||
) as c:
|
|
||||||
r = c.head(get_file_url)
|
|
||||||
final = str(r.url)
|
|
||||||
if "cdn.privatehost.com" in final and ".m3u8" not in final:
|
|
||||||
# Generic master URL: /hls/contents/... CDN serves jako m3u8 mime
|
|
||||||
# nawet bez .m3u8 w path (sprawdzone Content-Type).
|
|
||||||
return final
|
|
||||||
if ".m3u8" in final:
|
|
||||||
return final
|
|
||||||
log.info("pornhat resolve: unexpected final URL %s", final)
|
|
||||||
return None
|
|
||||||
except Exception as e:
|
|
||||||
log.warning("pornhat resolve %s failed: %s", get_file_url, e)
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def extract(page_url: str, *, timeout: float = 60.0) -> list[StreamSource] | None:
|
|
||||||
sources = extract_kvs_sources(
|
|
||||||
page_url, stream_type="m3u8", timeout=timeout, log_tag="pornhat"
|
|
||||||
)
|
|
||||||
if not sources:
|
|
||||||
return None
|
|
||||||
|
|
||||||
# Resolve każdy get_file URL → CDN signed manifest URL. Mobile dostaje direct.
|
|
||||||
resolved: list[StreamSource] = []
|
|
||||||
for s in sources:
|
|
||||||
final = _resolve_get_file_redirect(s.link)
|
|
||||||
if final:
|
|
||||||
resolved.append(
|
|
||||||
StreamSource(
|
|
||||||
link=final,
|
|
||||||
type="m3u8",
|
|
||||||
quality=s.quality,
|
|
||||||
referer=s.referer,
|
|
||||||
raw={"mobile_direct_ok": True},
|
|
||||||
)
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
# Fallback: keep original (proxy will re-resolve)
|
|
||||||
resolved.append(s)
|
|
||||||
|
|
||||||
return resolved
|
|
||||||
163
app/extractors/tubes/porntrex.py
Normal file
163
app/extractors/tubes/porntrex.py
Normal file
|
|
@ -0,0 +1,163 @@
|
||||||
|
"""porntrex.com — KVS engine direct stream extractor.
|
||||||
|
|
||||||
|
2026-05-22: VPS Hetzner IP znów dociera do porntrex (HTTP 200) — wcześniej blokada
|
||||||
|
trzymała `porntrexcom` na `_vps_blocked_fallback`. Patrz [[goon_porntrex_vps_unblocked]].
|
||||||
|
|
||||||
|
KVS player: detail page ma `flashvars` z `video_url` / `video_alt_url` / `video_alt_url2`
|
||||||
|
(480p / 720p / 1080p), każdy to `get_file/<srv>/<token>/<path>.mp4/` URL.
|
||||||
|
|
||||||
|
`get_file` 302 → `cdn.pcdn.cloudswitches.com/...mp4?expires=<ts>&md5=<sig>` — to
|
||||||
|
**time-bound signed URL** (nie IP-bound, NIE cookie-bound) → po rozwiązaniu jest
|
||||||
|
portable: mobile gra direct z CDN, zero VPS bandwidth.
|
||||||
|
|
||||||
|
REVISION 2026-05-31 (bug usera "porntrex wolno + brak wyboru jakości + chyba proxy"):
|
||||||
|
Wcześniejsze założenie "mobile zużyje get_file sam" było BŁĘDNE — `get_file` token jest
|
||||||
|
**cookie/session-bound**: działa tylko w tej samej sesji curl_cffi która pobrała stronę.
|
||||||
|
Osobny request mobile (ExoPlayer, bez cookies) → 410 → mobile spadał na VPS proxy
|
||||||
|
(stąd flicker = nav.replace + wolne odtwarzanie). Zweryfikowane: same-session follow
|
||||||
|
get_file → 200 (streamuje wideo); fresh session → 410. Finalny CDN url (cloudswitches,
|
||||||
|
expires+md5) jest natomiast portable (fresh session → 206).
|
||||||
|
|
||||||
|
FIX: resolvujemy 302 NA BACKENDZIE (w tej samej sesji co fetch strony) i oddajemy
|
||||||
|
**finalny CDN url** per jakość. Mobile gra direct, multi-quality picker działa, zero proxy.
|
||||||
|
Token get_file zużywamy raz tu; CDN url jest time-bound (nie single-use) → starcza na
|
||||||
|
sesję odtwarzania.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import re
|
||||||
|
import time
|
||||||
|
|
||||||
|
from app.extractors._fetch import _DEFAULT_IMPERSONATE, _DEFAULT_UA, _HAS_CURL_CFFI, fetch_tube_html
|
||||||
|
from app.extractors._models import HosterDead, StreamSource
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
_BASE = "https://www.porntrex.com"
|
||||||
|
|
||||||
|
# Porntrex soft-delete: usunięte wideo zwraca HTTP 200 ze stroną z komunikatem (np.
|
||||||
|
# "this video was deleted per copyright owner request") zamiast playera → extract bez
|
||||||
|
# tego zwracał [] = transient → źródło NIGDY nie oznaczone dead → user wciąż klika
|
||||||
|
# martwy link (zgłoszenie 75dbf53e). Match → raise HosterDead → resolve mark-dead.
|
||||||
|
_DEAD_RE = re.compile(
|
||||||
|
r"this video (?:was|has been) deleted|video (?:was|has been) removed"
|
||||||
|
r"|no longer available|video is unavailable",
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
|
||||||
|
# flashvars: `video_url: 'https://.../get_file/...mp4/'` + `video_url_text: '480p'`.
|
||||||
|
# Warianty: video_url, video_alt_url, video_alt_url2, video_alt_url3...
|
||||||
|
_URL_RE = re.compile(
|
||||||
|
r"(video(?:_alt)?_url\d*)\s*:\s*'(https?://[^']+/get_file/[^']+)'",
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
_TEXT_RE = re.compile(
|
||||||
|
r"(video(?:_alt)?_url\d*)_text\s*:\s*'([^']*)'",
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _quality_rank(label: str | None) -> int:
|
||||||
|
"""`1080p` → 1080, `720p HD` → 720. Do sortowania malejąco."""
|
||||||
|
if not label:
|
||||||
|
return -1
|
||||||
|
m = re.search(r"(\d{3,4})\s*p", label, re.IGNORECASE)
|
||||||
|
return int(m.group(1)) if m else -1
|
||||||
|
|
||||||
|
|
||||||
|
def _resolve_get_file(session, get_file_url: str, timeout: float) -> str | None:
|
||||||
|
"""Follow get_file 302 → finalny portable CDN url (w sesji która ma cookies strony).
|
||||||
|
|
||||||
|
`?rnd=` cache-bust jak kt_player. stream=True + Range → łapiemy tylko nagłówki/finalny
|
||||||
|
URL po redirectach, NIE pobieramy 644MB body. Zwraca None gdy resolve padł."""
|
||||||
|
sep = "&" if "?" in get_file_url else "?"
|
||||||
|
url = f"{get_file_url}{sep}rnd={int(time.time() * 1000)}"
|
||||||
|
try:
|
||||||
|
r = session.get(
|
||||||
|
url,
|
||||||
|
timeout=timeout,
|
||||||
|
allow_redirects=True,
|
||||||
|
stream=True,
|
||||||
|
headers={"Referer": _BASE + "/", "Range": "bytes=0-1"},
|
||||||
|
)
|
||||||
|
final = str(r.url)
|
||||||
|
status = r.status_code
|
||||||
|
r.close()
|
||||||
|
except Exception as e:
|
||||||
|
log.info("porntrex: get_file resolve failed (%s): %s", get_file_url[:60], e)
|
||||||
|
return None
|
||||||
|
if status >= 400 or "/get_file/" in final:
|
||||||
|
log.info("porntrex: get_file resolve bad status=%s final=%s", status, final[:70])
|
||||||
|
return None
|
||||||
|
return final
|
||||||
|
|
||||||
|
|
||||||
|
def extract(page_url: str, *, timeout: float = 60.0) -> list[StreamSource] | None:
|
||||||
|
# Wspólna sesja: get_file token jest cookie/session-bound, więc 302 MUSI być
|
||||||
|
# rozwiązany w tej samej sesji curl_cffi co fetch strony (patrz docstring).
|
||||||
|
session = None
|
||||||
|
if _HAS_CURL_CFFI:
|
||||||
|
from curl_cffi import requests as _cf_requests
|
||||||
|
session = _cf_requests.Session(impersonate=_DEFAULT_IMPERSONATE)
|
||||||
|
try:
|
||||||
|
resp = session.get(
|
||||||
|
page_url,
|
||||||
|
headers={"User-Agent": _DEFAULT_UA, "Accept": "text/html,application/xhtml+xml"},
|
||||||
|
timeout=timeout,
|
||||||
|
allow_redirects=True,
|
||||||
|
)
|
||||||
|
html = resp.text if resp.status_code < 400 else ""
|
||||||
|
except Exception as e:
|
||||||
|
log.info("porntrex: page fetch failed %s: %s", page_url, e)
|
||||||
|
html = ""
|
||||||
|
if not html:
|
||||||
|
html = fetch_tube_html(page_url, timeout=timeout)
|
||||||
|
session = None # fetch_tube_html użył innej sesji → nie resolvuj w `session`
|
||||||
|
else:
|
||||||
|
html = fetch_tube_html(page_url, timeout=timeout)
|
||||||
|
|
||||||
|
# Soft-delete: strona żyje (200) ale wideo skasowane → mark dead (nie transient).
|
||||||
|
# Tylko gdy html niepuste (puste = fetch fail = transient, NIE dead).
|
||||||
|
if html and _DEAD_RE.search(html):
|
||||||
|
raise HosterDead(f"porntrex {page_url}: video deleted/removed")
|
||||||
|
|
||||||
|
# Mapa <var_name> → quality label (np. video_alt_url → "720p HD").
|
||||||
|
quality_by_var: dict[str, str] = {}
|
||||||
|
for m in _TEXT_RE.finditer(html):
|
||||||
|
quality_by_var[m.group(1).lower()] = m.group(2).strip()
|
||||||
|
|
||||||
|
seen: set[str] = set()
|
||||||
|
result: list[StreamSource] = []
|
||||||
|
for m in _URL_RE.finditer(html):
|
||||||
|
var_name = m.group(1).lower()
|
||||||
|
url = m.group(2)
|
||||||
|
if url in seen:
|
||||||
|
continue
|
||||||
|
seen.add(url)
|
||||||
|
quality = quality_by_var.get(var_name)
|
||||||
|
# Rozwiąż get_file → portable CDN url (w sesji ze stroną). Gdy resolve padnie,
|
||||||
|
# oddaj get_file jako fallback (mobile spróbuje direct → ewentualnie proxy).
|
||||||
|
final_link = url
|
||||||
|
if session is not None:
|
||||||
|
resolved = _resolve_get_file(session, url, timeout)
|
||||||
|
if resolved:
|
||||||
|
final_link = resolved
|
||||||
|
result.append(
|
||||||
|
StreamSource(
|
||||||
|
link=final_link,
|
||||||
|
type="mp4",
|
||||||
|
quality=quality or None,
|
||||||
|
referer=_BASE + "/",
|
||||||
|
# Finalny CDN url (cloudswitches) jest time-bound (expires+md5), nie
|
||||||
|
# cookie/IP-bound → mobile gra direct, zero VPS proxy bandwidth.
|
||||||
|
raw={"mobile_direct_ok": True},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
if not result:
|
||||||
|
log.info("porntrex: no KVS video_url in flashvars on %s", page_url)
|
||||||
|
return None
|
||||||
|
|
||||||
|
result.sort(key=lambda s: _quality_rank(s.quality), reverse=True)
|
||||||
|
return result
|
||||||
112
app/extractors/tubes/supjav.py
Normal file
112
app/extractors/tubes/supjav.py
Normal file
|
|
@ -0,0 +1,112 @@
|
||||||
|
"""supjav.com — JAV embed-aggregator. Resolve data-link → hoster (phone-side).
|
||||||
|
|
||||||
|
supjav chowa realny hoster za `data-link` (hex) na przyciskach serwerów (TV/FST/ST/VOE).
|
||||||
|
base.js: klik serwera → iframe `src = lk1.supremejav.com/supjav.php?l=<data-link>&bg=`.
|
||||||
|
supjav.php robi `OLID = data-link.reverse()` (odwrócenie stringa hex) i ładuje
|
||||||
|
`supjav.php?c=<OLID>`, które 302-uje na realny hoster (RE 2026-07-10):
|
||||||
|
TV → turbovidhls.com, FST → fc2stream.tv, ST → streamtape.com, VOE → voe.sx
|
||||||
|
|
||||||
|
Flow ekstraktora (on-demand, play time):
|
||||||
|
1. fetch detail `/<id>.html` PRZEZ proxy (Bright Data — supjav CF-blokuje VPS IP),
|
||||||
|
2. parse `data-link` z `.btn-server`,
|
||||||
|
3. reverse hex → GET `lk1.supremejav.com/supjav.php?c=<rev>` (direct z VPS, follow 302)
|
||||||
|
→ finalny hoster URL,
|
||||||
|
4. zwróć type='hoster' → telefon resolwuje (dood/filemoon natywnie, reszta WebView
|
||||||
|
fallback z residential IP). lk1 osiągalny z VPS bez proxy; tylko detail wymaga proxy.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import re
|
||||||
|
|
||||||
|
import curl_cffi.requests as _rq
|
||||||
|
|
||||||
|
from app.config import get_settings
|
||||||
|
from app.extractors import browser_get
|
||||||
|
from app.extractors._fetch import _DEFAULT_IMPERSONATE
|
||||||
|
from app.extractors._models import StreamSource
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
_LK = "https://lk1.supremejav.com/supjav.php"
|
||||||
|
_DATA_LINK_RE = re.compile(r'class="btn-server[^"]*"\s+data-link="([0-9a-f]{16,})"', re.IGNORECASE)
|
||||||
|
# reklama/tracker domeny które lk1 czasem zwraca zamiast hostera — odrzucamy.
|
||||||
|
_AD_RE = re.compile(r"(snaptrckr|trackwilltrk|mayzaent|eix304|doppiocdn|/ad\?)", re.IGNORECASE)
|
||||||
|
|
||||||
|
|
||||||
|
def fetch_supjav_html(url: str, *, proxy: str | None, timeout: float, tries: int = 6) -> str:
|
||||||
|
"""Fetch supjav przez proxy z retry. Bright Data rotuje IP per-request, a CF
|
||||||
|
przepuszcza tylko część IP (reszta dostaje ~6KB challenge). Retry aż trafimy IP
|
||||||
|
który przechodzi (realna strona ma `data-link=`/`<h1>` i jest duża). Zwraca '' gdy
|
||||||
|
wszystkie próby padły."""
|
||||||
|
last = ""
|
||||||
|
for _ in range(max(1, tries)):
|
||||||
|
try:
|
||||||
|
res = browser_get(url, timeout=timeout, proxy=proxy)
|
||||||
|
html = res.text if hasattr(res, "text") else res
|
||||||
|
except Exception:
|
||||||
|
html = ""
|
||||||
|
last = html or last
|
||||||
|
if html and len(html) > 15000 and ("data-link=" in html or "<h1>" in html):
|
||||||
|
return html
|
||||||
|
return last
|
||||||
|
|
||||||
|
|
||||||
|
def _resolve_hoster(data_link: str, timeout: float) -> str | None:
|
||||||
|
"""reverse hex → lk1 supjav.php?c= → follow 302 → finalny hoster URL."""
|
||||||
|
olid = data_link[::-1]
|
||||||
|
try:
|
||||||
|
s = _rq.Session(impersonate=_DEFAULT_IMPERSONATE)
|
||||||
|
r = s.get(
|
||||||
|
f"{_LK}?c={olid}",
|
||||||
|
headers={"Referer": "https://lk1.supremejav.com/"},
|
||||||
|
allow_redirects=True,
|
||||||
|
timeout=timeout,
|
||||||
|
)
|
||||||
|
final = str(r.url)
|
||||||
|
r.close()
|
||||||
|
except Exception as e:
|
||||||
|
log.info("supjav: lk1 resolve failed (%s): %s", data_link[:12], e)
|
||||||
|
return None
|
||||||
|
# supjav.php bez realnego hostera zostaje na lk1 (albo leci na ad) → odrzuć.
|
||||||
|
if "supremejav.com" in final or _AD_RE.search(final):
|
||||||
|
return None
|
||||||
|
# lk1 dokleja fragment `#supjav.com@<code>` (metadata) — hostery go ignorują, tniemy.
|
||||||
|
return final.split("#", 1)[0]
|
||||||
|
|
||||||
|
|
||||||
|
def extract(page_url: str, *, timeout: float = 60.0) -> list[StreamSource] | None:
|
||||||
|
proxy = get_settings().brightdata_proxy_url
|
||||||
|
if not proxy:
|
||||||
|
log.info("supjav: brak proxy — nie mogę pobrać CF-blokowanego detalu %s", page_url)
|
||||||
|
return None
|
||||||
|
html = fetch_supjav_html(page_url, proxy=proxy, timeout=timeout)
|
||||||
|
if not html:
|
||||||
|
log.info("supjav: detail fetch failed (CF) %s", page_url)
|
||||||
|
return None
|
||||||
|
|
||||||
|
data_links = list(dict.fromkeys(_DATA_LINK_RE.findall(html)))
|
||||||
|
if not data_links:
|
||||||
|
log.info("supjav: brak data-link na %s", page_url)
|
||||||
|
return None
|
||||||
|
|
||||||
|
seen: set[str] = set()
|
||||||
|
out: list[StreamSource] = []
|
||||||
|
for dl in data_links:
|
||||||
|
hoster = _resolve_hoster(dl, timeout=min(timeout, 40.0))
|
||||||
|
if not hoster or hoster in seen:
|
||||||
|
continue
|
||||||
|
seen.add(hoster)
|
||||||
|
host = hoster.split("/")[2] if "://" in hoster else hoster
|
||||||
|
out.append(
|
||||||
|
StreamSource(
|
||||||
|
link=hoster,
|
||||||
|
type="hoster",
|
||||||
|
quality=host,
|
||||||
|
referer="https://supjav.com/",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if not out:
|
||||||
|
log.info("supjav: żaden data-link nie rozwiązał się na hoster (%s)", page_url)
|
||||||
|
return None
|
||||||
|
return out
|
||||||
|
|
@ -21,7 +21,7 @@ import logging
|
||||||
import re
|
import re
|
||||||
|
|
||||||
from app.extractors._fetch import fetch_tube_html
|
from app.extractors._fetch import fetch_tube_html
|
||||||
from app.extractors._models import StreamSource
|
from app.extractors._models import HosterDead, StreamSource
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
@ -48,6 +48,12 @@ def _boo(ss: int, es: int) -> str:
|
||||||
|
|
||||||
def extract(page_url: str, *, timeout: float = 60.0) -> list[StreamSource] | None:
|
def extract(page_url: str, *, timeout: float = 60.0) -> list[StreamSource] | None:
|
||||||
html = fetch_tube_html(page_url, timeout=timeout)
|
html = fetch_tube_html(page_url, timeout=timeout)
|
||||||
|
# sxyprn soft-404: usunięty post zwraca HTTP 200 ze stroną "Post Not Found"
|
||||||
|
# (nie 404), więc bez tego extractor zwracał None → resolve traktował jako
|
||||||
|
# transient i NIGDY nie oznaczał źródła dead → user wciąż dostawał martwy link
|
||||||
|
# (bug-report 2026-06-07, scena 75aa3316). Raise HosterDead → resolve mark-dead.
|
||||||
|
if "Post Not Found" in html:
|
||||||
|
raise HosterDead(f"sxyprn {page_url}: post deleted (Post Not Found)")
|
||||||
m = _VNFO_RE.search(html)
|
m = _VNFO_RE.search(html)
|
||||||
if not m:
|
if not m:
|
||||||
log.warning("sxyprn: no data-vnfo in %s", page_url)
|
log.warning("sxyprn: no data-vnfo in %s", page_url)
|
||||||
|
|
|
||||||
134
app/extractors/tubes/vjav.py
Normal file
134
app/extractors/tubes/vjav.py
Normal file
|
|
@ -0,0 +1,134 @@
|
||||||
|
"""vjav.com — TXXX-network JAV tube. HLS stream extractor.
|
||||||
|
|
||||||
|
vjav to SPA na silniku TXXX (ktk_player + MSE/blob src). Stream NIE jest w HTML
|
||||||
|
strony (player renderuje się JS-owo). Zamiast tego AJAX endpoint zwraca zaciemniony
|
||||||
|
URL pliku:
|
||||||
|
|
||||||
|
GET /api/videofile.php?video_id=<id>&lifetime=8640000
|
||||||
|
-> [{"format":"_hq.mp4","video_url":"<base64+cyrylica>","is_default":1, ...}]
|
||||||
|
|
||||||
|
`video_url` to base64 w DWÓCH warstwach zaciemnienia: (1) wielkie łacińskie litery
|
||||||
|
podmienione na cyrylicę-homoglif (М->M, С->C, А->A, Е->E), (2) custom alfabet base64
|
||||||
|
gdzie `,`->`/`, `~`->`=`, `-`->`+`. Po odkręceniu obu + b64decode dostajemy get_file:
|
||||||
|
|
||||||
|
# stare wideo (shared txxx pool) — URL absolutny:
|
||||||
|
https://videotxxx.com/ext/get_file/9/<hash>/<floor>/<mid>/<mid>_hq.mp4/?d=..&br=..&ti=..
|
||||||
|
# nowe wideo (własna infra vjav, server N) — URL RELATYWNY (prepend https://vjav.com):
|
||||||
|
/get_file/3/<hash>/<floor>/<mid>/<mid>_hq.mp4/?d=..&br=..&ti=..
|
||||||
|
|
||||||
|
Dopisanie `&f=video.m3u8` -> 302 chain -> finalny HLS na *.ahcdn.com
|
||||||
|
(`key=..,end=..,limit=3` — time-bound, NIE IP-bound, `referer=none` na whiteliscie
|
||||||
|
-> portable, gra z residential IP telefonu bez sesji vjav).
|
||||||
|
|
||||||
|
Zwracamy get_file (`?f=video.m3u8`) jako type='m3u8' + `mobile_direct_ok` -> playback.py
|
||||||
|
owija w `/proxy/hls/<token>/play.m3u8` (passthrough manifestu ~1KB przez VPS, segmenty
|
||||||
|
direct z telefonu; patrz stream_proxy.proxy_hls_manifest). Media id w get_file (318577)
|
||||||
|
!= page id (5197) — dlatego MUSIMY przejść przez videofile.php. RE 2026-07-10.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import base64
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import re
|
||||||
|
|
||||||
|
from app.extractors import browser_get
|
||||||
|
from app.extractors._models import StreamSource
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
_BASE = "https://vjav.com"
|
||||||
|
_VIDEO_ID_RE = re.compile(r"/videos/(\d+)/")
|
||||||
|
|
||||||
|
# KVS video_url: base64 z cyrylica-homoglifami zamiast łacińskich liter (wielkie +
|
||||||
|
# część małych). Odkręcamy je z powrotem na łacinę przed b64decode.
|
||||||
|
_HOMOGLYPHS = str.maketrans(
|
||||||
|
{
|
||||||
|
"А": "A", "В": "B", "С": "C", "Е": "E", "Н": "H", "К": "K", "М": "M",
|
||||||
|
"О": "O", "Р": "P", "Т": "T", "Х": "X", "У": "Y",
|
||||||
|
"а": "a", "с": "c", "е": "e", "о": "o", "р": "p", "х": "x", "у": "y",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _decode_video_url(obfuscated: str) -> str | None:
|
||||||
|
# (1) cyrylica-homoglif -> łacina, (2) custom alfabet base64 -> standardowy.
|
||||||
|
clean = (
|
||||||
|
obfuscated.translate(_HOMOGLYPHS)
|
||||||
|
.replace(",", "/")
|
||||||
|
.replace("~", "=")
|
||||||
|
.replace("-", "+")
|
||||||
|
)
|
||||||
|
clean += "=" * (-len(clean) % 4) # padding do wielokrotności 4
|
||||||
|
try:
|
||||||
|
return base64.b64decode(clean).decode("utf-8", "ignore")
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _quality_label(fmt: str | None) -> str | None:
|
||||||
|
"""`_hq.mp4` -> 'HD', `_sd.mp4` -> 'SD'. Trailer (`_tr`) filtrowany osobno."""
|
||||||
|
if not fmt:
|
||||||
|
return None
|
||||||
|
tok = fmt.strip("_").replace(".mp4", "").lower()
|
||||||
|
return {"hq": "HD", "sd": "SD", "lq": "LOW", "hd": "HD"}.get(tok, tok.upper() or None)
|
||||||
|
|
||||||
|
|
||||||
|
def extract(page_url: str, *, timeout: float = 60.0) -> list[StreamSource] | None:
|
||||||
|
m = _VIDEO_ID_RE.search(page_url)
|
||||||
|
if not m:
|
||||||
|
log.info("vjav: brak video id w %s", page_url)
|
||||||
|
return None
|
||||||
|
vid = m.group(1)
|
||||||
|
api = f"{_BASE}/api/videofile.php?video_id={vid}&lifetime=8640000"
|
||||||
|
try:
|
||||||
|
res = browser_get(
|
||||||
|
api,
|
||||||
|
timeout=timeout,
|
||||||
|
headers={"Referer": page_url, "X-Requested-With": "XMLHttpRequest"},
|
||||||
|
)
|
||||||
|
body = res.text if hasattr(res, "text") else res
|
||||||
|
data = json.loads(body)
|
||||||
|
except Exception as e:
|
||||||
|
log.info("vjav: videofile.php fetch/parse fail %s: %s", api, e)
|
||||||
|
return None
|
||||||
|
if not isinstance(data, list):
|
||||||
|
return None
|
||||||
|
|
||||||
|
seen: set[str] = set()
|
||||||
|
out: list[StreamSource] = []
|
||||||
|
for entry in data:
|
||||||
|
if not isinstance(entry, dict):
|
||||||
|
continue
|
||||||
|
fmt = (entry.get("format") or "").strip()
|
||||||
|
# `_tr.mp4` = trailer preview, NIE pełne wideo — pomijamy.
|
||||||
|
if fmt.strip("_").replace(".mp4", "").lower() == "tr":
|
||||||
|
continue
|
||||||
|
vu = entry.get("video_url")
|
||||||
|
if not vu:
|
||||||
|
continue
|
||||||
|
get_file = _decode_video_url(vu)
|
||||||
|
if not get_file or "/get_file/" not in get_file:
|
||||||
|
continue
|
||||||
|
# Nowe wideo daje URL relatywny (/get_file/...) — prepend host vjav.
|
||||||
|
if get_file.startswith("/"):
|
||||||
|
get_file = _BASE + get_file
|
||||||
|
m3u8 = get_file + ("&" if "?" in get_file else "?") + "f=video.m3u8"
|
||||||
|
if m3u8 in seen:
|
||||||
|
continue
|
||||||
|
seen.add(m3u8)
|
||||||
|
out.append(
|
||||||
|
StreamSource(
|
||||||
|
link=m3u8,
|
||||||
|
type="m3u8",
|
||||||
|
quality=_quality_label(fmt),
|
||||||
|
referer=page_url,
|
||||||
|
# Finalny HLS (txxx.ahcdn.com) time-bound + referer=none whitelisted ->
|
||||||
|
# portable. mobile_direct_ok -> playback.py owija w /proxy/hls passthrough.
|
||||||
|
raw={"mobile_direct_ok": True},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if not out:
|
||||||
|
log.info("vjav: brak dekodowalnego video_url dla %s", page_url)
|
||||||
|
return None
|
||||||
|
return out
|
||||||
125
app/extractors/tubes/watchporn.py
Normal file
125
app/extractors/tubes/watchporn.py
Normal file
|
|
@ -0,0 +1,125 @@
|
||||||
|
"""watchporn.to — KVS engine direct stream extractor (re-enabled 2026-07-02).
|
||||||
|
|
||||||
|
Site przebudowany na KVS: detail page ma flashvars `video_url`/`video_alt_url*`,
|
||||||
|
każdy to `get_file/<srv>/<token>/<path>_<q>p.mp4/?v-acctoken=...`. Ten sam wzorzec co
|
||||||
|
porntrex. get_file 302 → finalny CDN url; resolvujemy same-session (token bywa
|
||||||
|
cookie/session-bound) i oddajemy finalny url per jakość. Token nie IP-bound
|
||||||
|
(cross-IP 206 z VPS zweryfikowane 2026-07-02) → mobile gra direct, zero proxy.
|
||||||
|
Wcześniejszy DoodStream-CAPTCHA zniknął.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import re
|
||||||
|
import time
|
||||||
|
|
||||||
|
from app.extractors._fetch import _DEFAULT_IMPERSONATE, _DEFAULT_UA, _HAS_CURL_CFFI, fetch_tube_html
|
||||||
|
from app.extractors._models import HosterDead, StreamSource
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
_BASE = "https://watchporn.to"
|
||||||
|
|
||||||
|
_DEAD_RE = re.compile(
|
||||||
|
r"this video (?:was|has been) deleted|video (?:was|has been) removed"
|
||||||
|
r"|no longer available|video is unavailable",
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
_URL_RE = re.compile(
|
||||||
|
r"(video(?:_alt)?_url\d*)\s*:\s*'(https?://[^']+/get_file/[^']+)'",
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
_TEXT_RE = re.compile(
|
||||||
|
r"(video(?:_alt)?_url\d*)_text\s*:\s*'([^']*)'",
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _quality_rank(label: str | None) -> int:
|
||||||
|
if not label:
|
||||||
|
return -1
|
||||||
|
m = re.search(r"(\d{3,4})\s*p", label, re.IGNORECASE)
|
||||||
|
return int(m.group(1)) if m else -1
|
||||||
|
|
||||||
|
|
||||||
|
def _resolve_get_file(session, get_file_url: str, timeout: float) -> str | None:
|
||||||
|
sep = "&" if "?" in get_file_url else "?"
|
||||||
|
url = f"{get_file_url}{sep}rnd={int(time.time() * 1000)}"
|
||||||
|
try:
|
||||||
|
r = session.get(
|
||||||
|
url,
|
||||||
|
timeout=timeout,
|
||||||
|
allow_redirects=True,
|
||||||
|
stream=True,
|
||||||
|
headers={"Referer": _BASE + "/", "Range": "bytes=0-1"},
|
||||||
|
)
|
||||||
|
final = str(r.url)
|
||||||
|
status = r.status_code
|
||||||
|
r.close()
|
||||||
|
except Exception as e:
|
||||||
|
log.info("watchporn: get_file resolve failed (%s): %s", get_file_url[:60], e)
|
||||||
|
return None
|
||||||
|
if status >= 400 or "/get_file/" in final:
|
||||||
|
log.info("watchporn: get_file resolve bad status=%s final=%s", status, final[:70])
|
||||||
|
return None
|
||||||
|
return final
|
||||||
|
|
||||||
|
|
||||||
|
def extract(page_url: str, *, timeout: float = 60.0) -> list[StreamSource] | None:
|
||||||
|
session = None
|
||||||
|
if _HAS_CURL_CFFI:
|
||||||
|
from curl_cffi import requests as _cf_requests
|
||||||
|
session = _cf_requests.Session(impersonate=_DEFAULT_IMPERSONATE)
|
||||||
|
try:
|
||||||
|
resp = session.get(
|
||||||
|
page_url,
|
||||||
|
headers={"User-Agent": _DEFAULT_UA, "Accept": "text/html,application/xhtml+xml"},
|
||||||
|
timeout=timeout,
|
||||||
|
allow_redirects=True,
|
||||||
|
)
|
||||||
|
html_text = resp.text if resp.status_code < 400 else ""
|
||||||
|
except Exception as e:
|
||||||
|
log.info("watchporn: page fetch failed %s: %s", page_url, e)
|
||||||
|
html_text = ""
|
||||||
|
if not html_text:
|
||||||
|
html_text = fetch_tube_html(page_url, timeout=timeout)
|
||||||
|
session = None
|
||||||
|
else:
|
||||||
|
html_text = fetch_tube_html(page_url, timeout=timeout)
|
||||||
|
|
||||||
|
if html_text and _DEAD_RE.search(html_text):
|
||||||
|
raise HosterDead(f"watchporn {page_url}: video deleted/removed")
|
||||||
|
|
||||||
|
quality_by_var: dict[str, str] = {}
|
||||||
|
for m in _TEXT_RE.finditer(html_text):
|
||||||
|
quality_by_var[m.group(1).lower()] = m.group(2).strip()
|
||||||
|
|
||||||
|
seen: set[str] = set()
|
||||||
|
result: list[StreamSource] = []
|
||||||
|
for m in _URL_RE.finditer(html_text):
|
||||||
|
var_name = m.group(1).lower()
|
||||||
|
url = m.group(2)
|
||||||
|
if url in seen:
|
||||||
|
continue
|
||||||
|
seen.add(url)
|
||||||
|
quality = quality_by_var.get(var_name)
|
||||||
|
final_link = url
|
||||||
|
if session is not None:
|
||||||
|
resolved = _resolve_get_file(session, url, timeout)
|
||||||
|
if resolved:
|
||||||
|
final_link = resolved
|
||||||
|
result.append(
|
||||||
|
StreamSource(
|
||||||
|
link=final_link,
|
||||||
|
type="mp4",
|
||||||
|
quality=quality or None,
|
||||||
|
referer=_BASE + "/",
|
||||||
|
raw={"mobile_direct_ok": True},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
if not result:
|
||||||
|
log.info("watchporn: no KVS video_url in flashvars on %s", page_url)
|
||||||
|
return None
|
||||||
|
result.sort(key=lambda s: _quality_rank(s.quality), reverse=True)
|
||||||
|
return result
|
||||||
87
app/extractors/tubes/xhamster.py
Normal file
87
app/extractors/tubes/xhamster.py
Normal file
|
|
@ -0,0 +1,87 @@
|
||||||
|
"""xhamster.com — natywny server-side HLS extractor.
|
||||||
|
|
||||||
|
2026-06-08: re-test Chrome DevTools + cross-IP NAPRAWIA założenie z `_vps_blocked_fallback`.
|
||||||
|
Wcześniej `xhamstercom` szedł przez WebView fallback (założenie: Cloudflare blokuje Hetzner
|
||||||
|
IP). Re-test pokazał:
|
||||||
|
1. VPS pobiera scene page (HTTP 200, BEZ Cloudflare challenge — blok się zdjął).
|
||||||
|
2. Master HLS URL jest w SSR HTML plain: `video-nss.xhcdn.com/<token>,<expiry>/media=hls4/
|
||||||
|
multi=.../...m3u8`. `<expiry>` to UNIX ts → token TIME-BOUND, nie IP-bound.
|
||||||
|
3. Cross-IP test (VPS Hetzner): master m3u8 → 200, wariant playlist → 200, segment .m4s
|
||||||
|
→ 206 video/mp4. Cały łańcuch PORTABLE — mobile gra HLS direct z residential IP,
|
||||||
|
zero VPS proxy bandwidth.
|
||||||
|
|
||||||
|
Dlatego resolvujemy SERVER-SIDE jak porntrex/freshporno: fetch page (curl_cffi chrome) →
|
||||||
|
wyłuskaj master m3u8 → oddaj jako type='m3u8' mobile_direct. ExoPlayer robi adaptive
|
||||||
|
multi-quality z jednego master URL.
|
||||||
|
|
||||||
|
NB `sources.standard.av1/h264` w HTML to ZASZYFROWANE hex-bloby (player deszyfruje w JS),
|
||||||
|
bezużyteczne server-side — dlatego bierzemy HLS, nie mp4.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import re
|
||||||
|
|
||||||
|
from app.extractors._fetch import _DEFAULT_IMPERSONATE, _DEFAULT_UA, _HAS_CURL_CFFI, fetch_tube_html
|
||||||
|
from app.extractors._models import HosterDead, StreamSource
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
_BASE = "https://xhamster.com"
|
||||||
|
|
||||||
|
# Master HLS na xhcdn (video-nss.xhcdn.com / fallback inne sub-domeny). JSON w HTML
|
||||||
|
# escape'uje slashe (`https:\/\/...`), więc unescape przed matchowaniem.
|
||||||
|
_M3U8_RE = re.compile(r"https://[a-z0-9.\-]*xhcdn\.com/[^\"'\\ ]+?\.m3u8", re.IGNORECASE)
|
||||||
|
|
||||||
|
# Markery skasowanej sceny (strona istnieje, ale bez wideo) → HosterDead.
|
||||||
|
_DEAD_MARKERS = (
|
||||||
|
"this video has been deleted",
|
||||||
|
"this video was deleted",
|
||||||
|
"video is no longer available",
|
||||||
|
"has been removed",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def extract(page_url: str, *, timeout: float = 60.0) -> list[StreamSource] | None:
|
||||||
|
html = ""
|
||||||
|
if _HAS_CURL_CFFI:
|
||||||
|
from curl_cffi import requests as _cf_requests
|
||||||
|
|
||||||
|
session = _cf_requests.Session(impersonate=_DEFAULT_IMPERSONATE)
|
||||||
|
try:
|
||||||
|
resp = session.get(
|
||||||
|
page_url,
|
||||||
|
headers={"User-Agent": _DEFAULT_UA, "Accept": "text/html,application/xhtml+xml"},
|
||||||
|
timeout=timeout,
|
||||||
|
allow_redirects=True,
|
||||||
|
)
|
||||||
|
html = resp.text if resp.status_code < 400 else ""
|
||||||
|
except Exception as e:
|
||||||
|
log.info("xhamster: page fetch failed %s: %s", page_url, e)
|
||||||
|
html = ""
|
||||||
|
if not html:
|
||||||
|
# fetch_tube_html podnosi TubePageError dla 404/410 (caller → dead_at).
|
||||||
|
html = fetch_tube_html(page_url, timeout=timeout)
|
||||||
|
|
||||||
|
# JSON-escaped slashe → plain, żeby regex złapał master URL.
|
||||||
|
unescaped = html.replace("\\/", "/")
|
||||||
|
m = _M3U8_RE.search(unescaped)
|
||||||
|
if not m:
|
||||||
|
low = unescaped.lower()
|
||||||
|
if any(marker in low for marker in _DEAD_MARKERS):
|
||||||
|
raise HosterDead(f"xhamster: scene deleted {page_url}")
|
||||||
|
log.info("xhamster: no HLS master URL on %s", page_url)
|
||||||
|
return None
|
||||||
|
|
||||||
|
master = m.group(0)
|
||||||
|
return [
|
||||||
|
StreamSource(
|
||||||
|
link=master,
|
||||||
|
type="m3u8",
|
||||||
|
quality=None, # HLS master = adaptive multi-quality (ExoPlayer wybiera)
|
||||||
|
referer=_BASE + "/",
|
||||||
|
# Master + warianty + segmenty są time-bound (nie IP/cookie-bound),
|
||||||
|
# zweryfikowane cross-IP 2026-06-08 → mobile gra direct, zero VPS proxy.
|
||||||
|
raw={"mobile_direct_ok": True},
|
||||||
|
)
|
||||||
|
]
|
||||||
17
app/extractors/tubes/yespornvip.py
Normal file
17
app/extractors/tubes/yespornvip.py
Normal file
|
|
@ -0,0 +1,17 @@
|
||||||
|
"""yesporn.vip — KVS (kt_player) direct stream extractor. Patrz app/extractors/tubes/_kvs.py.
|
||||||
|
|
||||||
|
VPS dociera do yesporn.vip (HTTP 200, odblokowane jak porntrex 2026-05-22) → resolvujemy
|
||||||
|
SERVER-SIDE (decode function/0 get_file + follow 302 → portable cdn5 url) zamiast WebView
|
||||||
|
fallback który pokazywał stronę + preroll-reklamę. Native, multi-quality, zero reklam.
|
||||||
|
Zweryfikowane 2026-05-31: prod 3×cdn5, emulator 1080p picker → native decode.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from app.extractors._models import StreamSource
|
||||||
|
from app.extractors.tubes import _kvs
|
||||||
|
|
||||||
|
_BASE = "https://yesporn.vip"
|
||||||
|
|
||||||
|
|
||||||
|
def extract(page_url: str, *, timeout: float = 60.0) -> list[StreamSource] | None:
|
||||||
|
return _kvs.resolve_kvs(page_url, base_url=_BASE, timeout=timeout)
|
||||||
105
app/ingest.py
105
app/ingest.py
|
|
@ -18,6 +18,7 @@ from __future__ import annotations
|
||||||
import hashlib
|
import hashlib
|
||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
|
import re
|
||||||
import uuid
|
import uuid
|
||||||
from collections.abc import Iterable
|
from collections.abc import Iterable
|
||||||
from datetime import UTC, datetime, timedelta
|
from datetime import UTC, datetime, timedelta
|
||||||
|
|
@ -26,19 +27,63 @@ from sqlalchemy import select
|
||||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app.config import get_settings
|
||||||
from app.connectors.base import BaseConnector, BaseMovieConnector, RawMovie, RawScene
|
from app.connectors.base import BaseConnector, BaseMovieConnector, RawMovie, RawScene
|
||||||
from app.db import session_scope
|
from app.db import session_scope
|
||||||
from app.models.external_record import EntityKind, ExternalRecord
|
from app.models.external_record import EntityKind, ExternalRecord
|
||||||
from app.models.ingest_run import IngestRun, IngestStatus
|
from app.models.ingest_run import IngestRun, IngestStatus
|
||||||
from app.models.source import Source, SourceKind
|
from app.models.source import Source, SourceKind
|
||||||
from app.normalize.movies import normalize_movie
|
from app.normalize.movies import normalize_movie
|
||||||
from app.normalize.scenes import normalize_scene
|
from app.normalize.scenes import NormalizedScene, normalize_scene
|
||||||
from app.resolve.movie_resolver import resolve_movie
|
from app.resolve.movie_resolver import resolve_movie
|
||||||
from app.resolve.scene_resolver import resolve_scene
|
from app.resolve.scene_resolver import resolve_scene
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
# Clip-store studia (ManyVids/IWantClips/Clips4Sale/...) — content twórców z paywalla.
|
||||||
|
# Darmowe tube'y go nie hostują, więc z canonical (TPDB/StashDB) wjeżdża jako permanentny
|
||||||
|
# orphan (56% ingestu canonical, ~860/dzień, ~550k w DB). Skipujemy resolve dla tych scen
|
||||||
|
# gdy źródło jest canonical — oszczędza ~połowę resolve-time i nie zaśmieca katalogu.
|
||||||
|
# NIE skipujemy dla tube'ów: tube scena z clip-store studiem MA playback, więc jest grywalna.
|
||||||
|
_CLIP_STORE_RE = re.compile(
|
||||||
|
r"^\s*(manyvids|i ?want ?clips|clips4sale|fancentro|loyalfans|onlyfans|fansly|modelhub)\b",
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def is_clip_store_studio(name: str | None) -> bool:
|
||||||
|
return bool(name) and _CLIP_STORE_RE.match(name) is not None
|
||||||
|
|
||||||
|
|
||||||
|
def _skip_clip_store_canonical(session: Session, *, source_id: uuid.UUID, studio_name: str | None) -> bool:
|
||||||
|
"""True gdy scena to clip-store content z canonical source → pomijamy resolve."""
|
||||||
|
if not getattr(get_settings(), "skip_clip_store", True):
|
||||||
|
return False
|
||||||
|
if not is_clip_store_studio(studio_name):
|
||||||
|
return False
|
||||||
|
src = session.get(Source, source_id)
|
||||||
|
return src is not None and src.kind in (SourceKind.tpdb, SourceKind.stashdb)
|
||||||
|
|
||||||
|
|
||||||
|
def _skip_short_tube_scene(session: Session, *, source_id: uuid.UUID, norm: NormalizedScene) -> bool:
|
||||||
|
"""True gdy scena ze scrapera/tube ma ZNANY duration < `min_ingest_duration_sec`
|
||||||
|
(trailer/teaser/preview — śmieć). Nieznany duration → NIE wycinamy (mogłaby być pełna
|
||||||
|
scena bez metadanych). Tylko scraper-source — canonical (TPDB/StashDB) zostawiamy.
|
||||||
|
porndoe/deep-crawl ciągną z głębi katalogu sporo trailerów <3min (2026-06-03)."""
|
||||||
|
floor = getattr(get_settings(), "min_ingest_duration_sec", 0)
|
||||||
|
if not floor:
|
||||||
|
return False
|
||||||
|
dur = norm.duration_sec
|
||||||
|
if dur is None:
|
||||||
|
ps_durs = [ps.duration_sec for ps in norm.playback_sources if ps.duration_sec]
|
||||||
|
dur = max(ps_durs) if ps_durs else None
|
||||||
|
if dur is None or dur >= floor:
|
||||||
|
return False
|
||||||
|
src = session.get(Source, source_id)
|
||||||
|
return src is not None and src.kind == SourceKind.scraper
|
||||||
|
|
||||||
|
|
||||||
def _canonical_json(payload: dict) -> bytes:
|
def _canonical_json(payload: dict) -> bytes:
|
||||||
return json.dumps(payload, sort_keys=True, separators=(",", ":"), default=str).encode()
|
return json.dumps(payload, sort_keys=True, separators=(",", ":"), default=str).encode()
|
||||||
|
|
||||||
|
|
@ -47,6 +92,34 @@ def _hash_raw(payload: dict) -> bytes:
|
||||||
return hashlib.sha256(_canonical_json(payload)).digest()
|
return hashlib.sha256(_canonical_json(payload)).digest()
|
||||||
|
|
||||||
|
|
||||||
|
def _has_nul(obj) -> bool:
|
||||||
|
"""Szybki rekurencyjny check na NUL byte (U+0000) w stringach. Short-circuituje,
|
||||||
|
więc dla czystych rekordów (99.99%) zero narzutu poza tanim spacerem."""
|
||||||
|
if isinstance(obj, str):
|
||||||
|
return "\x00" in obj
|
||||||
|
if isinstance(obj, dict):
|
||||||
|
return any(_has_nul(k) or _has_nul(v) for k, v in obj.items())
|
||||||
|
if isinstance(obj, list):
|
||||||
|
return any(_has_nul(v) for v in obj)
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _strip_nul(obj):
|
||||||
|
"""Rekurencyjnie usuwa NUL byte (U+0000) ze stringów (wartości i klucze dict).
|
||||||
|
|
||||||
|
Postgres JSONB/text NIE umie przechowywać `\\u0000` (DataError UntranslatableCharacter,
|
||||||
|
GOON-Z 2026-06-11: TPDB podał alias performera "Ramon\\u0000..."). Strip przed hash +
|
||||||
|
insert ORAZ przed normalize (aliasy → kolumny TEXT), żeby cała scena/film weszły do
|
||||||
|
ingestu zamiast się wywalić."""
|
||||||
|
if isinstance(obj, str):
|
||||||
|
return obj.replace("\x00", "") if "\x00" in obj else obj
|
||||||
|
if isinstance(obj, dict):
|
||||||
|
return {_strip_nul(k): _strip_nul(v) for k, v in obj.items()}
|
||||||
|
if isinstance(obj, list):
|
||||||
|
return [_strip_nul(v) for v in obj]
|
||||||
|
return obj
|
||||||
|
|
||||||
|
|
||||||
def get_or_create_source(
|
def get_or_create_source(
|
||||||
session: Session, *, kind: SourceKind, name: str, base_url: str | None = None
|
session: Session, *, kind: SourceKind, name: str, base_url: str | None = None
|
||||||
) -> Source:
|
) -> Source:
|
||||||
|
|
@ -194,8 +267,20 @@ def ingest_from_connector(
|
||||||
return counters
|
return counters
|
||||||
|
|
||||||
|
|
||||||
def _process_scene(*, source_id: uuid.UUID, raw_scene: RawScene, counters: dict[str, int]) -> None:
|
def _process_scene(
|
||||||
|
*,
|
||||||
|
source_id: uuid.UUID,
|
||||||
|
raw_scene: RawScene,
|
||||||
|
counters: dict[str, int],
|
||||||
|
backfill: bool = False,
|
||||||
|
) -> None:
|
||||||
payload = raw_scene.raw or raw_scene.model_dump(mode="json")
|
payload = raw_scene.raw or raw_scene.model_dump(mode="json")
|
||||||
|
if _has_nul(payload):
|
||||||
|
# Strip NUL z payloadu (→ external_records.raw JSONB) ORAZ ze structured fields
|
||||||
|
# (→ normalize_scene → kolumny TEXT performera/aliasów). Inaczej insert pada na
|
||||||
|
# UntranslatableCharacter i scena nigdy nie wchodzi (GOON-Z).
|
||||||
|
payload = _strip_nul(payload)
|
||||||
|
raw_scene = raw_scene.model_validate(_strip_nul(raw_scene.model_dump()))
|
||||||
raw_hash = _hash_raw(payload)
|
raw_hash = _hash_raw(payload)
|
||||||
now = datetime.now(UTC)
|
now = datetime.now(UTC)
|
||||||
|
|
||||||
|
|
@ -214,7 +299,18 @@ def _process_scene(*, source_id: uuid.UUID, raw_scene: RawScene, counters: dict[
|
||||||
return
|
return
|
||||||
|
|
||||||
norm = normalize_scene(raw_scene)
|
norm = normalize_scene(raw_scene)
|
||||||
result = resolve_scene(session, norm=norm, source_id=source_id)
|
|
||||||
|
if _skip_clip_store_canonical(
|
||||||
|
session, source_id=source_id, studio_name=norm.studio.name if norm.studio else None
|
||||||
|
):
|
||||||
|
counters["skipped"] += 1
|
||||||
|
return
|
||||||
|
|
||||||
|
if _skip_short_tube_scene(session, source_id=source_id, norm=norm):
|
||||||
|
counters["skipped"] += 1
|
||||||
|
return
|
||||||
|
|
||||||
|
result = resolve_scene(session, norm=norm, source_id=source_id, backfill=backfill)
|
||||||
|
|
||||||
if result.was_created:
|
if result.was_created:
|
||||||
counters["new"] += 1
|
counters["new"] += 1
|
||||||
|
|
@ -307,6 +403,9 @@ def ingest_movies_from_connector(
|
||||||
|
|
||||||
def _process_movie(*, source_id: uuid.UUID, raw_movie: RawMovie, counters: dict[str, int]) -> None:
|
def _process_movie(*, source_id: uuid.UUID, raw_movie: RawMovie, counters: dict[str, int]) -> None:
|
||||||
payload = raw_movie.raw or raw_movie.model_dump(mode="json")
|
payload = raw_movie.raw or raw_movie.model_dump(mode="json")
|
||||||
|
if _has_nul(payload):
|
||||||
|
payload = _strip_nul(payload)
|
||||||
|
raw_movie = raw_movie.model_validate(_strip_nul(raw_movie.model_dump()))
|
||||||
raw_hash = _hash_raw(payload)
|
raw_hash = _hash_raw(payload)
|
||||||
now = datetime.now(UTC)
|
now = datetime.now(UTC)
|
||||||
|
|
||||||
|
|
|
||||||
12
app/main.py
12
app/main.py
|
|
@ -14,11 +14,16 @@ from app.api.blacklist import router as blacklist_router
|
||||||
from app.api.bug_reports import router as bug_reports_router
|
from app.api.bug_reports import router as bug_reports_router
|
||||||
from app.api.expo_updates import router as expo_updates_router
|
from app.api.expo_updates import router as expo_updates_router
|
||||||
from app.api.favorites import router as favorites_router
|
from app.api.favorites import router as favorites_router
|
||||||
|
from app.api.me import router as me_router
|
||||||
from app.api.movies import router as movies_router
|
from app.api.movies import router as movies_router
|
||||||
from app.api.playback import movies_router as movies_playback_router
|
from app.api.playback import movies_router as movies_playback_router
|
||||||
from app.api.playback import router as playback_router
|
from app.api.playback import router as playback_router
|
||||||
|
from app.api.playback_events import router as playback_events_router
|
||||||
|
from app.api.saved_searches import router as saved_searches_router
|
||||||
from app.api.scene_favorites import router as scene_favorites_router
|
from app.api.scene_favorites import router as scene_favorites_router
|
||||||
from app.api.scenes import router as scenes_router
|
from app.api.scenes import router as scenes_router
|
||||||
|
from app.api.seo import router as seo_router
|
||||||
|
from app.api.sources import router as sources_router
|
||||||
from app.api.stream_proxy import router as stream_proxy_router
|
from app.api.stream_proxy import router as stream_proxy_router
|
||||||
from app.api.taxonomies import router as taxonomies_router
|
from app.api.taxonomies import router as taxonomies_router
|
||||||
from app.api.watch import router as watch_router
|
from app.api.watch import router as watch_router
|
||||||
|
|
@ -66,8 +71,10 @@ if _settings.sentry_dsn:
|
||||||
|
|
||||||
app = FastAPI(title="goon", version="0.1.8")
|
app = FastAPI(title="goon", version="0.1.8")
|
||||||
app.include_router(scenes_router)
|
app.include_router(scenes_router)
|
||||||
|
app.include_router(sources_router)
|
||||||
app.include_router(movies_router)
|
app.include_router(movies_router)
|
||||||
app.include_router(playback_router)
|
app.include_router(playback_router)
|
||||||
|
app.include_router(playback_events_router)
|
||||||
app.include_router(movies_playback_router)
|
app.include_router(movies_playback_router)
|
||||||
app.include_router(scene_favorites_router)
|
app.include_router(scene_favorites_router)
|
||||||
app.include_router(stream_proxy_router)
|
app.include_router(stream_proxy_router)
|
||||||
|
|
@ -75,10 +82,13 @@ app.include_router(taxonomies_router)
|
||||||
app.include_router(favorites_router)
|
app.include_router(favorites_router)
|
||||||
app.include_router(blacklist_router)
|
app.include_router(blacklist_router)
|
||||||
app.include_router(bug_reports_router)
|
app.include_router(bug_reports_router)
|
||||||
|
app.include_router(saved_searches_router)
|
||||||
app.include_router(expo_updates_router)
|
app.include_router(expo_updates_router)
|
||||||
app.include_router(watch_router)
|
app.include_router(watch_router)
|
||||||
|
app.include_router(me_router)
|
||||||
app.include_router(admin_router)
|
app.include_router(admin_router)
|
||||||
app.include_router(admin_html_router)
|
app.include_router(admin_html_router)
|
||||||
|
app.include_router(seo_router)
|
||||||
mount_static(app)
|
mount_static(app)
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -111,7 +121,7 @@ def version() -> dict[str, str | None]:
|
||||||
# mobile sklei z baseUrl.
|
# mobile sklei z baseUrl.
|
||||||
public_url = os.environ.get("BACKEND_PUBLIC_URL", "").rstrip("/")
|
public_url = os.environ.get("BACKEND_PUBLIC_URL", "").rstrip("/")
|
||||||
apk_url = f"{public_url}/static/app-release.apk" if public_url else "/static/app-release.apk"
|
apk_url = f"{public_url}/static/app-release.apk" if public_url else "/static/app-release.apk"
|
||||||
return {"version": "0.1.8", "apk_url": apk_url}
|
return {"version": "0.2.1", "apk_url": apk_url}
|
||||||
|
|
||||||
|
|
||||||
@app.get("/readyz")
|
@app.get("/readyz")
|
||||||
|
|
|
||||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue