diff --git a/app/api/stream_proxy.py b/app/api/stream_proxy.py index 4d7ce6f..3683e7a 100644 --- a/app/api/stream_proxy.py +++ b/app/api/stream_proxy.py @@ -267,6 +267,12 @@ def _build_headers(referer: str | None) -> dict[str, str]: "User-Agent": DEFAULT_UA, "Accept": "*/*", "Accept-Language": "en-US,en;q=0.9", + # Bez tego httpx prosi o gzip, a `proxy_stream` przepuszcza cialo surowe (aiter_raw) + # i zdejmuje Content-Encoding (HOP_BY_HOP): odtwarzacz dostawal skompresowane bajty + # manifestu bez naglowka i konczyl na "Input does not start with the #EXTM3U header" + # (xtremestream przez Cloudflare, 189 zdarzen w 7 dni do 2026-10-14). Strumienie + # wideo i tak sie nie kompresuja, wiec identity nic nie kosztuje. + "Accept-Encoding": "identity", } if referer: h["Referer"] = _ascii_safe_url(referer) @@ -785,6 +791,32 @@ async def proxy_hls_manifest(token: str, _basename: str) -> Response: # Hoster-producent: manifest nie ma własnego URL-a (loadvid oddaje jego TREŚĆ # dopiero na POST /videos/resolve-token), więc zamiast GET-a wołamy producenta. producer = payload.get("mp") + if producer == "xtremestream": + # Manifest ma segmenty AES-128 i klucz pod WZGLEDNYM `key.bin` (ten sam token, ta + # sama trasa), wiec nie absolutyzujemy: segmenty juz sa pelnymi adresami CDN, a klucz + # telefon pobiera spod `/proxy/hls//key.bin`. Szczegoly: hosters/xtremestream.py. + import anyio + from app.extractors.hosters import xtremestream + + if _basename.endswith("key.bin"): + key = await anyio.to_thread.run_sync(functools.partial(xtremestream.fetch_key, target)) + if not key: + raise HTTPException(status_code=502, detail="key fetch failed") + return Response( + content=key, + media_type="application/octet-stream", + headers=_with_cast_cors({"Cache-Control": "no-store"}), + ) + manifest = await anyio.to_thread.run_sync( + functools.partial(xtremestream.build_manifest, target, key_uri="key.bin") + ) + if manifest is None: + raise HTTPException(status_code=502, detail="manifest producer failed") + return Response( + content=manifest, + media_type="application/vnd.apple.mpegurl", + headers=_with_cast_cors({"Cache-Control": "no-store"}), + ) if producer: manifest = await _produce_manifest(producer, target) if manifest is None: @@ -871,7 +903,10 @@ async def proxy_stream( # Powód: pornhat `get_file/.../.mp4/` 302 → CDN m3u8 manifest mimo `.mp4` # w path. Bez content-type check proxy traktuje jako binary, mobile dostaje # m3u8 z RAW CDN URLs (IP-bound do VPS) → "no extractors" w ExoPlayer. - path_suggests_m3u8 = path_lower.endswith(".m3u8") + # `_basename` to `play.m3u8` nadane w `_proxify_link` z typu zrodla, wiec manifest bez + # `.m3u8` w sciezce upstreamu (xtremestream `xs1.php?data=`) tez trafia do przepisania, + # zamiast leciec dalej jako plik z absolutnymi adresami segmentow. + path_suggests_m3u8 = path_lower.endswith(".m3u8") or _basename.lower().endswith(".m3u8") client = httpx.AsyncClient(timeout=timeout, follow_redirects=True) try: diff --git a/app/extractors/hosters/xtremestream.py b/app/extractors/hosters/xtremestream.py new file mode 100644 index 0000000..c871bbb --- /dev/null +++ b/app/extractors/hosters/xtremestream.py @@ -0,0 +1,153 @@ +"""xtremestream (player perverzija): manifest HLS z segmentami AES-128 skladany po stronie backendu. + +Od ok. 2026-10-12 player (`ftk_engine.js`, zaciemniony, z blokada devtools) nie gra juz +z `xs1.php`: segmenty pod starymi adresami `/cdn/down///.html` oddaja 404, +a prawdziwe leza pod `/cdn/down//_enc/.html` i sa zaszyfrowane AES-128-CBC, +IV = numer segmentu (16 bajtow big-endian), dopelnienie PKCS7, czyli dokladnie tak jak +standardowe `#EXT-X-KEY:METHOD=AES-128` bez atrybutu IV (sprawdzone 2026-10-14: openssl +z kluczem i IV=0 daje poprawny MPEG-TS, ffprobe widzi h264 i aac). + +Klucz (16 B) oddaje `proto_api.php?data=&op=key&vt=` z naglowkiem `X-XS-P: 1`. +`vt` to token ze strony playera (`XSFTK.boot(id, vt, player)`), zwiazany z IP i UA +pobierajacego (ten sam vt z innego IP = 403 `forbidden-vt`), ale klucz jest jeden na film, +niezalezny od vt (sprawdzone z dwoch IP). Segmenty i stary manifest `xs1.php` (nadal +serwowany, z dobrymi dlugosciami i rotacja hostow cdn[i % n]) nie sa zwiazane z IP. + +Dlatego telefon nic nie musi robic: backend bierze `xs1.php`, przepisuje sciezki na `_enc`, +dokleja `#EXT-X-KEY` z URI wzglednym `key.bin`, a `/proxy/hls//key.bin` oddaje klucz +pobrany na wlasny vt. Telefon ciagnie segmenty prosto z CDN (przez VPS leci ~50 kB manifestu +i 16 B klucza). Czesc katalogu (starszy storage) ma nadal zwykle segmenty: wtedy manifest +idzie bez klucza, tak jak dotad. +""" +from __future__ import annotations + +import logging +import re +import time +from urllib.parse import parse_qs, quote, urlparse + +import httpx + +log = logging.getLogger(__name__) + +_UA = ( + "Mozilla/5.0 (Linux; Android 14; Pixel 7) AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/128.0.0.0 Mobile Safari/537.36" +) +_PLAYER_RE = re.compile( + r"https?://(?P[a-z0-9\-]+\.xtremestream\.[a-z]+)/player/index\.php\?data=(?P[0-9a-f]{16,64})", + re.IGNORECASE, +) +_BOOT_RE = re.compile(r'XSFTK\.boot\("[0-9a-f]+",\s*"([^"]+)"') +# `https://pervl7.xspcdn01.click/cdn/down//720/7200.html` -> grupy: prefiks, jakosc, numer +_SEG_RE = re.compile(r"^(https?://[^/]+/cdn/down/[0-9a-f]+/)(\d+p?)/\2(\d+)\.html\s*$") +_DEFAULT_QUALITY = "720" +_KEY_TTL_SEC = 6 * 3600 +_key_cache: dict[str, tuple[bytes, float]] = {} + + +def parse_player(player_url: str) -> tuple[str, str] | None: + m = _PLAYER_RE.search(player_url) + return (m.group("host"), m.group("id")) if m else None + + +def quality_of(player_url: str) -> str: + q = parse_qs(urlparse(player_url).query).get("q", [_DEFAULT_QUALITY])[0] + return q if re.fullmatch(r"\d{3,4}", q) else _DEFAULT_QUALITY + + +def _to_enc(line: str) -> str: + m = _SEG_RE.match(line) + if not m: + return line + return f"{m.group(1)}{m.group(2)}_enc/{m.group(2)}{m.group(3)}.html" + + +def _reachable(url: str, timeout: float) -> bool: + try: + r = httpx.get(url, headers={"User-Agent": _UA, "Range": "bytes=0-0"}, timeout=timeout) + return r.status_code in (200, 206) + except httpx.HTTPError: + return False + + +def fetch_key(player_url: str, *, timeout: float = 20.0) -> bytes | None: + """16-bajtowy klucz AES filmu, z pamiecia podreczna na 6 h (klucz nie zalezy od vt).""" + parsed = parse_player(player_url) + if not parsed: + return None + host, vid = parsed + cached = _key_cache.get(vid) + if cached and time.time() - cached[1] < _KEY_TTL_SEC: + return cached[0] + index_url = f"https://{host}/player/index.php?data={vid}" + try: + page = httpx.get( + index_url, + headers={"User-Agent": _UA, "Referer": "https://tube.perverzija.com/"}, + timeout=timeout, + follow_redirects=True, + ) + m = _BOOT_RE.search(page.text) + if not m: + log.info("xtremestream: brak tokenu vt na %s (http %s)", index_url, page.status_code) + return None + vt = m.group(1) + # UA musi byc ten sam co przy stronie playera, vt jest pod niego liczony. + k = httpx.get( + f"https://{host}/player/proto_api.php?data={vid}&op=key&vt={quote(vt, safe='')}", + headers={"User-Agent": _UA, "X-XS-P": "1", "Referer": index_url}, + timeout=timeout, + ) + except httpx.HTTPError as e: + log.info("xtremestream: pobranie klucza padlo dla %s: %s", vid, e) + return None + if k.status_code != 200 or len(k.content) != 16: + log.info("xtremestream: klucz dla %s http %s, %d B", vid, k.status_code, len(k.content)) + return None + _key_cache[vid] = (k.content, time.time()) + return k.content + + +def build_manifest(player_url: str, *, key_uri: str = "key.bin", timeout: float = 20.0) -> str | None: + """Manifest HLS do odegrania przez telefon: segmenty `_enc` z CDN i klucz pod `key_uri` + (wzgledny wobec adresu manifestu). None, gdy xs1.php nie oddaje manifestu albo zaden + wariant segmentow (zaszyfrowany ani zwykly) nie odpowiada.""" + parsed = parse_player(player_url) + if not parsed: + return None + host, vid = parsed + q = quality_of(player_url) + index_url = f"https://{host}/player/index.php?data={vid}" + try: + m = httpx.get( + f"https://{host}/player/xs1.php?data={vid}&q={q}", + headers={"User-Agent": _UA, "Referer": index_url}, + timeout=timeout, + ) + except httpx.HTTPError as e: + log.info("xtremestream: xs1.php padlo dla %s: %s", vid, e) + return None + text = m.text + if m.status_code != 200 or not text.lstrip(" \t\r\n").startswith("#EXTM3U"): + log.info("xtremestream: xs1.php dla %s http %s, nie-manifest (%d B)", vid, m.status_code, len(m.content)) + return None + lines = text.splitlines() + segments = [l for l in lines if l.startswith("http")] + if not segments: + return None + first_enc = _to_enc(segments[0]) + if first_enc == segments[0] or not _reachable(first_enc, timeout): + # Starszy storage: zwykle segmenty, manifest bez zmian. Jesli i te nie odpowiadaja, + # film jest martwy po stronie hostera. + return text if _reachable(segments[0], timeout) else None + out: list[str] = [] + key_done = False + for line in lines: + out.append(_to_enc(line) if line.startswith("http") else line) + if not key_done and line.startswith("#EXT-X-MEDIA-SEQUENCE"): + out.append(f'#EXT-X-KEY:METHOD=AES-128,URI="{key_uri}"') + key_done = True + if not key_done: + out.insert(1, f'#EXT-X-KEY:METHOD=AES-128,URI="{key_uri}"') + return "\n".join(out) + "\n" diff --git a/app/extractors/tubes/perverzija.py b/app/extractors/tubes/perverzija.py index cc8fbdb..f13f326 100644 --- a/app/extractors/tubes/perverzija.py +++ b/app/extractors/tubes/perverzija.py @@ -68,11 +68,16 @@ def extract(page_url: str, *, timeout: float = 60.0) -> list[StreamSource] | Non host, vid = m.group("host"), m.group("id") player_url = f"https://{host}/player/index.php?data={vid}" + # Od 2026-10-12 segmenty sa szyfrowane (AES-128, klucz spod tokenu zwiazanego z IP), + # wiec manifest sklada backend w `/proxy/hls` (producent `xtremestream`, patrz + # extractors/hosters/xtremestream.py). Link to adres playera z jakoscia w zapytaniu; + # producent bierze z niego id i `q`. `mobile_direct_ok` zostaje: segmenty NIE sa + # IP-bound (test z dwoch IP 2026-10-14), przez VPS leci manifest i 16 B klucza. out: list[StreamSource] = [] for q in _QUALITIES: out.append( StreamSource( - link=f"https://{host}/player/xs1.php?data={vid}&q={q}", + link=f"{player_url}&q={q}", quality=f"{q}p", type="m3u8", referer=player_url, @@ -80,11 +85,7 @@ def extract(page_url: str, *, timeout: float = 60.0) -> list[StreamSource] | Non "extractor": "perverzija", "player": player_url, "q": q, - # Segmenty NIE są IP-bound i nie potrzebują Referera: segment pobrany - # z domowego łącza po manifeście wziętym na VPS oddał 200 i 76 kB - # zaczynające się od 0x47 (sync MPEG-TS), test 2026-08-20. Dzięki temu - # przez VPS idzie sam manifest (~16 kB) przez /proxy/hls, a segmenty - # telefon ciągnie prosto z xspcdn. Bez tej flagi całe wideo szło proxy. + "manifest_producer": "xtremestream", "mobile_direct_ok": True, }, ) diff --git a/app/scheduler/ingest_watchdog.py b/app/scheduler/ingest_watchdog.py index e513c68..fc24c69 100644 --- a/app/scheduler/ingest_watchdog.py +++ b/app/scheduler/ingest_watchdog.py @@ -68,6 +68,14 @@ _BROWSE_MAX_AGE_OVERRIDE_H: dict[str, int] = { "hentaimama": 240, } +# Zrodla swiadomie zamrozone: alarm nic nie wniesie, dopoki ktos nie zmieni stanu rzeczy. +# porndoe (2026-10-14): kazdy nowy /watch/ z sitemapy przekierowuje na /videos z kazdego IP +# (proxy PL i UK, z ciastkami sesji i po zdjeciu bramki wieku), darmowy katalog stoi na +# 11.09; stare 64 tys. scen gra. Kontrola reczna co jakis czas, nie co 6 h w Sentry. +_FROZEN_BROWSE: dict[str, str] = { + "porndoecom": "2026-10-14 nowe sceny przekierowuja na /videos z kazdego IP", +} + # Prog z kadencji: mediana odstepu (w dobach) miedzy dniami z nowymi pozycjami w ostatnich # 60 dniach, razy 3, nie mniej niz prog klasy i nie wiecej niz 14 dni. Zrodlo codzienne # zostaje przy 48 h, tygodniowe dostaje ok. 14 dni zamiast alarmu co dobe (2026-10-14). @@ -137,6 +145,7 @@ def run_ingest_freshness_watchdog( [ (f"tube:{t}", "playback_sources", _BROWSE_MAX_AGE_OVERRIDE_H.get(t, max_age_hours), "browse") for t in sorted(browse_tags) + if t not in _FROZEN_BROWSE ] + [(f"tube:{t}", "playback_sources", search_max_age_hours, "search") for t in sorted(search_tags)] diff --git a/scripts/porntrex_duration_walk.py b/scripts/porntrex_duration_walk.py new file mode 100644 index 0000000..02eaaa3 --- /dev/null +++ b/scripts/porntrex_duration_walk.py @@ -0,0 +1,176 @@ +"""porntrex: dolicza długość scenom bez niej z listingu `/latest-updates/N/`. + +Strona szczegółów porntrex nie podaje własnej długości (tylko długości filmów z paska +obok), więc backfill ze strony (`duration_backfill.py`) tu nie działa. Kafelek listingu +ją pokazuje (`class="durations"`), 85 kafelków na stronę, katalog ma ok. 8300 stron +(strona 15000 daje 404). Przebieg idzie po listingu od najnowszych i dla każdego kafelka +ustawia `scenes.duration_sec`, jeśli jest NULL. Nic nie nadpisuje. + +Adresy: ścieżka search zapisuje `https://www.porntrex.com/video//`, browse +`https://porntrex.com/video//` (bez www), dlatego dopasowanie po obu wariantach. + +Uruchomienie (długi przebieg, nie przez `exec`, patrz pamięć o restartach workera): + docker compose run -d --rm worker python3 scripts/porntrex_duration_walk.py --start 1 + docker compose run --rm worker python3 scripts/porntrex_duration_walk.py --start 1 --end 20 --dry-run +Postęp w `--state-file` (domyślnie /tmp/porntrex_duration_walk.json), wznowienie od +zapisanej strony przy braku `--start`. +""" +from __future__ import annotations + +import argparse +import json +import logging +import sys +import time + +from sqlalchemy import text + +from app.connectors.direct_scrapers.porntrex_browse import _A_RE, _DUR_RE, _parse_duration +from app.db import session_scope +from app.extractors import browser_get + +log = logging.getLogger("porntrex_duration_walk") +_LISTING = "https://www.porntrex.com/latest-updates/{page}/" +_ORIGIN = "tube:porntrexcom" + + +def parse_cards(html: str) -> dict[str, int]: + """Kafelki listingu → {url bez www i bez końcowego '/': sekundy}.""" + out: dict[str, int] = {} + anchors = list(_A_RE.finditer(html)) + for idx, m in enumerate(anchors): + url = m.group("url").replace("://www.", "://").rstrip("/") + if url in out: + continue + end = anchors[idx + 1].start() if idx + 1 < len(anchors) else m.end() + 700 + dm = _DUR_RE.search(html[m.start():end]) + sec = _parse_duration(dm.group(1) if dm else None) + if sec: + out[url] = sec + return out + + +def variants(url: str) -> list[str]: + bare = url.replace("://www.", "://").rstrip("/") + www = bare.replace("https://", "https://www.", 1) + return [bare, bare + "/", www, www + "/"] + + +def apply(cards: dict[str, int], *, dry_run: bool) -> dict[str, int]: + """Zwraca liczniki: matched (źródła w bazie), filled (ustawione), had (już miały).""" + counters = {"matched": 0, "filled": 0, "had": 0} + if not cards: + return counters + lookup: dict[str, int] = {} + for url, sec in cards.items(): + for v in variants(url): + lookup[v] = sec + with session_scope() as s: + rows = s.execute( + text( + """ + SELECT ps.page_url, ps.id AS ps_id, sc.id AS scene_id, sc.duration_sec + FROM playback_sources ps + JOIN scenes sc ON sc.id = ps.scene_id + WHERE ps.origin = :origin AND ps.page_url = ANY(:urls) + """ + ), + {"origin": _ORIGIN, "urls": list(lookup)}, + ).fetchall() + for page_url, ps_id, scene_id, have in rows: + counters["matched"] += 1 + sec = lookup.get(page_url) + if not sec: + continue + if have: + counters["had"] += 1 + continue + counters["filled"] += 1 + if dry_run: + continue + s.execute( + text("UPDATE scenes SET duration_sec = :d WHERE id = :id AND duration_sec IS NULL"), + {"d": sec, "id": scene_id}, + ) + s.execute( + text( + "UPDATE playback_sources SET duration_sec = :d " + " WHERE id = :id AND duration_sec IS NULL" + ), + {"d": sec, "id": ps_id}, + ) + return counters + + +def _fetch(page: int) -> tuple[int, str]: + res = browser_get(_LISTING.format(page=page), timeout=45) + status = int(getattr(res, "status_code", 200) or 200) + html = res.text if hasattr(res, "text") else str(res) + return status, html + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--start", type=int, default=None, help="pierwsza strona (domyślnie ze state-file albo 1)") + ap.add_argument("--end", type=int, default=20000, help="ostatnia strona (przebieg i tak staje na 404)") + ap.add_argument("--pace", type=float, default=1.0, help="odstęp między stronami w sekundach") + ap.add_argument("--dry-run", action="store_true") + ap.add_argument("--state-file", default="/tmp/porntrex_duration_walk.json") + args = ap.parse_args() + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") + + start = args.start + if start is None: + try: + with open(args.state_file, encoding="utf-8") as f: + start = int(json.load(f).get("next_page", 1)) + except (OSError, ValueError): + start = 1 + + total = {"pages": 0, "cards": 0, "matched": 0, "filled": 0, "had": 0} + empty_streak = 0 + t0 = time.time() + for page in range(start, args.end + 1): + try: + status, html = _fetch(page) + except Exception as e: + log.warning("strona %d: błąd pobrania %s, czekam 30 s", page, e) + time.sleep(30) + try: + status, html = _fetch(page) + except Exception as e2: + log.error("strona %d: drugi błąd %s, przerywam", page, e2) + break + cards = parse_cards(html) if status < 400 else {} + if not cards: + empty_streak += 1 + log.info("strona %d: http %s, brak kafelków (%d z rzędu)", page, status, empty_streak) + if empty_streak >= 3: + log.info("koniec listingu na stronie %d", page) + break + time.sleep(args.pace) + continue + empty_streak = 0 + c = apply(cards, dry_run=args.dry_run) + total["pages"] += 1 + total["cards"] += len(cards) + for k in ("matched", "filled", "had"): + total[k] += c[k] + log.info( + "strona %d: kafelków %d, w bazie %d, doliczone %d, miały %d%s", + page, len(cards), c["matched"], c["filled"], c["had"], " (dry-run)" if args.dry_run else "", + ) + if not args.dry_run: + try: + with open(args.state_file, "w", encoding="utf-8") as f: + json.dump({"next_page": page + 1, "total": total, "ts": time.time()}, f) + except OSError: + pass + time.sleep(args.pace) + log.info("razem: %s w %.0f s", total, time.time() - t0) + print(json.dumps(total)) + return 0 + + +if __name__ == "__main__": + sys.exit(main())