From 671ba97b5250c0b1fae9cf57a14a05c9f7857d1b Mon Sep 17 00:00:00 2001 From: goon-foss Date: Thu, 15 Oct 2026 12:16:24 +0200 Subject: [PATCH] fix(perverzija): strona i listing przez proxy przy 403 Cloudflare Od 2026-10-15 ok. 04:31 UTC Cloudflare oddaje 403 na IPv4 VPS-a (IPv6 200). Ekstraktor nie patrzyl na status, API dawalo 503 i perverzija nie grala nikomu; listing zwracal [] jak koniec katalogu, a uzupelnianie dlugosci palilo proby. HTML przez Bright Data tylko przy 403, player bez zmian. Co-Authored-By: Claude Opus 5.5 --- app/connectors/direct_scrapers/perverzija.py | 10 +++++--- app/extractors/tubes/perverzija.py | 27 +++++++++++++++++++- app/scheduler/duration_backfill.py | 9 ++++++- 3 files changed, 41 insertions(+), 5 deletions(-) diff --git a/app/connectors/direct_scrapers/perverzija.py b/app/connectors/direct_scrapers/perverzija.py index 6c46768..e831ff6 100644 --- a/app/connectors/direct_scrapers/perverzija.py +++ b/app/connectors/direct_scrapers/perverzija.py @@ -34,7 +34,7 @@ from app.connectors.direct_scrapers._browse_base import ( BaseBrowseScraper, compute_thumbnail_phash, ) -from app.extractors import browser_get +from app.extractors.tubes.perverzija import fetch_perverzija from app.normalize.text import slugify log = logging.getLogger(__name__) @@ -68,13 +68,17 @@ class PerverzijaScraper(BaseBrowseScraper): def crawl_page(self, page: int) -> list[RawScene] | None: url = self._listing_url(page) try: - res = browser_get(url, timeout=self._timeout) + res = fetch_perverzija(url, timeout=self._timeout) except Exception as e: log.warning("perverzija REST fetch failed (page %d): %s", page, e) return None # WP zwraca 400 (rest_post_invalid_page_number) za ostatnią stroną → exhausted. - if res.status_code != 200: + # Inny kod (403 Cloudflare, 5xx) to awaria, nie koniec katalogu. + if res.status_code == 400: return [] + if res.status_code != 200: + log.warning("perverzija REST HTTP %s (page %d)", res.status_code, page) + return None try: posts = json.loads(res.text) except (json.JSONDecodeError, ValueError): diff --git a/app/extractors/tubes/perverzija.py b/app/extractors/tubes/perverzija.py index f13f326..3608af2 100644 --- a/app/extractors/tubes/perverzija.py +++ b/app/extractors/tubes/perverzija.py @@ -26,11 +26,32 @@ from __future__ import annotations import logging import re +from app.config import get_settings from app.extractors._models import StreamSource from app.extractors import browser_get log = logging.getLogger(__name__) + +def fetch_perverzija(url: str, *, timeout: float): + """GET strony perverzija, przy blokadzie Cloudflare przez Bright Data. + + 2026-10-15 ok. 04:31 UTC Cloudflare zaczal odpowiadac 403 z challenge'em na IPv4 VPS-a + (IPv6 dalej 200), dla stron scen i dla REST-a listingu. Ekstraktor nie patrzyl na status, + szedl w zapas embed-iframe, ten tez dostawal 403, API oddawalo 503 i perverzija + przestala grac wszystkim (zgloszenie 1c662356). Proxy (ryczalt, ISP) tylko dla HTML-a, + player xtremestream i segmenty ida jak dotad. + """ + res = browser_get(url, timeout=timeout) + if res.status_code != 403: + return res + proxy = get_settings().brightdata_proxy_url + if not proxy: + log.warning("perverzija: 403 (Cloudflare) na %s, brak proxy", url) + return res + log.info("perverzija: 403 na %s, ponawiam przez proxy", url) + return browser_get(url, timeout=timeout, proxy=proxy) + # iframe playera: https://pervl4.xtremestream.xyz/player/index.php?data=<32 hex> _PLAYER_IFRAME_RE = re.compile( r"https?://(?P[a-z0-9\-]+\.xtremestream\.[a-z]+)/player/index\.php\?data=(?P[0-9a-f]{16,64})", @@ -44,11 +65,15 @@ _QUALITIES = ("720", "480") def extract(page_url: str, *, timeout: float = 60.0) -> list[StreamSource] | None: try: - res = browser_get(page_url, timeout=timeout) + res = fetch_perverzija(page_url, timeout=timeout) html = res.text if hasattr(res, "text") else str(res) except Exception as e: log.info("perverzija page fetch failed %s: %s", page_url, e) return None + if getattr(res, "status_code", 200) == 403: + # Zapas embed-iframe pobiera strone jeszcze raz bez proxy i tez dostaje 403. + log.warning("perverzija: strona %s nadal 403", page_url) + return None m = _PLAYER_IFRAME_RE.search(html) if not m: diff --git a/app/scheduler/duration_backfill.py b/app/scheduler/duration_backfill.py index f2153fa..d7cff68 100644 --- a/app/scheduler/duration_backfill.py +++ b/app/scheduler/duration_backfill.py @@ -84,7 +84,14 @@ def _fetch_one(row: tuple) -> tuple: from app.extractors._fetch import browser_get try: - r = browser_get(page_url, timeout=_FETCH_TIMEOUT, follow_redirects=True) + if origin == "tube:perverzijacom": + # Cloudflare blokuje IPv4 VPS-a (403, od 2026-10-14/15); bez proxy kazda proba + # palila sie i wiersz zamykal po trzech bez dlugosci. + from app.extractors.tubes.perverzija import fetch_perverzija + + r = fetch_perverzija(page_url, timeout=_FETCH_TIMEOUT) + else: + r = browser_get(page_url, timeout=_FETCH_TIMEOUT, follow_redirects=True) if r.status_code in (404, 410): return (scene_id, None, True) if r.status_code >= 400: