feat(hqfap): restore hqfap.com scraper + extractor (site back on new CDN)
hqfap was hard-removed 2026-06-25 when its entire CDN library started serving a fixed /upload/videos/video_down.mp4 "server down" stub for every scene. The site is back and now serves real video from a new CDN (d*.vstor.top/whlvid/...). Probe of the live sitemap: 5/6 sampled scenes return a real mp4 (0 stubs), and the CDN is portable cross-IP (HTTP 206 from a residential IP and from the VPS with the hqfap referer). Restores the PlayTube CMS browse scraper (HQFapScraper, sitetag hqfapcom) and the JSON-LD contentUrl extractor, re-registered in both registries. The extractor now sets mobile_direct_ok=True (vstor.top is not matched by _TIME_BOUND_CDN_RE, but is verified portable) so mobile plays direct with zero proxy. The video_down.mp4 stub guard is kept defensively. 4k69 stays removed (not re-checked). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
goon-foss
parent
cb22174657
commit
b66817175a
4 changed files with 124 additions and 10 deletions
|
|
@ -134,6 +134,7 @@ from app.connectors.direct_scrapers.shyfap import ShyfapScraper # noqa: E402, F
|
|||
from app.connectors.direct_scrapers.yesporn import YesPornVipScraper # noqa: E402
|
||||
from app.connectors.direct_scrapers.fullmovies import FullmoviesScraper # noqa: E402
|
||||
from app.connectors.direct_scrapers.hdporngg import HDPornGGScraper # noqa: E402
|
||||
from app.connectors.direct_scrapers.hqfap import HQFapScraper # noqa: E402
|
||||
from app.connectors.direct_scrapers.neporn import NepornScraper # noqa: E402
|
||||
from app.connectors.direct_scrapers.superporn import SuperpornScraper # noqa: E402
|
||||
from app.connectors.direct_scrapers.eporner_api import EpornerApiScraper # noqa: E402
|
||||
|
|
@ -212,12 +213,11 @@ ALL_BROWSE_SCRAPERS: list[type[BaseBrowseScraper]] = [
|
|||
# Mega-katalog ~13M → deep_crawl._PAGE_CAP["xvideoscom"]=1800 (~50k najnowszych), nie
|
||||
# full-crawl. (youporn pominięty — JSON-LD bez actor/keywords, scene-perf/tagi = nav A-Z.)
|
||||
XVideosBrowseScraper,
|
||||
# HQFapScraper / FourK69Scraper — USUNIĘTE CAŁKOWICIE 2026-06-25. Oba PlayTube CMS;
|
||||
# disabled 2026-06-22 gdy playback padł, re-check 2026-06-25 potwierdził że CAŁA
|
||||
# biblioteka CDN znikła: wide-sample przez pełny zakres id (hqfap 0/80 real, 4k69
|
||||
# 0/40 real) — każda scena serwuje stały `/upload/videos/video_down.mp4` "server
|
||||
# down" stub, nie realny plik. Dane (28k solo-orphan scen + 46k sources) skasowane
|
||||
# z DB, pliki scraperów/extractorów i wpisy w _REGISTRY usunięte.
|
||||
# HQFapScraper — PlayTube CMS (re-uploader pornhd.pet). Usunięty 2026-06-25 gdy CAŁA
|
||||
# biblioteka CDN serwowała `/upload/videos/video_down.mp4` stub; PRZYWRÓCONY 2026-07-07
|
||||
# bo strona wróciła na CDN vstor.top z realnymi plikami (portable cross-IP), user request.
|
||||
HQFapScraper,
|
||||
# FourK69Scraper — USUNIĘTY 2026-06-25 (ten sam stub), NIE sprawdzany ponownie.
|
||||
# NepornScraper — dołączony 2026-06-10 (user request). KVS engine (jak freshporno/
|
||||
# porn00), /latest-updates/N/. JSON-LD (title+desc+uploadDate+thumb) + video:duration
|
||||
# meta + /models/ performerzy + /categories/ tagi. Brak studio (tytuł bywa
|
||||
|
|
|
|||
28
app/connectors/direct_scrapers/hqfap.py
Normal file
28
app/connectors/direct_scrapers/hqfap.py
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
"""hqfap.com — latest-vids browse scraper (PlayTube CMS, patrz _playtube.py).
|
||||
|
||||
Dołączony 2026-06-10 (user request), usunięty 2026-06-25 (cała biblioteka CDN padła
|
||||
na stub), przywrócony 2026-07-07 (strona wróciła na CDN vstor.top z realnymi plikami,
|
||||
user request). Re-uploader katalogu pornhd.pet (~120k scen, thumbnaile to base64-encoded
|
||||
oryginalne URL-e w `/uploads/images/`).
|
||||
|
||||
Specyfika vs baza: studio siedzi w kategoriach z suffixem " Clips"
|
||||
("Filthy Kings Clips" → studio "Filthy Kings"); reszta kategorii → tagi.
|
||||
Playback: direct mp4 z JSON-LD contentUrl (vstor.top obecnie, cdnde.com/okcdn.ru
|
||||
starsze), tokeny time-bound i portable cross-IP → natywny extractor `hqfapcom`.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from app.connectors.direct_scrapers._playtube import BasePlayTubeScraper
|
||||
|
||||
|
||||
class HQFapScraper(BasePlayTubeScraper):
|
||||
sitetag = "hqfapcom"
|
||||
base_url = "https://hqfap.com"
|
||||
|
||||
def _pick_studio(self, category_names: list[str]) -> str | None:
|
||||
for name in category_names:
|
||||
if name.lower().endswith(" clips"):
|
||||
studio_name = name[: -len(" clips")].strip()
|
||||
if studio_name:
|
||||
return studio_name
|
||||
return None
|
||||
|
|
@ -31,6 +31,7 @@ from app.extractors.tubes import (
|
|||
freshporno,
|
||||
fullmovies,
|
||||
hdporngg,
|
||||
hqfap,
|
||||
hqporner,
|
||||
neporn,
|
||||
latestpornvideo,
|
||||
|
|
@ -164,10 +165,11 @@ _REGISTRY: dict[str, Callable[[str], list[StreamSource] | None]] = {
|
|||
# (#19866e9e wcześniej źle: założyłem „get_file 403 IP-bound" testem plain-curl.)
|
||||
"fullmoviesxxx": fullmovies.extract,
|
||||
"hdporngg": hdporngg.extract,
|
||||
# hqfap + 4k69 (PlayTube CMS) — USUNIĘTE CAŁKOWICIE 2026-06-25. Cała biblioteka CDN
|
||||
# znikła: każda scena serwuje stały `/upload/videos/video_down.mp4` "server down" stub
|
||||
# zamiast realnego pliku (wide-sample przez pełny zakres id: hqfap 0/80 real, 4k69
|
||||
# 0/40 real). Dane skasowane, scrapery/extractory usunięte.
|
||||
# hqfap (PlayTube CMS) — JSON-LD contentUrl = direct mp4. USUNIĘTY 2026-06-25 (cała
|
||||
# biblioteka serwowała `/upload/videos/video_down.mp4` stub), PRZYWRÓCONY 2026-07-07:
|
||||
# strona wróciła na CDN vstor.top z realnymi plikami (portable cross-IP, zweryfikowane),
|
||||
# user request. 4k69 zostaje usunięty (nie sprawdzany ponownie).
|
||||
"hqfapcom": hqfap.extract,
|
||||
# neporn — KVS function/0 + license (jak freshporno). Server-side _kvs resolve →
|
||||
# data001.neporn.com/remote_control.php portable (cross-IP 206, 2026-06-10).
|
||||
"neporncom": neporn.extract,
|
||||
|
|
|
|||
84
app/extractors/tubes/hqfap.py
Normal file
84
app/extractors/tubes/hqfap.py
Normal file
|
|
@ -0,0 +1,84 @@
|
|||
"""hqfap.com — direct stream extractor.
|
||||
|
||||
Scene page (SSR, za Cloudflare → curl_cffi w fetch_tube_html) ma JSON-LD
|
||||
VideoObject z `contentUrl` = direct mp4. Generacje hostingu w katalogu:
|
||||
|
||||
- obecne sceny (2026-07-07): `d*.vstor.top/whlvid/<epoch>/<token>/.../<id>_<q>p.mp4`
|
||||
— token time-bound, PORTABLE cross-IP (206 z residential ISP i VPS Hetzner,
|
||||
weryfikacja 2026-07-07 po powrocie strony),
|
||||
- starsze: `v4.cdnde.com/...?video=<b64>&time=<epoch>&ip=<addr>` (`ip` nieegzekwowany)
|
||||
oraz `vd*.okcdn.ru/?expires=...&srcIp=...` (ok.ru) — również portable cross-IP.
|
||||
|
||||
Wszystkie generacje grają direct z telefonu → StreamSource ma `mobile_direct_ok`
|
||||
(vstor.top nie łapie się w `_TIME_BOUND_CDN_RE` w playback.py, więc flagujemy jawnie),
|
||||
zero proxy/WebView.
|
||||
|
||||
Historia: wyłączony 2026-06-22 i USUNIĘTY 2026-06-25 gdy CAŁA biblioteka CDN
|
||||
serwowała stały `/upload/videos/video_down.mp4` stub. Strona wróciła na nowy CDN
|
||||
(vstor.top) z realnymi plikami → przywrócony 2026-07-07 (user request). Guard na
|
||||
stub zostaje defensywnie.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
|
||||
from app.extractors._fetch import fetch_tube_html
|
||||
from app.extractors._models import StreamSource
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
_JSONLD_RE = re.compile(
|
||||
r'<script[^>]+type=["\']application/ld\+json["\'][^>]*>(.*?)</script>',
|
||||
re.IGNORECASE | re.DOTALL,
|
||||
)
|
||||
# Fallback gdy JSON-LD nie parsuje się jako JSON (trailing comma itp.).
|
||||
_CONTENT_URL_RE = re.compile(r'"contentUrl"\s*:\s*"([^"]+)"')
|
||||
_QUALITY_RE = re.compile(r"_(\d{3,4})p\.mp4", re.IGNORECASE)
|
||||
|
||||
|
||||
def extract(page_url: str, *, timeout: float = 60.0) -> list[StreamSource] | None:
|
||||
html = fetch_tube_html(page_url, timeout=timeout)
|
||||
|
||||
content_url: str | None = None
|
||||
for m in _JSONLD_RE.finditer(html):
|
||||
raw = m.group(1).strip()
|
||||
if not raw:
|
||||
continue
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
continue
|
||||
items = data if isinstance(data, list) else [data]
|
||||
for obj in items:
|
||||
if isinstance(obj, dict) and obj.get("@type") == "VideoObject":
|
||||
content_url = (obj.get("contentUrl") or "").strip() or None
|
||||
break
|
||||
if content_url:
|
||||
break
|
||||
if not content_url:
|
||||
rm = _CONTENT_URL_RE.search(html)
|
||||
content_url = rm.group(1).strip() if rm else None
|
||||
if not content_url or not content_url.startswith("http"):
|
||||
log.warning("hqfap: no contentUrl in JSON-LD for %s", page_url)
|
||||
return None
|
||||
|
||||
# Guard: `/upload/videos/video_down.mp4` (+ mirror *.workers.dev) to stały ~3MB
|
||||
# "server down" placeholder (powód usunięcia 2026-06-25). Jeśli wróci — traktuj
|
||||
# jak brak źródła (lepiej żadne niż stub).
|
||||
if "/upload/videos/video_down.mp4" in content_url:
|
||||
log.info("hqfap: stub video_down.mp4 (placeholder, no real video) on %s", page_url)
|
||||
return None
|
||||
|
||||
qm = _QUALITY_RE.search(content_url)
|
||||
quality = f"{qm.group(1)}p" if qm else None
|
||||
return [
|
||||
StreamSource(
|
||||
link=content_url,
|
||||
quality=quality,
|
||||
type="mp4",
|
||||
referer="https://hqfap.com/",
|
||||
raw={"mobile_direct_ok": True},
|
||||
)
|
||||
]
|
||||
Loading…
Add table
Reference in a new issue