fix: hentaimama i xanime graja, filmy mangoporn z hosterami, ostatni link filmu chroniony
hentaimama: nowy zapis id posta i 5 playerow w admin-ajax, adresy w JSON-ie; bierzemy mp4 z gdvid (3 jakosci, bez tokenu), POST najpierw bez proxy. Ogon "Watch Online Free in HD" w tytulach ucinany. xanimeporn: przekierowanie download.php rozwiazywane na serwerze, apka dostawala .php, otwierala WebView i pobierala plik zamiast grac. Blad resolve (extractor pusty, HTTP strony) trafia do playback_events. Filmy: tytul z okruszkow (motyw dooplay losuje "Watch"/"Now Playing:"), linki nitroflare/rapidgator nie sa juz brane za wideo, sam landing idzie do WebView, a przytrzymanie w apce nie zabija ostatniego linku filmu, jesli serwer widzi, ze zyje. Skrypty naprawcze: tytuly dooplay, tytuly i miniatury hentaimama, repair_dooplay_movies dla streampornvip. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
goon-foss
parent
a872fc3a88
commit
45fbe0412e
9 changed files with 439 additions and 66 deletions
|
|
@ -21,9 +21,11 @@ from typing import Annotated, Any
|
|||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request, status
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.api.schemas import PlaybackSourceOut
|
||||
from app.api.device import get_device_id
|
||||
from app.auth import require_api_key
|
||||
from app.db import get_session
|
||||
from app.extractors import (
|
||||
|
|
@ -304,6 +306,18 @@ def resolve_movie_playback(
|
|||
raw={"origin": pb.origin},
|
||||
)
|
||||
)
|
||||
elif not links:
|
||||
# Sam adres strony serwisu (parser nie znalazl hosterow, np. mangoporn po zmianie
|
||||
# wygladu 10.2026). Ekstraktor nie ma tu czego szukac, telefon gra w WebView.
|
||||
links.append(
|
||||
StreamLink(
|
||||
stream_url=None,
|
||||
embed_url=pb.page_url,
|
||||
quality=pb.quality,
|
||||
type="hoster",
|
||||
raw={"origin": pb.origin},
|
||||
)
|
||||
)
|
||||
|
||||
if not links:
|
||||
raise HTTPException(status_code=502, detail="no playable links")
|
||||
|
|
@ -366,6 +380,43 @@ def mark_playback_dead(
|
|||
log.info("user marked playback %s dead (origin=%s %s)", pb.id, pb.origin, tag)
|
||||
|
||||
|
||||
def _movie_source_confirmed_dead(pb: Any) -> bool:
|
||||
"""Czy serwer sam widzi, ze link filmu nie zyje. Uzywane, gdy przytrzymanie w apce
|
||||
zabiloby OSTATNI link filmu.
|
||||
|
||||
Zgloszenie 284c9522 (2026-10-14): jedyny link "Italian She Male 44" oznaczyl telefon,
|
||||
ktory nie dochodzil do wezla CDN playmogo, a plik gral (206 z innego IP). Film zostal bez
|
||||
zadnego linku i apka chowala cala sekcje. Martwy jest tylko przy 404/410 strony embedu albo
|
||||
gdy ekstraktor mowi wprost, ze film usuniety. Rodzina DoodStream ma blok "Not Found"
|
||||
zawsze w HTML (ukryty), wiec tam zyje, dopoki strona ma pass_md5. Kazda niepewnosc
|
||||
(blad sieci, captcha, pusty ekstraktor) zostawia link.
|
||||
"""
|
||||
import httpx
|
||||
|
||||
target = pb.embed_url or pb.page_url
|
||||
try:
|
||||
r = httpx.get(
|
||||
target,
|
||||
headers={"User-Agent": "Mozilla/5.0", "Referer": pb.page_url or target},
|
||||
timeout=15.0,
|
||||
follow_redirects=True,
|
||||
)
|
||||
except Exception as e:
|
||||
log.info("kontrola linku filmu %s padla: %s", pb.id, e)
|
||||
return False
|
||||
if r.status_code in (404, 410):
|
||||
return True
|
||||
if r.status_code >= 400 or "pass_md5" in r.text:
|
||||
return False
|
||||
try:
|
||||
extract_stream_from_hoster(target, referer=pb.page_url or target)
|
||||
except HosterDead:
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
return False
|
||||
|
||||
|
||||
@movies_router.post(
|
||||
"/{movie_id}/playback/{playback_id}/mark-dead",
|
||||
status_code=status.HTTP_204_NO_CONTENT,
|
||||
|
|
@ -384,17 +435,62 @@ def mark_movie_playback_dead(
|
|||
raise HTTPException(status_code=404, detail="movie playback source not found")
|
||||
if pb.dead_at is None:
|
||||
tag = _requester_tag(request)
|
||||
others_alive = session.scalar(
|
||||
select(func.count())
|
||||
.select_from(MoviePlaybackSource)
|
||||
.where(
|
||||
MoviePlaybackSource.movie_id == movie_id,
|
||||
MoviePlaybackSource.id != pb.id,
|
||||
MoviePlaybackSource.dead_at.is_(None),
|
||||
)
|
||||
)
|
||||
if not others_alive and not _movie_source_confirmed_dead(pb):
|
||||
log.info(
|
||||
"user mark-dead na ostatnim zrodle filmu %s odrzucone, serwer go nie potwierdzil (%s)",
|
||||
pb.id, tag,
|
||||
)
|
||||
return
|
||||
pb.dead_at = datetime.now(UTC)
|
||||
pb.dead_reason = f"user-marked dead (mobile long-press) {tag}"[:512]
|
||||
session.commit()
|
||||
log.info("user marked movie playback %s dead (origin=%s %s)", pb.id, pb.origin, tag)
|
||||
|
||||
|
||||
def _record_resolve_failure(
|
||||
session: Session, pb: PlaybackSource, kind: str, detail: str, device_id: str
|
||||
) -> None:
|
||||
"""Blad rozwiazania po stronie serwera do playback_events.
|
||||
|
||||
Telefon po 503/502 z resolve pokazuje alert i nie dochodzi do odtwarzacza, wiec nic nie
|
||||
zglaszal. Hentaimama padala tak od zmiany strony i miala zero wierszy w telemetrii (ani
|
||||
sukcesow, ani bledow), wiec zaden alarm jej nie widzial (2026-10-15).
|
||||
"""
|
||||
from app.models.playback_event import PlaybackEvent
|
||||
|
||||
try:
|
||||
session.add(
|
||||
PlaybackEvent(
|
||||
origin=pb.origin[:64],
|
||||
scene_id=pb.scene_id,
|
||||
status="error",
|
||||
error_kind=kind,
|
||||
error_detail=detail[:512],
|
||||
via="backend",
|
||||
device_id=device_id,
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
except Exception as e:
|
||||
session.rollback()
|
||||
log.info("zapis bledu resolve do playback_events padl: %s", e)
|
||||
|
||||
|
||||
@router.post("/{scene_id}/playback/{playback_id}/resolve", response_model=ResolveOut)
|
||||
def resolve_playback(
|
||||
scene_id: uuid.UUID,
|
||||
playback_id: uuid.UUID,
|
||||
session: Annotated[Session, Depends(get_session)],
|
||||
device_id: Annotated[str, Depends(get_device_id)],
|
||||
) -> ResolveOut:
|
||||
pb = session.get(PlaybackSource, playback_id)
|
||||
if pb is None or pb.scene_id != scene_id:
|
||||
|
|
@ -438,6 +534,9 @@ def resolve_playback(
|
|||
log.info("marked playback %s dead (origin=%s reason=%s)", pb.id, pb.origin, reason)
|
||||
raise HTTPException(status_code=410, detail=f"playback dead: {reason}") from e
|
||||
log.warning("tube fetch http error %s for %s", e.status_code, pb.page_url)
|
||||
_record_resolve_failure(
|
||||
session, pb, "resolve_http", f"tube fetch HTTP {e.status_code}", device_id
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=502,
|
||||
detail=f"tube fetch failed: HTTP {e.status_code}",
|
||||
|
|
@ -457,6 +556,7 @@ def resolve_playback(
|
|||
"extractor None for playback %s (origin=%s) — transient, not marking dead",
|
||||
pb.id, pb.origin,
|
||||
)
|
||||
_record_resolve_failure(session, pb, "resolve_none", "extractor returned nothing", device_id)
|
||||
# 503 (not 410!) żeby mobile NIE pokazało "Tube usunął ten film", ten kod
|
||||
# jest dla permanent removal. 503 = transient, user może retry.
|
||||
# Sentry filtruje HTTPException 502/503/504 w `_sentry_before_send` (main.py),
|
||||
|
|
|
|||
|
|
@ -1,19 +1,19 @@
|
|||
"""hentaimama.io — browse scraper. Dodany 2026-08-04. Pierwsze źródło wertykału hentai.
|
||||
"""hentaimama.io \u2014 browse scraper. Dodany 2026-08-04. Pierwsze źródło wertykału hentai.
|
||||
|
||||
**Ten wertykał rządzi się inaczej niż reszta i to jest celowe.** Dla hentai NIE ISTNIEJE
|
||||
kanon: TPDB i StashDB nie mają animacji ani postaci rysowanych, więc każda scena stąd
|
||||
jest technicznie orphanem i to jest POPRAWNE, nie błędne. Kryterium „fabryka orphanów",
|
||||
którym oceniamy tuby z żywymi aktorami, tutaj po prostu nie ma zastosowania — nie ma też
|
||||
którym oceniamy tuby z żywymi aktorami, tutaj po prostu nie ma zastosowania \u2014 nie ma też
|
||||
obsady, więc żadnej bramki na performera. Miarą jakości jest komplet seria + numer
|
||||
odcinka + gatunki, a docelowo dedup MIĘDZY stronami hentai (re-hostują się nawzajem).
|
||||
|
||||
Jednostka treści to ODCINEK (`/episodes/<seria>-episode-N/`), nie seria. Każdy jest
|
||||
samodzielną pozycją z własnym plakatem i playbackiem, więc mapuje się na scenę tak samo
|
||||
jak w wertykale JAV — bez zmian w schemacie.
|
||||
jak w wertykale JAV \u2014 bez zmian w schemacie.
|
||||
|
||||
Motyw jest z rodziny Dooplay, ale `DooplayConnector` się NIE nadaje: nie ma tu
|
||||
`data-fl-source` ani `data-post`, na których stoi tamten parser. Player idzie przez
|
||||
`POST /wp-admin/admin-ajax.php` i jest za Cloudflare — obsługuje to extractor
|
||||
`POST /wp-admin/admin-ajax.php` i jest za Cloudflare \u2014 obsługuje to extractor
|
||||
`hentaimama`, patrz jego docstring.
|
||||
|
||||
Pułapka: `/genre/` w całym HTML to 118 linków (chmura w sidebarze, ten sam wzorzec co
|
||||
|
|
@ -38,14 +38,14 @@ log = logging.getLogger(__name__)
|
|||
_BASE = "https://hentaimama.io"
|
||||
|
||||
_EPISODE_HREF_RE = re.compile(r'href="(https://hentaimama\.io/episodes/[^"]+)"')
|
||||
# Sam listing `/episodes/` i jego paginacja też pasują do wzorca wyżej — bez tego
|
||||
# Sam listing `/episodes/` i jego paginacja też pasują do wzorca wyżej \u2014 bez tego
|
||||
# filtra wchodziły jako „odcinek" o tytule „Recent Episodes".
|
||||
# `/episodes/feed/` (RSS) tez wpada w href-regex i konczyl jako pusty detail z None.
|
||||
_NOT_EPISODE_RE = re.compile(r"/episodes/(?:page/\d+/|feed/)?$")
|
||||
# `og:title` zamiast <h1>: na stronie są trzy <h1>, dwa to logo serwisu, a trzeci bywa
|
||||
# nagłówkiem sekcji. og:title daje czysty „Shoujo Ramune Episode 7".
|
||||
_OG_TITLE_RE = re.compile(r"property=['\"]og:title['\"][^>]*content=['\"]([^'\"]{3,150})")
|
||||
# Gatunki TYLKO stąd — patrz pułapka w docstringu.
|
||||
# Gatunki TYLKO stąd \u2014 patrz pułapka w docstringu.
|
||||
_GENRES_BLOCK_RE = re.compile(r'<div class="sgeneros">(.*?)</div>', re.DOTALL)
|
||||
_ANCHOR_TEXT_RE = re.compile(r">\s*([^<]{2,40}?)\s*</a>")
|
||||
_DATE_RE = re.compile(r'<span class="date">\s*([A-Z][a-z]{2}\.?\s+\d{1,2},\s+\d{4})')
|
||||
|
|
@ -55,7 +55,11 @@ _DATE_RE = re.compile(r'<span class="date">\s*([A-Z][a-z]{2}\.?\s+\d{1,2},\s+\d{
|
|||
_DT_DURATION_RE = re.compile(r'"duration"\s*:\s*"(\d{2,6})"')
|
||||
_SERIES_RE = re.compile(r'href="https://hentaimama\.io/tvshows/([a-z0-9-]+)/"')
|
||||
# Uwaga: atrybuty bywają w innej kolejności niż `<meta property=... content=...>`,
|
||||
# a wartość ma na końcu \r — stąd luźny wzorzec i strip() przy użyciu.
|
||||
# a wartość ma na końcu \r \u2014 stąd luźny wzorzec i strip() przy użyciu.
|
||||
_TITLE_TAIL_RE = re.compile(
|
||||
r"(?:\s*[\u2013\u2014|-]\s*Watch\s+Online\s+Free\s+in\s+HD)?\s*[\u2013\u2014|-]\s*Hentaimama\s*$",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_OG_IMAGE_RE = re.compile(r"property=['\"]og:image['\"][^>]*content=['\"]([^'\"]+)")
|
||||
|
||||
|
||||
|
|
@ -86,6 +90,9 @@ class HentaiMamaScraper(BaseBrowseScraper):
|
|||
def _parse_detail(self, scene_url: str, detail_html: str) -> RawScene | None:
|
||||
tm = _OG_TITLE_RE.search(detail_html)
|
||||
title = html_mod.unescape(tm.group(1)).strip() if tm else ""
|
||||
# Od ~31.08.2026 og:title ma ogon "TYTUL - Watch Online Free in HD - Hentaimama"
|
||||
# (z polpauzami). Ucinamy go, inaczej tytul nie laczy sie z ta sama scena z innych stron.
|
||||
title = _TITLE_TAIL_RE.sub("", title).strip()
|
||||
if not title or title.lower() == "hentaimama":
|
||||
return None
|
||||
|
||||
|
|
@ -101,7 +108,7 @@ class HentaiMamaScraper(BaseBrowseScraper):
|
|||
tags.append(
|
||||
RawTag(external_id=f"{self.sitetag}:tag:{slug}", name=name, slug=slug)
|
||||
)
|
||||
# Seria jako tag — pozwala grupować odcinki bez zmian w schemacie.
|
||||
# Seria jako tag \u2014 pozwala grupować odcinki bez zmian w schemacie.
|
||||
sm = _SERIES_RE.search(detail_html)
|
||||
if sm and sm.group(1) not in seen:
|
||||
series = sm.group(1).replace("-", " ").title()
|
||||
|
|
|
|||
|
|
@ -171,8 +171,15 @@ _VID_TITLE_RE = re.compile(
|
|||
r'<h1[^>]*\bclass="vid__t[^"]*"[^>]*>\s*(?:<span[^>]*>)?\s*(.+?)\s*(?:</span>)?\s*</h1>',
|
||||
re.IGNORECASE | re.DOTALL,
|
||||
)
|
||||
# Motyw 2026-10 losuje przedrostek w h1 ("Watch ...", "Now Playing: ...", "Play ...", na
|
||||
# pandamovies "... in the Panda Database"). Czysty tytul stoi w okruszkach, bierzemy go
|
||||
# stamtad, a h1 zostaje do roku i studia (i jako zapas bez okruszkow). "Play" nie jest
|
||||
# zdejmowane z h1, bo bywa czescia tytulu ("Play Time!", "Play With Me").
|
||||
_BC_CURRENT_RE = re.compile(
|
||||
r'<span[^>]*\bclass="bc__c"[^>]*\baria-current="page"[^>]*>([^<]+)</span>', re.IGNORECASE
|
||||
)
|
||||
_VID_TITLE_PARTS_RE = re.compile(
|
||||
r"^(?:Watch\s+)?(?P<title>.+?)"
|
||||
r"^(?:Watch\s+|Now\s+Playing:\s*)?(?P<title>.+?)"
|
||||
r"(?:\s+(?P<year>(?:19|20)\d{2})\s+by\s+(?P<studio>.+?))?"
|
||||
r"(?:\s+Porn\s+Movie\s+Online\s+Free)?\s*$",
|
||||
re.IGNORECASE | re.DOTALL,
|
||||
|
|
@ -359,6 +366,10 @@ def _parse_dooplay_detail(
|
|||
raw_title = _decode_html(_strip_tags(m_vid.group(1))).strip()
|
||||
m_parts = _VID_TITLE_PARTS_RE.match(raw_title)
|
||||
title = (m_parts.group("title") if m_parts else raw_title).strip()
|
||||
title = re.sub(r"\s+in\s+the\s+Panda\s+Database$", "", title, flags=re.IGNORECASE)
|
||||
m_bc = _BC_CURRENT_RE.search(html)
|
||||
if m_bc and _decode_html(m_bc.group(1)).strip():
|
||||
title = _decode_html(m_bc.group(1)).strip()
|
||||
if m_parts and m_parts.group("year"):
|
||||
vid_year = int(m_parts.group("year"))
|
||||
if m_parts and m_parts.group("studio"):
|
||||
|
|
|
|||
|
|
@ -273,7 +273,9 @@ def extract_stream_from_hoster(
|
|||
"""Iterate matches, pomiń preroll/ad URLs. Zwraca pierwszy clean lub None."""
|
||||
for m in pattern.finditer(text):
|
||||
url = m.group(group)
|
||||
if not _looks_like_ad(url):
|
||||
# Link "Download" do nitroflare/rapidgator konczy sie na .mp4, a to strona
|
||||
# hostingu plikow (mangoporn 2026-10: 95 bledow ExoPlayera przez proxy).
|
||||
if not _looks_like_ad(url) and not _FILE_HOSTER_RE.search(url):
|
||||
return url
|
||||
return None
|
||||
|
||||
|
|
@ -356,7 +358,7 @@ def _try_ytdlp_hoster(iframe_url: str, *, timeout: float) -> str | None:
|
|||
def _looks_like_video_url(u: str | None) -> bool:
|
||||
if not u:
|
||||
return False
|
||||
if _looks_like_ad(u):
|
||||
if _looks_like_ad(u) or _FILE_HOSTER_RE.search(u):
|
||||
return False
|
||||
low = u.lower()
|
||||
# Standardowe formaty video. yt-dlp generic czasem zwraca page URL jako
|
||||
|
|
|
|||
|
|
@ -14,6 +14,13 @@ ale nie trzeba go składać ręcznie — bierzemy iframe taki, jaki przyszedł.
|
|||
`admin-ajax.php` dostaje challenge Cloudflare (403, „Just a moment..."), dokładnie jak
|
||||
`POST /api/video-info` przy sexu. Bez klucza proxy extractor zwraca None.
|
||||
|
||||
**2026-10-15 (zgloszenie 6ab3a973, zadna scena nie grala):** strona zmienila zapis na
|
||||
`{action: 'get_player_contents',a: '79669'}` (spacje), a odpowiedz admin-ajax ma 5 playerow
|
||||
(`?dt_embed=rtmp|hls|hls-mp4|r2-mp4` i `newjav.php`) z `&#038;` w src. Strony playerow
|
||||
trzymaja adresy w JSON-ie (ukosniki zapisane jako backslash i slash). javprovider daje 404 dla nowych odcinkow, gdvid
|
||||
(`/hls/output/.../mp4/<jakosc>/...mp4`, bez tokenu) gra wszedzie. POST przechodzi dzis z VPS-a
|
||||
bez proxy, Bright Data zostaje tylko na wypadek 403/Cloudflare.
|
||||
|
||||
**Preferujemy `gdvid.info` nad `javprovider.com`**: pierwszy oddaje plik BEZ tokenu,
|
||||
więc URL nigdy nie wygasa; drugi ma `?st=...&e=<epoch>`. Oba zweryfikowane z VPS-a:
|
||||
206 na Range, `ftypisom` w pierwszym KB (faststart), działają BEZ Referera — stąd
|
||||
|
|
@ -21,6 +28,7 @@ więc URL nigdy nie wygasa; drugi ma `?st=...&e=<epoch>`. Oba zweryfikowane z VP
|
|||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
|
|
@ -32,11 +40,13 @@ from app.extractors._models import StreamSource
|
|||
log = logging.getLogger(__name__)
|
||||
|
||||
_BASE = "https://hentaimama.io"
|
||||
_POST_ID_RE = re.compile(r"action:'get_player_contents',\s*a:'(\d+)'")
|
||||
_POST_ID_RE = re.compile(r"""action:\s*['"]get_player_contents['"],\s*a:\s*['"](\d+)['"]""")
|
||||
_BODY_POST_ID_RE = re.compile(r"\bpostid-(\d+)\b")
|
||||
_IFRAME_SRC_RE = re.compile(r'<iframe[^>]+src="([^"]+)"', re.IGNORECASE)
|
||||
_MP4_RE = re.compile(r'https?://[^\s"\'<>]+\.mp4[^\s"\'<>]*')
|
||||
#: Host bez tokenu idzie pierwszy — jego URL nie wygasa.
|
||||
_HOST_RANK = ("gdvid", "javprovider")
|
||||
_VIDEO_RE = re.compile(r"""https?://[^\s"'<>]+\.(?:mp4|m3u8)[^\s"'<>]*""")
|
||||
_QUALITY_RE = re.compile(r"/mp4/(\d{3,4})p/")
|
||||
#: Host bez tokenu idzie pierwszy, jego URL nie wygasa. javprovider na koniec (404 na nowych).
|
||||
_HOST_RANK = ("gdvid.info/hls/", "gdvid", "hentaidoge", "javprovider")
|
||||
|
||||
|
||||
def _rank(url: str) -> int:
|
||||
|
|
@ -46,42 +56,55 @@ def _rank(url: str) -> int:
|
|||
return len(_HOST_RANK)
|
||||
|
||||
|
||||
def extract(page_url: str, *, timeout: float = 60.0) -> list[StreamSource] | None:
|
||||
proxy = get_settings().brightdata_proxy_url
|
||||
if not proxy:
|
||||
log.info("hentaimama: brak BRIGHTDATA_PROXY_URL — CF zablokuje POST admin-ajax")
|
||||
return None
|
||||
def _quality(url: str) -> int:
|
||||
m = _QUALITY_RE.search(url)
|
||||
return int(m.group(1)) if m else 0
|
||||
|
||||
|
||||
def _player_contents(page_url: str, post_id: str, timeout: float):
|
||||
"""POST get_player_contents: najpierw bez proxy, Bright Data tylko przy blokadzie."""
|
||||
from curl_cffi import requests as cr
|
||||
|
||||
proxy = get_settings().brightdata_proxy_url
|
||||
attempts = [None] + ([proxy] if proxy else [])
|
||||
for px in attempts:
|
||||
session = cr.Session(
|
||||
impersonate="chrome120",
|
||||
proxies={"http": px, "https": px} if px else None,
|
||||
verify=False,
|
||||
)
|
||||
try:
|
||||
# Strona odcinka najpierw, bez jej ciasteczek CF potrafi odrzucic POST-a.
|
||||
session.get(page_url, timeout=timeout)
|
||||
res = session.post(
|
||||
f"{_BASE}/wp-admin/admin-ajax.php",
|
||||
data={"action": "get_player_contents", "a": post_id},
|
||||
headers={"Referer": page_url, "X-Requested-With": "XMLHttpRequest"},
|
||||
timeout=timeout,
|
||||
)
|
||||
if res.status_code == 200:
|
||||
return session, json.loads(res.text)
|
||||
log.info("hentaimama: admin-ajax %s (proxy=%s) dla %s", res.status_code, bool(px), page_url)
|
||||
except Exception as e:
|
||||
log.info("hentaimama: get_player_contents (proxy=%s) %s: %s", bool(px), page_url, e)
|
||||
return None, None
|
||||
|
||||
|
||||
def extract(page_url: str, *, timeout: float = 60.0) -> list[StreamSource] | None:
|
||||
html_text = fetch_tube_html(page_url, timeout=timeout)
|
||||
pm = _POST_ID_RE.search(html_text)
|
||||
pm = _POST_ID_RE.search(html_text) or _BODY_POST_ID_RE.search(html_text)
|
||||
if not pm:
|
||||
log.info("hentaimama: brak id posta na %s", page_url)
|
||||
return None
|
||||
|
||||
from curl_cffi import requests as cr
|
||||
|
||||
try:
|
||||
session = cr.Session(
|
||||
impersonate="chrome120",
|
||||
proxies={"http": proxy, "https": proxy},
|
||||
verify=False,
|
||||
)
|
||||
# Strona odcinka najpierw — bez jej ciasteczek CF odrzuca POST-a.
|
||||
session.get(page_url, timeout=timeout)
|
||||
res = session.post(
|
||||
f"{_BASE}/wp-admin/admin-ajax.php",
|
||||
data={"action": "get_player_contents", "a": pm.group(1)},
|
||||
headers={"Referer": page_url, "X-Requested-With": "XMLHttpRequest"},
|
||||
timeout=timeout,
|
||||
)
|
||||
fields = json.loads(res.text)
|
||||
except Exception as e:
|
||||
log.info("hentaimama: get_player_contents failed %s: %s", page_url, e)
|
||||
session, fields = _player_contents(page_url, pm.group(1), timeout)
|
||||
if session is None:
|
||||
return None
|
||||
|
||||
players: list[str] = []
|
||||
for field in fields if isinstance(fields, list) else []:
|
||||
for src in _IFRAME_SRC_RE.findall(str(field)):
|
||||
src = html.unescape(html.unescape(src))
|
||||
if src.startswith("//"):
|
||||
src = "https:" + src
|
||||
if src.startswith("http") and src not in players:
|
||||
|
|
@ -91,29 +114,29 @@ def extract(page_url: str, *, timeout: float = 60.0) -> list[StreamSource] | Non
|
|||
seen: set[str] = set()
|
||||
for player_url in players:
|
||||
try:
|
||||
body = session.get(
|
||||
player_url, headers={"Referer": page_url}, timeout=timeout
|
||||
).text
|
||||
body = session.get(player_url, headers={"Referer": page_url}, timeout=timeout).text
|
||||
except Exception as e:
|
||||
log.info("hentaimama: player fetch failed %s: %s", player_url, e)
|
||||
continue
|
||||
for link in _MP4_RE.findall(body):
|
||||
if link in seen:
|
||||
body = body.replace(chr(92) + "/", "/")
|
||||
for link in _VIDEO_RE.findall(body):
|
||||
if link in seen or "thumbnail" in link or link.endswith(".vtt"):
|
||||
continue
|
||||
seen.add(link)
|
||||
q = _quality(link)
|
||||
out.append(
|
||||
StreamSource(
|
||||
link=link,
|
||||
type="mp4",
|
||||
quality=None, # serwis nie podaje rozdzielczości
|
||||
type="m3u8" if ".m3u8" in link else "mp4",
|
||||
quality=f"{q}p" if q else None,
|
||||
referer=_BASE + "/",
|
||||
# Zweryfikowane: 206 bez Referera, faststart → telefon gra direct.
|
||||
# Zweryfikowane: 206 bez Referera, faststart, telefon gra direct.
|
||||
raw={"mobile_direct_ok": True},
|
||||
)
|
||||
)
|
||||
|
||||
if not out:
|
||||
log.info("hentaimama: nie znalazłem mp4 dla %s", page_url)
|
||||
log.info("hentaimama: nie znalazlem wideo dla %s", page_url)
|
||||
return None
|
||||
out.sort(key=lambda s: _rank(s.link))
|
||||
out.sort(key=lambda s: (_rank(s.link), s.type != "mp4", -_quality(s.link)))
|
||||
return out
|
||||
|
|
|
|||
|
|
@ -12,11 +12,14 @@ Wyjściem jest zakładka „Downloads", która niesie:
|
|||
`postId` siedzi w klasie `<body>` (`postid-69951`), więc nie trzeba parsować zakładki —
|
||||
składamy URL sami dla obu jakości.
|
||||
|
||||
**Oddajemy `download.php`, a NIE docelowego pliku.** Endpoint przekierowuje na
|
||||
`download.xanimeporn.com/...?s=<sig>&e=<epoch>`, czyli URL z tokenem i wygasaniem.
|
||||
Zwracając `download.php` zostawiamy generowanie tokenu na moment odtwarzania: telefon
|
||||
idzie za przekierowaniem i dostaje świeży podpis, więc nic nie wietrzeje. To ta sama
|
||||
lekcja co przy watchporn, gdzie zapisany statyczny URL po czasie padał.
|
||||
**Oddajemy docelowy plik, nie `download.php` (zmiana 2026-10-15).** Endpoint przekierowuje
|
||||
na `download.xanimeporn.com/videos/<q>/...mp4?s=<sig>&e=<epoch>` z `Content-Disposition:
|
||||
attachment`. Wczesniej oddawalismy `download.php`, zeby telefon sam bral swiezy podpis, ale
|
||||
apka po `.php` w adresie uznawala link za strone playera i otwierala go w WebView, ktory
|
||||
POBIERAL plik do Pobranych zamiast grac (zgloszenie 54b17aed; od dodania serwisu ani jedno
|
||||
odtworzenie). Podpis zyje ok. 2 h i nie jest zwiazany z IP (adres z VPS-a gra z domowego),
|
||||
a resolve dzieje sie przy kliknieciu, wiec rozwiazujemy przekierowanie tutaj. Gdy sie nie
|
||||
uda, zostaje `download.php` jako zapas.
|
||||
|
||||
Zweryfikowane z VPS-a: 1080p 222 MB i 720p 84 MB, oba 206 na Range z `ftypisom`/`ftypmp42`
|
||||
w pierwszym KB (faststart). Wymaga Referera i UA, oba dokłada `playback.py`. Bez
|
||||
|
|
@ -27,6 +30,8 @@ from __future__ import annotations
|
|||
import logging
|
||||
import re
|
||||
|
||||
import httpx
|
||||
|
||||
from app.extractors._fetch import fetch_tube_html
|
||||
from app.extractors._models import StreamSource
|
||||
|
||||
|
|
@ -36,6 +41,7 @@ _BASE = "https://xanimeporn.com"
|
|||
_POST_ID_RE = re.compile(r"\bpostid-(\d+)\b")
|
||||
#: Serwis publikuje dokładnie te dwie jakości; kolejność = preferencja.
|
||||
_QUALITIES = ("1080p", "720p")
|
||||
_UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36"
|
||||
|
||||
|
||||
def extract(page_url: str, *, timeout: float = 60.0) -> list[StreamSource] | None:
|
||||
|
|
@ -46,15 +52,29 @@ def extract(page_url: str, *, timeout: float = 60.0) -> list[StreamSource] | Non
|
|||
return None
|
||||
post_id = m.group(1)
|
||||
|
||||
return [
|
||||
out: list[StreamSource] = []
|
||||
for q in _QUALITIES:
|
||||
link = f"https://videos.xanimeporn.com/download.php?id={post_id}&quality={q}"
|
||||
try:
|
||||
res = httpx.get(
|
||||
link,
|
||||
headers={"User-Agent": _UA, "Referer": _BASE + "/"},
|
||||
follow_redirects=False,
|
||||
timeout=15.0,
|
||||
)
|
||||
location = res.headers.get("location") or ""
|
||||
if res.status_code in (301, 302, 303, 307, 308) and location.startswith("http"):
|
||||
link = location
|
||||
except Exception as e:
|
||||
log.info("xanimeporn: przekierowanie %s %s nie wyszlo: %s", post_id, q, e)
|
||||
out.append(
|
||||
StreamSource(
|
||||
link=f"https://videos.xanimeporn.com/download.php?id={post_id}&quality={q}",
|
||||
link=link,
|
||||
type="mp4",
|
||||
quality=q,
|
||||
referer=_BASE + "/",
|
||||
# Zweryfikowane 206 + faststart z VPS-a; telefon sam podąża za 302 i
|
||||
# dostaje własny, świeży token.
|
||||
# Zweryfikowane 206 + faststart, podpis nie jest zwiazany z IP.
|
||||
raw={"mobile_direct_ok": True},
|
||||
)
|
||||
for q in _QUALITIES
|
||||
]
|
||||
)
|
||||
return out
|
||||
|
|
|
|||
98
scripts/fix_dooplay_junk_titles.py
Normal file
98
scripts/fix_dooplay_junk_titles.py
Normal file
|
|
@ -0,0 +1,98 @@
|
|||
"""Czysci tytuly filmow z naglowka nowego motywu dooplay (10.2026) i scala duplikaty.
|
||||
|
||||
Motyw mangoporn/streamporn.li wstawia do h1 "Watch TYTUL 2026 by STUDIO Porn Movie Online
|
||||
Free" albo "Now Playing: TYTUL". Parser od 2026-10-15 bierze tytul z okruszkow, ten skrypt
|
||||
poprawia to, co weszlo wczesniej:
|
||||
|
||||
- "Now Playing: X" -> "X" (zawsze naglowek, nigdy prawdziwy tytul),
|
||||
- "Watch X RRRR by S Porn Movie Online Free" -> "X" (tylko z tym ogonem; "Watch Me Eat My
|
||||
Creampie 5" to prawdziwy tytul i zostaje),
|
||||
- "Play X" zostaje, bo bywa tytulem ("Play Time!").
|
||||
|
||||
Jesli po poprawce istnieje juz inny film o tym samym title_normalized i zgodnym roku
|
||||
(rowny albo jeden pusty), poprawiany film jest do niego dolaczany (merge_movies, starszy
|
||||
zostaje). Wiecej niz jedno trafienie = tylko poprawka tytulu, bez scalania.
|
||||
|
||||
python scripts/fix_dooplay_junk_titles.py # podglad
|
||||
python scripts/fix_dooplay_junk_titles.py --apply
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import re
|
||||
|
||||
import sqlalchemy as sa
|
||||
|
||||
from app.db import session_scope
|
||||
from app.models.movie import Movie
|
||||
from app.normalize.text import normalize
|
||||
from app.resolve.movie_merge import merge_movies
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
_NOW_PLAYING_RE = re.compile(r"^Now\s+Playing:\s*(?P<title>.+?)\s*$", re.IGNORECASE | re.DOTALL)
|
||||
_WATCH_TAIL_RE = re.compile(
|
||||
r"^Watch\s+(?P<title>.+?)(?:\s+(?:19|20)\d{2}\s+by\s+.+?)?\s+Porn\s+Movie\s+Online\s+Free\s*$",
|
||||
re.IGNORECASE | re.DOTALL,
|
||||
)
|
||||
|
||||
|
||||
def clean_title(title: str) -> str | None:
|
||||
for rx in (_NOW_PLAYING_RE, _WATCH_TAIL_RE):
|
||||
m = rx.match(title)
|
||||
if m and m.group("title").strip():
|
||||
return m.group("title").strip()
|
||||
return None
|
||||
|
||||
|
||||
def run(apply: bool) -> dict[str, int]:
|
||||
counters = {"candidates": 0, "retitled": 0, "merged": 0, "ambiguous": 0}
|
||||
with session_scope() as session:
|
||||
rows = session.execute(
|
||||
sa.select(Movie.id, Movie.title, Movie.release_year).where(
|
||||
sa.or_(
|
||||
Movie.title.op("~*")(r"^now playing:"),
|
||||
Movie.title.op("~*")(r"^watch .+ porn movie online free\s*$"),
|
||||
)
|
||||
)
|
||||
).all()
|
||||
for movie_id, title, year in rows:
|
||||
new_title = clean_title(title)
|
||||
if not new_title:
|
||||
continue
|
||||
counters["candidates"] += 1
|
||||
norm = normalize(new_title)
|
||||
twins = session.execute(
|
||||
sa.select(Movie.id, Movie.release_year, Movie.created_at)
|
||||
.where(Movie.title_normalized == norm, Movie.id != movie_id)
|
||||
.order_by(Movie.created_at)
|
||||
).all()
|
||||
twins = [t for t in twins if t.release_year is None or year is None or t.release_year == year]
|
||||
print(f"{title[:70]!r} -> {new_title[:60]!r} twins={len(twins)}")
|
||||
if not apply:
|
||||
continue
|
||||
movie = session.get(Movie, movie_id)
|
||||
movie.title = new_title[:512]
|
||||
movie.title_normalized = norm[:512]
|
||||
counters["retitled"] += 1
|
||||
if len(twins) == 1:
|
||||
merge_movies(session, keep_id=twins[0].id, drop_id=movie_id, resolved_by="fix_dooplay_junk_titles")
|
||||
counters["merged"] += 1
|
||||
elif len(twins) > 1:
|
||||
counters["ambiguous"] += 1
|
||||
session.commit()
|
||||
return counters
|
||||
|
||||
|
||||
def main() -> int:
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--apply", action="store_true")
|
||||
args = p.parse_args()
|
||||
print(run(args.apply))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
110
scripts/fix_hentaimama_titles_thumbs.py
Normal file
110
scripts/fix_hentaimama_titles_thumbs.py
Normal file
|
|
@ -0,0 +1,110 @@
|
|||
"""Hentaimama: ogon w tytulach i brakujace miniatury (2026-10-15).
|
||||
|
||||
1. Tytuly: od ~31.08.2026 og:title mial ogon "TYTUL - Watch Online Free in HD - Hentaimama"
|
||||
i tak weszlo ok. 100 scen. Scraper juz go ucina, tu poprawiamy istniejace (title i
|
||||
title_normalized).
|
||||
2. Miniatury: ok. 2100 zrodel z pierwszego przebiegu (12.08) nie ma thumbnail_url, a te sceny
|
||||
nie maja innego zrodla z obrazkiem. Strony maja og:image, pobieramy je po kolei.
|
||||
|
||||
python scripts/fix_hentaimama_titles_thumbs.py # podglad
|
||||
python scripts/fix_hentaimama_titles_thumbs.py --apply [--limit N] [--sleep 0.5]
|
||||
|
||||
Dlugie, wiec na prodzie przez `docker compose run -d --rm worker python3 scripts/...`
|
||||
(z pliku, nie przez stdin).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import html as html_mod
|
||||
import logging
|
||||
import time
|
||||
|
||||
import sqlalchemy as sa
|
||||
|
||||
from app.connectors.direct_scrapers.hentaimama import _OG_IMAGE_RE, _TITLE_TAIL_RE
|
||||
from app.db import session_scope
|
||||
from app.extractors._fetch import fetch_tube_html
|
||||
from app.normalize.text import normalize
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def fix_titles(apply: bool) -> int:
|
||||
n = 0
|
||||
with session_scope() as session:
|
||||
rows = session.execute(
|
||||
sa.text("SELECT id, title FROM scenes WHERE title ~* 'hentaimama\\s*$'")
|
||||
).all()
|
||||
for scene_id, title in rows:
|
||||
new = _TITLE_TAIL_RE.sub("", title).strip()
|
||||
if not new or new == title:
|
||||
continue
|
||||
n += 1
|
||||
if n <= 5:
|
||||
print(f"{title!r} -> {new!r}")
|
||||
if apply:
|
||||
session.execute(
|
||||
sa.text("UPDATE scenes SET title=:t, title_normalized=:n WHERE id=:id"),
|
||||
{"t": new[:512], "n": normalize(new)[:512], "id": scene_id},
|
||||
)
|
||||
if apply:
|
||||
session.commit()
|
||||
return n
|
||||
|
||||
|
||||
def fix_thumbs(apply: bool, limit: int, sleep: float) -> dict[str, int]:
|
||||
c = {"checked": 0, "filled": 0, "no_image": 0, "fetch_fail": 0}
|
||||
with session_scope() as session:
|
||||
rows = session.execute(
|
||||
sa.text(
|
||||
"SELECT id, page_url FROM playback_sources WHERE origin='tube:hentaimama' "
|
||||
"AND thumbnail_url IS NULL AND dead_at IS NULL ORDER BY id LIMIT :lim"
|
||||
),
|
||||
{"lim": limit},
|
||||
).all()
|
||||
for pb_id, page_url in rows:
|
||||
c["checked"] += 1
|
||||
try:
|
||||
page = fetch_tube_html(page_url, timeout=30.0)
|
||||
except Exception as e:
|
||||
c["fetch_fail"] += 1
|
||||
log.info("fetch %s: %s", page_url, e)
|
||||
time.sleep(sleep)
|
||||
continue
|
||||
m = _OG_IMAGE_RE.search(page)
|
||||
if not m:
|
||||
c["no_image"] += 1
|
||||
else:
|
||||
img = html_mod.unescape(m.group(1)).strip()
|
||||
if apply:
|
||||
with session_scope() as session:
|
||||
session.execute(
|
||||
sa.text("UPDATE playback_sources SET thumbnail_url=:u WHERE id=:id"),
|
||||
{"u": img[:2048], "id": pb_id},
|
||||
)
|
||||
session.commit()
|
||||
c["filled"] += 1
|
||||
if c["filled"] <= 3:
|
||||
print(page_url, "->", img)
|
||||
if c["checked"] % 100 == 0:
|
||||
log.info("thumbs %s", c)
|
||||
time.sleep(sleep)
|
||||
return c
|
||||
|
||||
|
||||
def main() -> int:
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--apply", action="store_true")
|
||||
p.add_argument("--limit", type=int, default=5)
|
||||
p.add_argument("--sleep", type=float, default=0.5)
|
||||
p.add_argument("--skip-titles", action="store_true")
|
||||
args = p.parse_args()
|
||||
if not args.skip_titles:
|
||||
print("titles:", fix_titles(args.apply))
|
||||
print("thumbs:", fix_thumbs(args.apply, args.limit, args.sleep))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
|
|
@ -25,6 +25,7 @@ from app.connectors.dooplay import (
|
|||
MangopornConnector,
|
||||
PandamoviesConnector,
|
||||
StreampornConnector,
|
||||
StreampornVipConnector,
|
||||
)
|
||||
from app.db import session_scope
|
||||
from app.models.movie import Movie
|
||||
|
|
@ -38,6 +39,7 @@ CONNECTORS = {
|
|||
"pandamovies": PandamoviesConnector,
|
||||
"mangoporn": MangopornConnector,
|
||||
"streamporn": StreampornConnector,
|
||||
"streampornvip": StreampornVipConnector,
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue