Compare commits
32 commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b613ac6827 | ||
|
|
e0f70f2574 | ||
|
|
37f8eb1762 | ||
|
|
bf45339c97 | ||
|
|
f58ae3ea2d | ||
|
|
f167c491c8 | ||
|
|
5e203e7e98 | ||
|
|
34138ff82f | ||
|
|
b3938abf06 | ||
|
|
3d71a071a3 | ||
|
|
1555408e20 | ||
|
|
763d38392c | ||
|
|
e0744da8c7 | ||
|
|
6da817dc79 | ||
|
|
2dbb9ba65f | ||
|
|
9ee15dc934 | ||
|
|
dcde8c8d66 | ||
|
|
4233833a2b | ||
|
|
0536b1b61c | ||
|
|
b19d041f09 | ||
|
|
97bca096e6 | ||
|
|
5eb72aeb7f | ||
|
|
bc674c2ec4 | ||
|
|
ca9ccf29f1 | ||
|
|
6379e98dd9 | ||
|
|
d521e6379d | ||
|
|
671ba97b52 | ||
|
|
45fbe0412e | ||
|
|
a872fc3a88 | ||
|
|
09f389100a | ||
|
|
1ad16ceae9 | ||
|
|
cbec97fc79 |
71 changed files with 3618 additions and 185 deletions
41
alembic/versions/20261014_0056_browse_runs.py
Normal file
41
alembic/versions/20261014_0056_browse_runs.py
Normal file
|
|
@ -0,0 +1,41 @@
|
||||||
|
"""Przebiegi browse-latest per scraper (watchdog listingu).
|
||||||
|
|
||||||
|
Revision ID: 0056_browse_runs
|
||||||
|
Revises: 0055_survey_responses
|
||||||
|
Create Date: 2026-10-14
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
from sqlalchemy.dialects.postgresql import UUID
|
||||||
|
|
||||||
|
revision = "0056_browse_runs"
|
||||||
|
down_revision: str | None = "0055_survey_responses"
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.create_table(
|
||||||
|
"browse_runs",
|
||||||
|
sa.Column("id", UUID(as_uuid=True), primary_key=True),
|
||||||
|
sa.Column("sitetag", sa.String(64), nullable=False),
|
||||||
|
sa.Column(
|
||||||
|
"started_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False
|
||||||
|
),
|
||||||
|
sa.Column("elapsed_s", sa.Float, nullable=True),
|
||||||
|
sa.Column("seen", sa.Integer, nullable=False, server_default="0"),
|
||||||
|
sa.Column("new", sa.Integer, nullable=False, server_default="0"),
|
||||||
|
sa.Column("updated", sa.Integer, nullable=False, server_default="0"),
|
||||||
|
sa.Column("skipped", sa.Integer, nullable=False, server_default="0"),
|
||||||
|
sa.Column("errors", sa.Integer, nullable=False, server_default="0"),
|
||||||
|
sa.Column("failed", sa.Boolean, nullable=False, server_default=sa.false()),
|
||||||
|
sa.Column("note", sa.Text, nullable=True),
|
||||||
|
)
|
||||||
|
op.create_index("ix_browse_runs_sitetag_started", "browse_runs", ["sitetag", "started_at"])
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_index("ix_browse_runs_sitetag_started", table_name="browse_runs")
|
||||||
|
op.drop_table("browse_runs")
|
||||||
|
|
@ -6,6 +6,7 @@ from collections import defaultdict
|
||||||
from typing import Annotated
|
from typing import Annotated
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||||
|
from pydantic import BaseModel
|
||||||
from sqlalchemy import exists, func, select
|
from sqlalchemy import exists, func, select
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
|
@ -384,3 +385,32 @@ def _movies_to_out(
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
return out
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
class MovieMergeOut(BaseModel):
|
||||||
|
keep_id: uuid.UUID
|
||||||
|
dropped_id: uuid.UUID
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/{keep_id}/merge/{drop_id}", response_model=MovieMergeOut)
|
||||||
|
def merge_duplicate_movie(
|
||||||
|
keep_id: uuid.UUID,
|
||||||
|
drop_id: uuid.UUID,
|
||||||
|
session: Annotated[Session, Depends(get_session)],
|
||||||
|
) -> MovieMergeOut:
|
||||||
|
"""Scal `drop_id` w `keep_id` z apki (przytrzymanie filmu, "merge into this one"),
|
||||||
|
jak przy scenach. Przenosi linki, obsade, tagi, rozdzialy i dane uzytkownikow
|
||||||
|
(movie_merge), kasuje `drop`. Zgloszenie f8c9486e (2026-10-16): te same filmy z dwoch
|
||||||
|
zrodel, ktorych automat nie zlaczyl ("Hardcore Gangbangs 8" i "... VIII")."""
|
||||||
|
from app.resolve.movie_merge import MovieMergeError, merge_movies
|
||||||
|
|
||||||
|
if keep_id == drop_id:
|
||||||
|
raise HTTPException(status_code=400, detail="keep_id == drop_id")
|
||||||
|
if session.get(Movie, keep_id) is None or session.get(Movie, drop_id) is None:
|
||||||
|
raise HTTPException(status_code=404, detail="movie not found")
|
||||||
|
try:
|
||||||
|
merge_movies(session, keep_id=keep_id, drop_id=drop_id, resolved_by="user_long_press_duplicate")
|
||||||
|
except MovieMergeError as e:
|
||||||
|
raise HTTPException(status_code=400, detail=str(e)) from e
|
||||||
|
session.commit()
|
||||||
|
return MovieMergeOut(keep_id=keep_id, dropped_id=drop_id)
|
||||||
|
|
|
||||||
|
|
@ -21,9 +21,11 @@ from typing import Annotated, Any
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends, HTTPException, Request, status
|
from fastapi import APIRouter, Depends, HTTPException, Request, status
|
||||||
from pydantic import BaseModel
|
from pydantic import BaseModel
|
||||||
|
from sqlalchemy import func, select
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from app.api.schemas import PlaybackSourceOut
|
from app.api.schemas import PlaybackSourceOut
|
||||||
|
from app.api.device import get_device_id
|
||||||
from app.auth import require_api_key
|
from app.auth import require_api_key
|
||||||
from app.db import get_session
|
from app.db import get_session
|
||||||
from app.extractors import (
|
from app.extractors import (
|
||||||
|
|
@ -111,6 +113,11 @@ _IP_BOUND_CDN_RE = re.compile(
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
#: Hostery filmow rozwiazywane wylacznie na telefonie (mobile/src/lib/packerHoster.ts).
|
||||||
|
#: sextapes.to (xtapes.la, 2026-10-15): HLS na cdn-centaurus zwiazany z IP rozwiazujacego.
|
||||||
|
_PHONE_RESOLVE_HOSTER_RE = re.compile(r"//(?:[a-z0-9-]+\.)*sextapes\.to/", re.IGNORECASE)
|
||||||
|
|
||||||
|
|
||||||
class StreamLink(BaseModel):
|
class StreamLink(BaseModel):
|
||||||
"""Pojedynczy variant stream URL (różne quality / kontener).
|
"""Pojedynczy variant stream URL (różne quality / kontener).
|
||||||
|
|
||||||
|
|
@ -221,7 +228,13 @@ def resolve_movie_playback(
|
||||||
target = pb.embed_url or pb.page_url
|
target = pb.embed_url or pb.page_url
|
||||||
stream: str | None = None
|
stream: str | None = None
|
||||||
try:
|
try:
|
||||||
stream = extract_stream_from_hoster(target, referer=referer)
|
# Hostery rozwiazywane na telefonie: link z VPS bylby zwiazany z IP serwera (403 na
|
||||||
|
# telefonie, potem cale wideo przez proxy). Telefon dostaje sam embed.
|
||||||
|
stream = (
|
||||||
|
None
|
||||||
|
if _PHONE_RESOLVE_HOSTER_RE.search(target)
|
||||||
|
else extract_stream_from_hoster(target, referer=referer)
|
||||||
|
)
|
||||||
except HosterDead as e:
|
except HosterDead as e:
|
||||||
# Hoster wprost mówi "video deleted", oznacz dead, NIE proponuj
|
# Hoster wprost mówi "video deleted", oznacz dead, NIE proponuj
|
||||||
# embed fallback (mobile ExoPlayer dostałby 404 HTML page i
|
# embed fallback (mobile ExoPlayer dostałby 404 HTML page i
|
||||||
|
|
@ -304,6 +317,18 @@ def resolve_movie_playback(
|
||||||
raw={"origin": pb.origin},
|
raw={"origin": pb.origin},
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
elif not links:
|
||||||
|
# Sam adres strony serwisu (parser nie znalazl hosterow, np. mangoporn po zmianie
|
||||||
|
# wygladu 10.2026). Ekstraktor nie ma tu czego szukac, telefon gra w WebView.
|
||||||
|
links.append(
|
||||||
|
StreamLink(
|
||||||
|
stream_url=None,
|
||||||
|
embed_url=pb.page_url,
|
||||||
|
quality=pb.quality,
|
||||||
|
type="hoster",
|
||||||
|
raw={"origin": pb.origin},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
if not links:
|
if not links:
|
||||||
raise HTTPException(status_code=502, detail="no playable links")
|
raise HTTPException(status_code=502, detail="no playable links")
|
||||||
|
|
@ -360,12 +385,88 @@ def mark_playback_dead(
|
||||||
raise HTTPException(status_code=404, detail="playback source not found for scene")
|
raise HTTPException(status_code=404, detail="playback source not found for scene")
|
||||||
if pb.dead_at is None:
|
if pb.dead_at is None:
|
||||||
tag = _requester_tag(request)
|
tag = _requester_tag(request)
|
||||||
|
others_alive = session.scalar(
|
||||||
|
select(func.count())
|
||||||
|
.select_from(PlaybackSource)
|
||||||
|
.where(
|
||||||
|
PlaybackSource.scene_id == scene_id,
|
||||||
|
PlaybackSource.id != pb.id,
|
||||||
|
PlaybackSource.dead_at.is_(None),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if not others_alive and not _scene_source_confirmed_dead(pb):
|
||||||
|
log.info(
|
||||||
|
"user mark-dead na ostatnim zrodle sceny %s odrzucone, serwer go nie potwierdzil (%s)",
|
||||||
|
pb.id, tag,
|
||||||
|
)
|
||||||
|
return
|
||||||
pb.dead_at = datetime.now(UTC)
|
pb.dead_at = datetime.now(UTC)
|
||||||
pb.dead_reason = f"user-marked dead (mobile long-press) {tag}"[:512]
|
pb.dead_reason = f"user-marked dead (mobile long-press) {tag}"[:512]
|
||||||
session.commit()
|
session.commit()
|
||||||
log.info("user marked playback %s dead (origin=%s %s)", pb.id, pb.origin, tag)
|
log.info("user marked playback %s dead (origin=%s %s)", pb.id, pb.origin, tag)
|
||||||
|
|
||||||
|
|
||||||
|
def _scene_source_confirmed_dead(pb: PlaybackSource) -> bool:
|
||||||
|
"""Czy serwer sam widzi, ze zrodlo sceny nie zyje. Uzywane, gdy przytrzymanie w apce
|
||||||
|
zabiloby OSTATNIE zywe zrodlo sceny (scena znika wtedy z list).
|
||||||
|
|
||||||
|
W 30 dni do 2026-10-15 z 617 zrodel oznaczonych w apce 490 zostawilo scene bez zadnego
|
||||||
|
zrodla, glownie porndoe (403 z sieci uzytkownika), galaxyporn, perverzija, javflix
|
||||||
|
(zgloszenie cbb87c8e: strona zyje, ekstraktor daje voe i doodstream). Martwe tylko,
|
||||||
|
gdy strona daje 404/410 albo ekstraktor mowi wprost, ze film usuniety; kazda
|
||||||
|
niepewnosc zostawia zrodlo.
|
||||||
|
"""
|
||||||
|
origin = pb.origin or ""
|
||||||
|
if not origin.startswith(("tube:", "pornapp:")):
|
||||||
|
return False
|
||||||
|
try:
|
||||||
|
try_extract(origin.split(":", 1)[1], pb.page_url)
|
||||||
|
except HosterDead:
|
||||||
|
return True
|
||||||
|
except TubePageError as e:
|
||||||
|
return e.status_code in (404, 410)
|
||||||
|
except Exception:
|
||||||
|
return False
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _movie_source_confirmed_dead(pb: Any) -> bool:
|
||||||
|
"""Czy serwer sam widzi, ze link filmu nie zyje. Uzywane, gdy przytrzymanie w apce
|
||||||
|
zabiloby OSTATNI link filmu.
|
||||||
|
|
||||||
|
Zgloszenie 284c9522 (2026-10-14): jedyny link "Italian She Male 44" oznaczyl telefon,
|
||||||
|
ktory nie dochodzil do wezla CDN playmogo, a plik gral (206 z innego IP). Film zostal bez
|
||||||
|
zadnego linku i apka chowala cala sekcje. Martwy jest tylko przy 404/410 strony embedu albo
|
||||||
|
gdy ekstraktor mowi wprost, ze film usuniety. Rodzina DoodStream ma blok "Not Found"
|
||||||
|
zawsze w HTML (ukryty), wiec tam zyje, dopoki strona ma pass_md5. Kazda niepewnosc
|
||||||
|
(blad sieci, captcha, pusty ekstraktor) zostawia link.
|
||||||
|
"""
|
||||||
|
import httpx
|
||||||
|
|
||||||
|
target = pb.embed_url or pb.page_url
|
||||||
|
try:
|
||||||
|
r = httpx.get(
|
||||||
|
target,
|
||||||
|
headers={"User-Agent": "Mozilla/5.0", "Referer": pb.page_url or target},
|
||||||
|
timeout=15.0,
|
||||||
|
follow_redirects=True,
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
log.info("kontrola linku filmu %s padla: %s", pb.id, e)
|
||||||
|
return False
|
||||||
|
if r.status_code in (404, 410):
|
||||||
|
return True
|
||||||
|
if r.status_code >= 400 or "pass_md5" in r.text:
|
||||||
|
return False
|
||||||
|
try:
|
||||||
|
extract_stream_from_hoster(target, referer=pb.page_url or target)
|
||||||
|
except HosterDead:
|
||||||
|
return True
|
||||||
|
except Exception:
|
||||||
|
return False
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
@movies_router.post(
|
@movies_router.post(
|
||||||
"/{movie_id}/playback/{playback_id}/mark-dead",
|
"/{movie_id}/playback/{playback_id}/mark-dead",
|
||||||
status_code=status.HTTP_204_NO_CONTENT,
|
status_code=status.HTTP_204_NO_CONTENT,
|
||||||
|
|
@ -384,17 +485,62 @@ def mark_movie_playback_dead(
|
||||||
raise HTTPException(status_code=404, detail="movie playback source not found")
|
raise HTTPException(status_code=404, detail="movie playback source not found")
|
||||||
if pb.dead_at is None:
|
if pb.dead_at is None:
|
||||||
tag = _requester_tag(request)
|
tag = _requester_tag(request)
|
||||||
|
others_alive = session.scalar(
|
||||||
|
select(func.count())
|
||||||
|
.select_from(MoviePlaybackSource)
|
||||||
|
.where(
|
||||||
|
MoviePlaybackSource.movie_id == movie_id,
|
||||||
|
MoviePlaybackSource.id != pb.id,
|
||||||
|
MoviePlaybackSource.dead_at.is_(None),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if not others_alive and not _movie_source_confirmed_dead(pb):
|
||||||
|
log.info(
|
||||||
|
"user mark-dead na ostatnim zrodle filmu %s odrzucone, serwer go nie potwierdzil (%s)",
|
||||||
|
pb.id, tag,
|
||||||
|
)
|
||||||
|
return
|
||||||
pb.dead_at = datetime.now(UTC)
|
pb.dead_at = datetime.now(UTC)
|
||||||
pb.dead_reason = f"user-marked dead (mobile long-press) {tag}"[:512]
|
pb.dead_reason = f"user-marked dead (mobile long-press) {tag}"[:512]
|
||||||
session.commit()
|
session.commit()
|
||||||
log.info("user marked movie playback %s dead (origin=%s %s)", pb.id, pb.origin, tag)
|
log.info("user marked movie playback %s dead (origin=%s %s)", pb.id, pb.origin, tag)
|
||||||
|
|
||||||
|
|
||||||
|
def _record_resolve_failure(
|
||||||
|
session: Session, pb: PlaybackSource, kind: str, detail: str, device_id: str
|
||||||
|
) -> None:
|
||||||
|
"""Blad rozwiazania po stronie serwera do playback_events.
|
||||||
|
|
||||||
|
Telefon po 503/502 z resolve pokazuje alert i nie dochodzi do odtwarzacza, wiec nic nie
|
||||||
|
zglaszal. Hentaimama padala tak od zmiany strony i miala zero wierszy w telemetrii (ani
|
||||||
|
sukcesow, ani bledow), wiec zaden alarm jej nie widzial (2026-10-15).
|
||||||
|
"""
|
||||||
|
from app.models.playback_event import PlaybackEvent
|
||||||
|
|
||||||
|
try:
|
||||||
|
session.add(
|
||||||
|
PlaybackEvent(
|
||||||
|
origin=pb.origin[:64],
|
||||||
|
scene_id=pb.scene_id,
|
||||||
|
status="error",
|
||||||
|
error_kind=kind,
|
||||||
|
error_detail=detail[:512],
|
||||||
|
via="backend",
|
||||||
|
device_id=device_id,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
except Exception as e:
|
||||||
|
session.rollback()
|
||||||
|
log.info("zapis bledu resolve do playback_events padl: %s", e)
|
||||||
|
|
||||||
|
|
||||||
@router.post("/{scene_id}/playback/{playback_id}/resolve", response_model=ResolveOut)
|
@router.post("/{scene_id}/playback/{playback_id}/resolve", response_model=ResolveOut)
|
||||||
def resolve_playback(
|
def resolve_playback(
|
||||||
scene_id: uuid.UUID,
|
scene_id: uuid.UUID,
|
||||||
playback_id: uuid.UUID,
|
playback_id: uuid.UUID,
|
||||||
session: Annotated[Session, Depends(get_session)],
|
session: Annotated[Session, Depends(get_session)],
|
||||||
|
device_id: Annotated[str, Depends(get_device_id)],
|
||||||
) -> ResolveOut:
|
) -> ResolveOut:
|
||||||
pb = session.get(PlaybackSource, playback_id)
|
pb = session.get(PlaybackSource, playback_id)
|
||||||
if pb is None or pb.scene_id != scene_id:
|
if pb is None or pb.scene_id != scene_id:
|
||||||
|
|
@ -438,6 +584,9 @@ def resolve_playback(
|
||||||
log.info("marked playback %s dead (origin=%s reason=%s)", pb.id, pb.origin, reason)
|
log.info("marked playback %s dead (origin=%s reason=%s)", pb.id, pb.origin, reason)
|
||||||
raise HTTPException(status_code=410, detail=f"playback dead: {reason}") from e
|
raise HTTPException(status_code=410, detail=f"playback dead: {reason}") from e
|
||||||
log.warning("tube fetch http error %s for %s", e.status_code, pb.page_url)
|
log.warning("tube fetch http error %s for %s", e.status_code, pb.page_url)
|
||||||
|
_record_resolve_failure(
|
||||||
|
session, pb, "resolve_http", f"tube fetch HTTP {e.status_code}", device_id
|
||||||
|
)
|
||||||
raise HTTPException(
|
raise HTTPException(
|
||||||
status_code=502,
|
status_code=502,
|
||||||
detail=f"tube fetch failed: HTTP {e.status_code}",
|
detail=f"tube fetch failed: HTTP {e.status_code}",
|
||||||
|
|
@ -457,6 +606,7 @@ def resolve_playback(
|
||||||
"extractor None for playback %s (origin=%s) — transient, not marking dead",
|
"extractor None for playback %s (origin=%s) — transient, not marking dead",
|
||||||
pb.id, pb.origin,
|
pb.id, pb.origin,
|
||||||
)
|
)
|
||||||
|
_record_resolve_failure(session, pb, "resolve_none", "extractor returned nothing", device_id)
|
||||||
# 503 (not 410!) żeby mobile NIE pokazało "Tube usunął ten film", ten kod
|
# 503 (not 410!) żeby mobile NIE pokazało "Tube usunął ten film", ten kod
|
||||||
# jest dla permanent removal. 503 = transient, user może retry.
|
# jest dla permanent removal. 503 = transient, user może retry.
|
||||||
# Sentry filtruje HTTPException 502/503/504 w `_sentry_before_send` (main.py),
|
# Sentry filtruje HTTPException 502/503/504 w `_sentry_before_send` (main.py),
|
||||||
|
|
|
||||||
|
|
@ -764,7 +764,21 @@ def list_scenes(
|
||||||
).all()
|
).all()
|
||||||
id_by_slug = {slug: tid for slug, tid, _ in tag_rows}
|
id_by_slug = {slug: tid for slug, tid, _ in tag_rows}
|
||||||
if sort == "release_date" and tag_rows:
|
if sort == "release_date" and tag_rows:
|
||||||
tag_sort_id = min(tag_rows, key=lambda r: r[2] or 0)[1]
|
smallest = min(tag_rows, key=lambda r: r[2] or 0)
|
||||||
|
tag_sort_id = smallest[1]
|
||||||
|
# Ekran serwisu z tagiem: idziemy po mniejszej stronie. Gdy serwis ma mniej scen
|
||||||
|
# niz najmniejszy tag, przejscie po indeksie tagu przez caly katalog szukalo
|
||||||
|
# rzadkich scen serwisu (fullmovies 53 tys. + blowjob 555 tys.: 2-17 s,
|
||||||
|
# zgloszenie 80da1ce7). Wtedy sort po indeksie serwisu (origin_sort nizej), a tag
|
||||||
|
# jako zwykly filtr: blowjob 36 ms, rzadki tag z przejsciem calego serwisu ~0,1 s.
|
||||||
|
if origin and ":" in origin and not q:
|
||||||
|
from app.models.source_stats import SourceStats
|
||||||
|
|
||||||
|
origin_scenes = session.execute(
|
||||||
|
select(SourceStats.scenes).where(SourceStats.origin == origin)
|
||||||
|
).scalar()
|
||||||
|
if origin_scenes and origin_scenes < (smallest[2] or 0):
|
||||||
|
tag_sort_id = None
|
||||||
for slug in tag_slug_list:
|
for slug in tag_slug_list:
|
||||||
tag_id = id_by_slug.get(slug)
|
tag_id = id_by_slug.get(slug)
|
||||||
if tag_id is None:
|
if tag_id is None:
|
||||||
|
|
|
||||||
|
|
@ -267,6 +267,12 @@ def _build_headers(referer: str | None) -> dict[str, str]:
|
||||||
"User-Agent": DEFAULT_UA,
|
"User-Agent": DEFAULT_UA,
|
||||||
"Accept": "*/*",
|
"Accept": "*/*",
|
||||||
"Accept-Language": "en-US,en;q=0.9",
|
"Accept-Language": "en-US,en;q=0.9",
|
||||||
|
# Bez tego httpx prosi o gzip, a `proxy_stream` przepuszcza cialo surowe (aiter_raw)
|
||||||
|
# i zdejmuje Content-Encoding (HOP_BY_HOP): odtwarzacz dostawal skompresowane bajty
|
||||||
|
# manifestu bez naglowka i konczyl na "Input does not start with the #EXTM3U header"
|
||||||
|
# (xtremestream przez Cloudflare, 189 zdarzen w 7 dni do 2026-10-14). Strumienie
|
||||||
|
# wideo i tak sie nie kompresuja, wiec identity nic nie kosztuje.
|
||||||
|
"Accept-Encoding": "identity",
|
||||||
}
|
}
|
||||||
if referer:
|
if referer:
|
||||||
h["Referer"] = _ascii_safe_url(referer)
|
h["Referer"] = _ascii_safe_url(referer)
|
||||||
|
|
@ -445,6 +451,15 @@ async def proxy_image(
|
||||||
media_type="image/jpeg",
|
media_type="image/jpeg",
|
||||||
)
|
)
|
||||||
ct = r.headers.get("content-type", "image/jpeg")
|
ct = r.headers.get("content-type", "image/jpeg")
|
||||||
|
# Atrapy zamiast obrazka (zgloszenia eb20f1c0/4d8d5daf, latestpornvideo): pixhost
|
||||||
|
# przekierowuje usuniete obrazki na baner `/image/removed.png`, a link do strony
|
||||||
|
# podgladu (`pixhost.to/show/...`) daje HTML. Oba szly do apki jako 200, wiec kafelek
|
||||||
|
# pokazywal baner albo byl pusty. 404 = apka rysuje swoj zastepnik.
|
||||||
|
final_url = str(getattr(r, "url", "") or "")
|
||||||
|
# Tylko text/*: czesc CDN-ow (S3) podaje prawdziwe obrazki jako octet-stream.
|
||||||
|
if final_url.endswith("/image/removed.png") or ct.lower().startswith("text/"):
|
||||||
|
log.info("img proxy: atrapa zamiast obrazka (%s, %s) dla %s", ct, final_url[:80], target)
|
||||||
|
return Response(content=b"", status_code=404, media_type="image/jpeg")
|
||||||
return Response(
|
return Response(
|
||||||
content=r.content,
|
content=r.content,
|
||||||
media_type=ct,
|
media_type=ct,
|
||||||
|
|
@ -785,6 +800,32 @@ async def proxy_hls_manifest(token: str, _basename: str) -> Response:
|
||||||
# Hoster-producent: manifest nie ma własnego URL-a (loadvid oddaje jego TREŚĆ
|
# Hoster-producent: manifest nie ma własnego URL-a (loadvid oddaje jego TREŚĆ
|
||||||
# dopiero na POST /videos/resolve-token), więc zamiast GET-a wołamy producenta.
|
# dopiero na POST /videos/resolve-token), więc zamiast GET-a wołamy producenta.
|
||||||
producer = payload.get("mp")
|
producer = payload.get("mp")
|
||||||
|
if producer == "xtremestream":
|
||||||
|
# Manifest ma segmenty AES-128 i klucz pod WZGLEDNYM `key.bin` (ten sam token, ta
|
||||||
|
# sama trasa), wiec nie absolutyzujemy: segmenty juz sa pelnymi adresami CDN, a klucz
|
||||||
|
# telefon pobiera spod `/proxy/hls/<token>/key.bin`. Szczegoly: hosters/xtremestream.py.
|
||||||
|
import anyio
|
||||||
|
from app.extractors.hosters import xtremestream
|
||||||
|
|
||||||
|
if _basename.endswith("key.bin"):
|
||||||
|
key = await anyio.to_thread.run_sync(functools.partial(xtremestream.fetch_key, target))
|
||||||
|
if not key:
|
||||||
|
raise HTTPException(status_code=502, detail="key fetch failed")
|
||||||
|
return Response(
|
||||||
|
content=key,
|
||||||
|
media_type="application/octet-stream",
|
||||||
|
headers=_with_cast_cors({"Cache-Control": "no-store"}),
|
||||||
|
)
|
||||||
|
manifest = await anyio.to_thread.run_sync(
|
||||||
|
functools.partial(xtremestream.build_manifest, target, key_uri="key.bin")
|
||||||
|
)
|
||||||
|
if manifest is None:
|
||||||
|
raise HTTPException(status_code=502, detail="manifest producer failed")
|
||||||
|
return Response(
|
||||||
|
content=manifest,
|
||||||
|
media_type="application/vnd.apple.mpegurl",
|
||||||
|
headers=_with_cast_cors({"Cache-Control": "no-store"}),
|
||||||
|
)
|
||||||
if producer:
|
if producer:
|
||||||
manifest = await _produce_manifest(producer, target)
|
manifest = await _produce_manifest(producer, target)
|
||||||
if manifest is None:
|
if manifest is None:
|
||||||
|
|
@ -871,7 +912,10 @@ async def proxy_stream(
|
||||||
# Powód: pornhat `get_file/.../<id>.mp4/` 302 → CDN m3u8 manifest mimo `.mp4`
|
# Powód: pornhat `get_file/.../<id>.mp4/` 302 → CDN m3u8 manifest mimo `.mp4`
|
||||||
# w path. Bez content-type check proxy traktuje jako binary, mobile dostaje
|
# w path. Bez content-type check proxy traktuje jako binary, mobile dostaje
|
||||||
# m3u8 z RAW CDN URLs (IP-bound do VPS) → "no extractors" w ExoPlayer.
|
# m3u8 z RAW CDN URLs (IP-bound do VPS) → "no extractors" w ExoPlayer.
|
||||||
path_suggests_m3u8 = path_lower.endswith(".m3u8")
|
# `_basename` to `play.m3u8` nadane w `_proxify_link` z typu zrodla, wiec manifest bez
|
||||||
|
# `.m3u8` w sciezce upstreamu (xtremestream `xs1.php?data=`) tez trafia do przepisania,
|
||||||
|
# zamiast leciec dalej jako plik z absolutnymi adresami segmentow.
|
||||||
|
path_suggests_m3u8 = path_lower.endswith(".m3u8") or _basename.lower().endswith(".m3u8")
|
||||||
|
|
||||||
client = httpx.AsyncClient(timeout=timeout, follow_redirects=True)
|
client = httpx.AsyncClient(timeout=timeout, follow_redirects=True)
|
||||||
try:
|
try:
|
||||||
|
|
|
||||||
|
|
@ -254,7 +254,10 @@ def score_candidate_v1(c: Candidate, aff: Affinity, now: datetime | None = None)
|
||||||
|
|
||||||
#: Odsetek urzadzen na starej punktacji (grupa kontrolna). Przydzial staly po hashu id
|
#: Odsetek urzadzen na starej punktacji (grupa kontrolna). Przydzial staly po hashu id
|
||||||
#: urzadzenia, wiec do porownania wystarczy ten sam hash przy analizie zdarzen.
|
#: urzadzenia, wiec do porownania wystarczy ten sam hash przy analizie zdarzen.
|
||||||
V1_CONTROL_PERCENT = 25
|
#: 0 od 2026-10-15: test zakonczony (scripts/edge_ab_report.py, 973 urzadzenia). A/A przed
|
||||||
|
#: wdrozeniem 13,1% vs 13,2% dokonczonych; po: nowa 14,6% vs 12,5%, lajki 3,9% vs 2,9%,
|
||||||
|
#: ucieczki 45% vs 50%. Stara punktacja zostaje w kodzie na wypadek ponownego testu.
|
||||||
|
V1_CONTROL_PERCENT = 0
|
||||||
|
|
||||||
|
|
||||||
def uses_v1_scoring(device_id: str) -> bool:
|
def uses_v1_scoring(device_id: str) -> bool:
|
||||||
|
|
|
||||||
|
|
@ -35,10 +35,12 @@ def get_movie_connectors() -> list[tuple[str, type]]:
|
||||||
"""
|
"""
|
||||||
from app.connectors.dooplay import (
|
from app.connectors.dooplay import (
|
||||||
MangopornConnector,
|
MangopornConnector,
|
||||||
|
PandamoviesConnector,
|
||||||
StreampornVipConnector,
|
StreampornVipConnector,
|
||||||
)
|
)
|
||||||
from app.connectors.freeomovie import FreeoMovieConnector
|
from app.connectors.freeomovie import FreeoMovieConnector
|
||||||
from app.connectors.paradisehill import ParadisehillConnector
|
from app.connectors.paradisehill import ParadisehillConnector
|
||||||
|
from app.connectors.xtapes import XtapesConnector
|
||||||
from app.connectors.yourdailypornmovies import YourDailyPornMoviesConnector
|
from app.connectors.yourdailypornmovies import YourDailyPornMoviesConnector
|
||||||
|
|
||||||
# Kolejność ingestu: paradisehill FIRST (canonical primary, mirrory się do
|
# Kolejność ingestu: paradisehill FIRST (canonical primary, mirrory się do
|
||||||
|
|
@ -54,17 +56,11 @@ def get_movie_connectors() -> list[tuple[str, type]]:
|
||||||
("paradisehill", ParadisehillConnector),
|
("paradisehill", ParadisehillConnector),
|
||||||
("mangoporn", MangopornConnector),
|
("mangoporn", MangopornConnector),
|
||||||
("streampornvip", StreampornVipConnector),
|
("streampornvip", StreampornVipConnector),
|
||||||
# pandamovies WYLACZONY 2026-08-20: serwis padl, zostal im tylko zapamietany
|
# pandamovies: wylaczony 2026-08-20 (serwis padl, kazda sciezka poza strona
|
||||||
# fronton. Strona glowna oddaje 291 kB i wypisuje 56 filmow, ale KAZDA inna
|
# glowna dawala 404), wlaczony z powrotem 2026-10-16 po przeprowadzce na
|
||||||
# sciezka to 404: listing /movies/ (ten, ktory czytal konektor), /movies/page/2/
|
# pandamovies.org: listing /movies/ 27 filmow na strone, 7 hosterow na film,
|
||||||
# linkowana przez ich wlasny szablon, /genre/* i /release-year/* z ich menu,
|
# dziala wprost z VPS bez proxy. Ten sam motyw "vid" co mangoporn i streamporn.li.
|
||||||
# sitemapa, a przede wszystkim same strony filmow (0 na 6 dzialajacych).
|
("pandamovies", PandamoviesConnector),
|
||||||
# To nie blokada na nasze IP: przez residential proxy wychodzi identyczne 404
|
|
||||||
# (te same 796 bajtow). Konektor konczyl runy jako success z zerem rekordow,
|
|
||||||
# co falszowalo obraz w monitoringu. Playback tego, co juz mamy, dziala dalej,
|
|
||||||
# bo zrodla wskazuja na hostery (doply, luluvid, seekplayer), nie na ten serwis.
|
|
||||||
# Klasa zostaje w dooplay.py dla backrefu origins `pandamovies:*`.
|
|
||||||
# ("pandamovies", PandamoviesConnector),
|
|
||||||
# freeomovie.to (bestia theme, nie dooplay), mirror: świeże filmy DVD,
|
# freeomovie.to (bestia theme, nie dooplay), mirror: świeże filmy DVD,
|
||||||
# title-trigram attach do canonical + playback z TABS. Ocena 2026-07-02.
|
# title-trigram attach do canonical + playback z TABS. Ocena 2026-07-02.
|
||||||
("freeomovie", FreeoMovieConnector),
|
("freeomovie", FreeoMovieConnector),
|
||||||
|
|
@ -76,4 +72,7 @@ def get_movie_connectors() -> list[tuple[str, type]]:
|
||||||
# id produktu Adult Empire. Bez długości. Playback: voe + mixdrop (oba mają
|
# id produktu Adult Empire. Bez długości. Playback: voe + mixdrop (oba mają
|
||||||
# gotowe moduły hosterów), vidara.to nieznana, leci jako zwykły embed.
|
# gotowe moduły hosterów), vidara.to nieznana, leci jako zwykły embed.
|
||||||
("yourdailypornmovies", YourDailyPornMoviesConnector),
|
("yourdailypornmovies", YourDailyPornMoviesConnector),
|
||||||
|
# xtapes.la (WordPress, REST API), kategoria Full Movies: 3-5 filmow dziennie,
|
||||||
|
# czesto w dniu premiery. Ocena 2026-10-15, szczegoly w app/connectors/xtapes.py.
|
||||||
|
("xtapes", XtapesConnector),
|
||||||
]
|
]
|
||||||
|
|
|
||||||
|
|
@ -95,12 +95,22 @@ class BaseBrowseScraper(BaseDirectTubeScraper, abc.ABC):
|
||||||
|
|
||||||
log.info("%s browse page %d: %d scene URLs", self.sitetag, page, len(urls))
|
log.info("%s browse page %d: %d scene URLs", self.sitetag, page, len(urls))
|
||||||
out: list[RawScene] = []
|
out: list[RawScene] = []
|
||||||
|
# Pobrania szczegolow, ktore padly (wyjatek albo HTTP >= 400). Gdy padly WSZYSTKIE,
|
||||||
|
# to dlawienie serwisu, a nie pusta strona: zwracamy None (sprobuj pozniej), inaczej
|
||||||
|
# deep-crawl liczy to do pustych stron pod rzad i oznacza koniec katalogu.
|
||||||
|
# naughtymachinima: exhausted na stronie 21 z ~1800 (2026-08-21, wykryte 10-15).
|
||||||
|
failed = 0
|
||||||
for scene_url in urls:
|
for scene_url in urls:
|
||||||
try:
|
try:
|
||||||
res = browser_get(scene_url, timeout=self._timeout, proxy=self._proxy)
|
res = browser_get(scene_url, timeout=self._timeout, proxy=self._proxy)
|
||||||
detail_html = res.text if hasattr(res, "text") else res
|
detail_html = res.text if hasattr(res, "text") else res
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.info("%s detail fetch failed %s: %s", self.sitetag, scene_url, e)
|
log.info("%s detail fetch failed %s: %s", self.sitetag, scene_url, e)
|
||||||
|
failed += 1
|
||||||
|
continue
|
||||||
|
if getattr(res, "status_code", 200) >= 400:
|
||||||
|
log.info("%s detail HTTP %s %s", self.sitetag, res.status_code, scene_url)
|
||||||
|
failed += 1
|
||||||
continue
|
continue
|
||||||
try:
|
try:
|
||||||
raw = self._parse_detail(scene_url, detail_html)
|
raw = self._parse_detail(scene_url, detail_html)
|
||||||
|
|
@ -109,6 +119,9 @@ class BaseBrowseScraper(BaseDirectTubeScraper, abc.ABC):
|
||||||
continue
|
continue
|
||||||
if raw is not None:
|
if raw is not None:
|
||||||
out.append(raw)
|
out.append(raw)
|
||||||
|
if not out and failed == len(urls):
|
||||||
|
log.warning("%s browse page %d: wszystkie %d pobrania padly", self.sitetag, page, failed)
|
||||||
|
return None
|
||||||
return out
|
return out
|
||||||
|
|
||||||
def latest_scenes(self, *, max_pages: int = 5) -> Iterator[RawScene]:
|
def latest_scenes(self, *, max_pages: int = 5) -> Iterator[RawScene]:
|
||||||
|
|
|
||||||
|
|
@ -7,6 +7,7 @@ Identyczny engine co hdporn.gg (KVS sponsor_groups stack): `/videos/<slug>/`,
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import re
|
import re
|
||||||
|
from datetime import date
|
||||||
|
|
||||||
from app.connectors.base import (
|
from app.connectors.base import (
|
||||||
RawFingerprint,
|
RawFingerprint,
|
||||||
|
|
@ -28,6 +29,7 @@ _SCENE_URL_RE = re.compile(r'href="(https://www\.fullmovies\.xxx/videos/[a-z0-9\
|
||||||
_NETWORK_LINK_RE = re.compile(
|
_NETWORK_LINK_RE = re.compile(
|
||||||
r'href="https://www\.fullmovies\.xxx/networks/([a-z0-9\-]+)/"[^>]*>([^<]+)', re.IGNORECASE
|
r'href="https://www\.fullmovies\.xxx/networks/([a-z0-9\-]+)/"[^>]*>([^<]+)', re.IGNORECASE
|
||||||
)
|
)
|
||||||
|
_UPLOAD_DATE_RE = re.compile(r'"uploadDate"\s*:\s*"(\d{4})-(\d{2})-(\d{2})')
|
||||||
_MODEL_LINK_RE = re.compile(
|
_MODEL_LINK_RE = re.compile(
|
||||||
r'href="https://www\.fullmovies\.xxx/models/([a-z0-9\-]+)/"[^>]*>([^<]+)', re.IGNORECASE
|
r'href="https://www\.fullmovies\.xxx/models/([a-z0-9\-]+)/"[^>]*>([^<]+)', re.IGNORECASE
|
||||||
)
|
)
|
||||||
|
|
@ -68,6 +70,14 @@ class FullmoviesScraper(BaseBrowseScraper):
|
||||||
dur_meta = meta_content(detail_html, property="video:duration")
|
dur_meta = meta_content(detail_html, property="video:duration")
|
||||||
if dur_meta and dur_meta.isdigit():
|
if dur_meta and dur_meta.isdigit():
|
||||||
duration_sec = int(dur_meta)
|
duration_sec = int(dur_meta)
|
||||||
|
# Data z JSON-LD `"uploadDate": "2026-10-13T20:48:46Z"` (audyt 2026-10-14: 0 dat).
|
||||||
|
release_date = None
|
||||||
|
um = _UPLOAD_DATE_RE.search(detail_html)
|
||||||
|
if um:
|
||||||
|
try:
|
||||||
|
release_date = date(int(um.group(1)), int(um.group(2)), int(um.group(3)))
|
||||||
|
except ValueError:
|
||||||
|
release_date = None
|
||||||
|
|
||||||
# Studio z PREFIKSU tytułu ("Studio - Scene Title"), nie z sidebara /networks/.
|
# Studio z PREFIKSU tytułu ("Studio - Scene Title"), nie z sidebara /networks/.
|
||||||
# Sidebar listuje WSZYSTKIE sieci → `_NETWORK_LINK_RE.finditer().first()` zawsze
|
# Sidebar listuje WSZYSTKIE sieci → `_NETWORK_LINK_RE.finditer().first()` zawsze
|
||||||
|
|
@ -122,6 +132,7 @@ class FullmoviesScraper(BaseBrowseScraper):
|
||||||
external_id=f"{self.sitetag}:{scene_url}",
|
external_id=f"{self.sitetag}:{scene_url}",
|
||||||
title=title,
|
title=title,
|
||||||
description=description,
|
description=description,
|
||||||
|
release_date=release_date,
|
||||||
duration_sec=duration_sec,
|
duration_sec=duration_sec,
|
||||||
url=scene_url,
|
url=scene_url,
|
||||||
studio=studio,
|
studio=studio,
|
||||||
|
|
|
||||||
|
|
@ -62,6 +62,9 @@ _IFRAME_RE = re.compile(r'<iframe[^>]+src="([^"]+)"', re.IGNORECASE)
|
||||||
_TITLE_STUDIO_DATE_RE = re.compile(r"^(?P<studio>[A-Za-z0-9][A-Za-z0-9._-]{1,30})\s+(?P<y>\d{2})\s+(?P<m>\d{2})\s+(?P<d>\d{2})\s+(?P<rest>.+)$")
|
_TITLE_STUDIO_DATE_RE = re.compile(r"^(?P<studio>[A-Za-z0-9][A-Za-z0-9._-]{1,30})\s+(?P<y>\d{2})\s+(?P<m>\d{2})\s+(?P<d>\d{2})\s+(?P<rest>.+)$")
|
||||||
# `[Onlyfans] Some Title` / `[Darkkotv] …`
|
# `[Onlyfans] Some Title` / `[Darkkotv] …`
|
||||||
_TITLE_BRACKET_RE = re.compile(r"^\[(?P<studio>[^\]]{2,30})\]\s*(?P<rest>.+)$")
|
_TITLE_BRACKET_RE = re.compile(r"^\[(?P<studio>[^\]]{2,30})\]\s*(?P<rest>.+)$")
|
||||||
|
# Data z JSON-LD strony, gdy tytul jej nie niesie (audyt 2026-10-14: 66% scen z 7 dni
|
||||||
|
# bez daty i bez dlugosci; data jest w `"datePublished":"2026-10-13T18:40:00+03:00"`).
|
||||||
|
_JSONLD_DATE_RE = re.compile(r'"datePublished"\s*:\s*"(\d{4})-(\d{2})-(\d{2})')
|
||||||
# `… [2018-09-28]`
|
# `… [2018-09-28]`
|
||||||
_TITLE_ISO_DATE_RE = re.compile(r"\[(\d{4})-(\d{2})-(\d{2})\]")
|
_TITLE_ISO_DATE_RE = re.compile(r"\[(\d{4})-(\d{2})-(\d{2})\]")
|
||||||
|
|
||||||
|
|
@ -178,6 +181,11 @@ class GalaxyPornScraper(BaseBrowseScraper):
|
||||||
release_date = date(int(m.group(1)), int(m.group(2)), int(m.group(3)))
|
release_date = date(int(m.group(1)), int(m.group(2)), int(m.group(3)))
|
||||||
except ValueError:
|
except ValueError:
|
||||||
release_date = None
|
release_date = None
|
||||||
|
if release_date is None and (m := _JSONLD_DATE_RE.search(detail_html)):
|
||||||
|
try:
|
||||||
|
release_date = date(int(m.group(1)), int(m.group(2)), int(m.group(3)))
|
||||||
|
except ValueError:
|
||||||
|
release_date = None
|
||||||
# Guard: prefiks nie może być nazwiskiem performera (wtedy to nie studio).
|
# Guard: prefiks nie może być nazwiskiem performera (wtedy to nie studio).
|
||||||
if studio_name and slugify(studio_name) not in seen_p:
|
if studio_name and slugify(studio_name) not in seen_p:
|
||||||
studio = RawStudio(
|
studio = RawStudio(
|
||||||
|
|
|
||||||
|
|
@ -10,6 +10,7 @@ factory, wyłącz; analogia do shyfap).
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import re
|
import re
|
||||||
|
from datetime import date
|
||||||
from urllib.parse import urljoin
|
from urllib.parse import urljoin
|
||||||
|
|
||||||
from app.connectors.base import (
|
from app.connectors.base import (
|
||||||
|
|
@ -32,6 +33,7 @@ _SCENE_URL_RE = re.compile(r'href="(https://www\.hdporn\.gg/videos/[a-z0-9\-]+/)
|
||||||
_NETWORK_LINK_RE = re.compile(
|
_NETWORK_LINK_RE = re.compile(
|
||||||
r'href="https://www\.hdporn\.gg/networks/([a-z0-9\-]+)/"[^>]*>([^<]+)', re.IGNORECASE
|
r'href="https://www\.hdporn\.gg/networks/([a-z0-9\-]+)/"[^>]*>([^<]+)', re.IGNORECASE
|
||||||
)
|
)
|
||||||
|
_UPLOAD_DATE_RE = re.compile(r'"uploadDate"\s*:\s*"(\d{4})-(\d{2})-(\d{2})')
|
||||||
_MODEL_LINK_RE = re.compile(
|
_MODEL_LINK_RE = re.compile(
|
||||||
r'href="https://www\.hdporn\.gg/models/([a-z0-9\-]+)/"[^>]*>([^<]+)', re.IGNORECASE
|
r'href="https://www\.hdporn\.gg/models/([a-z0-9\-]+)/"[^>]*>([^<]+)', re.IGNORECASE
|
||||||
)
|
)
|
||||||
|
|
@ -75,6 +77,14 @@ class HDPornGGScraper(BaseBrowseScraper):
|
||||||
dur_meta = meta_content(detail_html, property="video:duration")
|
dur_meta = meta_content(detail_html, property="video:duration")
|
||||||
if dur_meta and dur_meta.isdigit():
|
if dur_meta and dur_meta.isdigit():
|
||||||
duration_sec = int(dur_meta)
|
duration_sec = int(dur_meta)
|
||||||
|
# Data z JSON-LD `"uploadDate": "2026-10-13T16:51:32Z"` (audyt 2026-10-14: 0 dat).
|
||||||
|
release_date = None
|
||||||
|
um = _UPLOAD_DATE_RE.search(detail_html)
|
||||||
|
if um:
|
||||||
|
try:
|
||||||
|
release_date = date(int(um.group(1)), int(um.group(2)), int(um.group(3)))
|
||||||
|
except ValueError:
|
||||||
|
release_date = None
|
||||||
|
|
||||||
# Studio z PREFIKSU tytułu ("Studio - Scene Title"), nie z sidebara /networks/.
|
# Studio z PREFIKSU tytułu ("Studio - Scene Title"), nie z sidebara /networks/.
|
||||||
# Sidebar listuje WSZYSTKIE sieci → pierwszy match zawsze ten sam (Brazzers) dla
|
# Sidebar listuje WSZYSTKIE sieci → pierwszy match zawsze ten sam (Brazzers) dla
|
||||||
|
|
@ -130,6 +140,7 @@ class HDPornGGScraper(BaseBrowseScraper):
|
||||||
external_id=f"{self.sitetag}:{scene_url}",
|
external_id=f"{self.sitetag}:{scene_url}",
|
||||||
title=title,
|
title=title,
|
||||||
description=description,
|
description=description,
|
||||||
|
release_date=release_date,
|
||||||
duration_sec=duration_sec,
|
duration_sec=duration_sec,
|
||||||
url=scene_url,
|
url=scene_url,
|
||||||
studio=studio,
|
studio=studio,
|
||||||
|
|
|
||||||
|
|
@ -1,19 +1,19 @@
|
||||||
"""hentaimama.io — browse scraper. Dodany 2026-08-04. Pierwsze źródło wertykału hentai.
|
"""hentaimama.io \u2014 browse scraper. Dodany 2026-08-04. Pierwsze źródło wertykału hentai.
|
||||||
|
|
||||||
**Ten wertykał rządzi się inaczej niż reszta i to jest celowe.** Dla hentai NIE ISTNIEJE
|
**Ten wertykał rządzi się inaczej niż reszta i to jest celowe.** Dla hentai NIE ISTNIEJE
|
||||||
kanon: TPDB i StashDB nie mają animacji ani postaci rysowanych, więc każda scena stąd
|
kanon: TPDB i StashDB nie mają animacji ani postaci rysowanych, więc każda scena stąd
|
||||||
jest technicznie orphanem i to jest POPRAWNE, nie błędne. Kryterium „fabryka orphanów",
|
jest technicznie orphanem i to jest POPRAWNE, nie błędne. Kryterium „fabryka orphanów",
|
||||||
którym oceniamy tuby z żywymi aktorami, tutaj po prostu nie ma zastosowania — nie ma też
|
którym oceniamy tuby z żywymi aktorami, tutaj po prostu nie ma zastosowania \u2014 nie ma też
|
||||||
obsady, więc żadnej bramki na performera. Miarą jakości jest komplet seria + numer
|
obsady, więc żadnej bramki na performera. Miarą jakości jest komplet seria + numer
|
||||||
odcinka + gatunki, a docelowo dedup MIĘDZY stronami hentai (re-hostują się nawzajem).
|
odcinka + gatunki, a docelowo dedup MIĘDZY stronami hentai (re-hostują się nawzajem).
|
||||||
|
|
||||||
Jednostka treści to ODCINEK (`/episodes/<seria>-episode-N/`), nie seria. Każdy jest
|
Jednostka treści to ODCINEK (`/episodes/<seria>-episode-N/`), nie seria. Każdy jest
|
||||||
samodzielną pozycją z własnym plakatem i playbackiem, więc mapuje się na scenę tak samo
|
samodzielną pozycją z własnym plakatem i playbackiem, więc mapuje się na scenę tak samo
|
||||||
jak w wertykale JAV — bez zmian w schemacie.
|
jak w wertykale JAV \u2014 bez zmian w schemacie.
|
||||||
|
|
||||||
Motyw jest z rodziny Dooplay, ale `DooplayConnector` się NIE nadaje: nie ma tu
|
Motyw jest z rodziny Dooplay, ale `DooplayConnector` się NIE nadaje: nie ma tu
|
||||||
`data-fl-source` ani `data-post`, na których stoi tamten parser. Player idzie przez
|
`data-fl-source` ani `data-post`, na których stoi tamten parser. Player idzie przez
|
||||||
`POST /wp-admin/admin-ajax.php` i jest za Cloudflare — obsługuje to extractor
|
`POST /wp-admin/admin-ajax.php` i jest za Cloudflare \u2014 obsługuje to extractor
|
||||||
`hentaimama`, patrz jego docstring.
|
`hentaimama`, patrz jego docstring.
|
||||||
|
|
||||||
Pułapka: `/genre/` w całym HTML to 118 linków (chmura w sidebarze, ten sam wzorzec co
|
Pułapka: `/genre/` w całym HTML to 118 linków (chmura w sidebarze, ten sam wzorzec co
|
||||||
|
|
@ -38,20 +38,28 @@ log = logging.getLogger(__name__)
|
||||||
_BASE = "https://hentaimama.io"
|
_BASE = "https://hentaimama.io"
|
||||||
|
|
||||||
_EPISODE_HREF_RE = re.compile(r'href="(https://hentaimama\.io/episodes/[^"]+)"')
|
_EPISODE_HREF_RE = re.compile(r'href="(https://hentaimama\.io/episodes/[^"]+)"')
|
||||||
# Sam listing `/episodes/` i jego paginacja też pasują do wzorca wyżej — bez tego
|
# Sam listing `/episodes/` i jego paginacja też pasują do wzorca wyżej \u2014 bez tego
|
||||||
# filtra wchodziły jako „odcinek" o tytule „Recent Episodes".
|
# filtra wchodziły jako „odcinek" o tytule „Recent Episodes".
|
||||||
# `/episodes/feed/` (RSS) tez wpada w href-regex i konczyl jako pusty detail z None.
|
# `/episodes/feed/` (RSS) tez wpada w href-regex i konczyl jako pusty detail z None.
|
||||||
_NOT_EPISODE_RE = re.compile(r"/episodes/(?:page/\d+/|feed/)?$")
|
_NOT_EPISODE_RE = re.compile(r"/episodes/(?:page/\d+/|feed/)?$")
|
||||||
# `og:title` zamiast <h1>: na stronie są trzy <h1>, dwa to logo serwisu, a trzeci bywa
|
# `og:title` zamiast <h1>: na stronie są trzy <h1>, dwa to logo serwisu, a trzeci bywa
|
||||||
# nagłówkiem sekcji. og:title daje czysty „Shoujo Ramune Episode 7".
|
# nagłówkiem sekcji. og:title daje czysty „Shoujo Ramune Episode 7".
|
||||||
_OG_TITLE_RE = re.compile(r"property=['\"]og:title['\"][^>]*content=['\"]([^'\"]{3,150})")
|
_OG_TITLE_RE = re.compile(r"property=['\"]og:title['\"][^>]*content=['\"]([^'\"]{3,150})")
|
||||||
# Gatunki TYLKO stąd — patrz pułapka w docstringu.
|
# Gatunki TYLKO stąd \u2014 patrz pułapka w docstringu.
|
||||||
_GENRES_BLOCK_RE = re.compile(r'<div class="sgeneros">(.*?)</div>', re.DOTALL)
|
_GENRES_BLOCK_RE = re.compile(r'<div class="sgeneros">(.*?)</div>', re.DOTALL)
|
||||||
_ANCHOR_TEXT_RE = re.compile(r">\s*([^<]{2,40}?)\s*</a>")
|
_ANCHOR_TEXT_RE = re.compile(r">\s*([^<]{2,40}?)\s*</a>")
|
||||||
_DATE_RE = re.compile(r'<span class="date">\s*([A-Z][a-z]{2}\.?\s+\d{1,2},\s+\d{4})')
|
_DATE_RE = re.compile(r'<span class="date">\s*([A-Z][a-z]{2}\.?\s+\d{1,2},\s+\d{4})')
|
||||||
|
# Dlugosc odcinka w sekundach siedzi w konfiguracji skryptu historii motywu dooplay:
|
||||||
|
# `var dtHist = {..., "episode":"77667", "duration":"953", ...}`. Bez niej 100% scen
|
||||||
|
# mialo NULL (audyt 2026-10-14), a apka filtruje min 60 s.
|
||||||
|
_DT_DURATION_RE = re.compile(r'"duration"\s*:\s*"(\d{2,6})"')
|
||||||
_SERIES_RE = re.compile(r'href="https://hentaimama\.io/tvshows/([a-z0-9-]+)/"')
|
_SERIES_RE = re.compile(r'href="https://hentaimama\.io/tvshows/([a-z0-9-]+)/"')
|
||||||
# Uwaga: atrybuty bywają w innej kolejności niż `<meta property=... content=...>`,
|
# Uwaga: atrybuty bywają w innej kolejności niż `<meta property=... content=...>`,
|
||||||
# a wartość ma na końcu \r — stąd luźny wzorzec i strip() przy użyciu.
|
# a wartość ma na końcu \r \u2014 stąd luźny wzorzec i strip() przy użyciu.
|
||||||
|
_TITLE_TAIL_RE = re.compile(
|
||||||
|
r"(?:\s*[\u2013\u2014|-]\s*Watch\s+Online\s+Free\s+in\s+HD)?\s*[\u2013\u2014|-]\s*Hentaimama\s*$",
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
_OG_IMAGE_RE = re.compile(r"property=['\"]og:image['\"][^>]*content=['\"]([^'\"]+)")
|
_OG_IMAGE_RE = re.compile(r"property=['\"]og:image['\"][^>]*content=['\"]([^'\"]+)")
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -82,6 +90,9 @@ class HentaiMamaScraper(BaseBrowseScraper):
|
||||||
def _parse_detail(self, scene_url: str, detail_html: str) -> RawScene | None:
|
def _parse_detail(self, scene_url: str, detail_html: str) -> RawScene | None:
|
||||||
tm = _OG_TITLE_RE.search(detail_html)
|
tm = _OG_TITLE_RE.search(detail_html)
|
||||||
title = html_mod.unescape(tm.group(1)).strip() if tm else ""
|
title = html_mod.unescape(tm.group(1)).strip() if tm else ""
|
||||||
|
# Od ~31.08.2026 og:title ma ogon "TYTUL - Watch Online Free in HD - Hentaimama"
|
||||||
|
# (z polpauzami). Ucinamy go, inaczej tytul nie laczy sie z ta sama scena z innych stron.
|
||||||
|
title = _TITLE_TAIL_RE.sub("", title).strip()
|
||||||
if not title or title.lower() == "hentaimama":
|
if not title or title.lower() == "hentaimama":
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
@ -97,7 +108,7 @@ class HentaiMamaScraper(BaseBrowseScraper):
|
||||||
tags.append(
|
tags.append(
|
||||||
RawTag(external_id=f"{self.sitetag}:tag:{slug}", name=name, slug=slug)
|
RawTag(external_id=f"{self.sitetag}:tag:{slug}", name=name, slug=slug)
|
||||||
)
|
)
|
||||||
# Seria jako tag — pozwala grupować odcinki bez zmian w schemacie.
|
# Seria jako tag \u2014 pozwala grupować odcinki bez zmian w schemacie.
|
||||||
sm = _SERIES_RE.search(detail_html)
|
sm = _SERIES_RE.search(detail_html)
|
||||||
if sm and sm.group(1) not in seen:
|
if sm and sm.group(1) not in seen:
|
||||||
series = sm.group(1).replace("-", " ").title()
|
series = sm.group(1).replace("-", " ").title()
|
||||||
|
|
@ -112,17 +123,21 @@ class HentaiMamaScraper(BaseBrowseScraper):
|
||||||
dm = _DATE_RE.search(detail_html)
|
dm = _DATE_RE.search(detail_html)
|
||||||
om = _OG_IMAGE_RE.search(detail_html)
|
om = _OG_IMAGE_RE.search(detail_html)
|
||||||
slug = scene_url.rstrip("/").rsplit("/", 1)[-1]
|
slug = scene_url.rstrip("/").rsplit("/", 1)[-1]
|
||||||
|
dur_m = _DT_DURATION_RE.search(detail_html)
|
||||||
|
duration_sec = int(dur_m.group(1)) if dur_m and int(dur_m.group(1)) > 0 else None
|
||||||
|
|
||||||
return RawScene(
|
return RawScene(
|
||||||
external_id=f"{self.sitetag}:{slug}",
|
external_id=f"{self.sitetag}:{slug}",
|
||||||
title=title,
|
title=title,
|
||||||
release_date=_parse_date(dm.group(1) if dm else None),
|
release_date=_parse_date(dm.group(1) if dm else None),
|
||||||
|
duration_sec=duration_sec,
|
||||||
url=scene_url,
|
url=scene_url,
|
||||||
tags=tags,
|
tags=tags,
|
||||||
playback_sources=[
|
playback_sources=[
|
||||||
RawPlaybackSource(
|
RawPlaybackSource(
|
||||||
origin=f"tube:{self.sitetag}",
|
origin=f"tube:{self.sitetag}",
|
||||||
page_url=scene_url,
|
page_url=scene_url,
|
||||||
|
duration_sec=duration_sec,
|
||||||
thumbnail_url=om.group(1).strip() if om else None,
|
thumbnail_url=om.group(1).strip() if om else None,
|
||||||
)
|
)
|
||||||
],
|
],
|
||||||
|
|
|
||||||
|
|
@ -18,6 +18,7 @@ import re
|
||||||
|
|
||||||
from app.connectors.base import RawPlaybackSource, RawScene, RawTag
|
from app.connectors.base import RawPlaybackSource, RawScene, RawTag
|
||||||
from app.connectors.direct_scrapers._browse_base import BaseBrowseScraper
|
from app.connectors.direct_scrapers._browse_base import BaseBrowseScraper
|
||||||
|
from app.connectors.base import RawPerformer
|
||||||
from app.connectors.direct_scrapers._playtube import _parse_iso_date
|
from app.connectors.direct_scrapers._playtube import _parse_iso_date
|
||||||
|
|
||||||
_BASE = "https://javflix.cc"
|
_BASE = "https://javflix.cc"
|
||||||
|
|
@ -30,6 +31,11 @@ _NON_POST = (
|
||||||
"/terms", "/faq", "/policy", "/disclaimer", "/sitemap",
|
"/terms", "/faq", "/policy", "/disclaimer", "/sitemap",
|
||||||
)
|
)
|
||||||
_CODE_RE = re.compile(r"^([a-z]+-?\d+[a-z]?)", re.IGNORECASE)
|
_CODE_RE = re.compile(r"^([a-z]+-?\d+[a-z]?)", re.IGNORECASE)
|
||||||
|
# Aktorki: tylko z bloku `<div id="video-actors">Actress: <a href=".../actor/<slug>/">`,
|
||||||
|
# jeden na scene zwykle. Do 2026-10-14 scraper ich nie bral (0 performerow na scene),
|
||||||
|
# a JAV bez aktorki nie da sie laczyc z kanonem ani filtrowac po osobie.
|
||||||
|
_ACTORS_BLOCK_RE = re.compile(r'<div id="video-actors">(.*?)</div>', re.IGNORECASE | re.DOTALL)
|
||||||
|
_ACTOR_RE = re.compile(r'href="https://javflix\.cc/actor/([a-z0-9\-]+)/"[^>]*>([^<]+)</a>', re.IGNORECASE)
|
||||||
|
|
||||||
|
|
||||||
def _itemprop(html: str, name: str) -> str | None:
|
def _itemprop(html: str, name: str) -> str | None:
|
||||||
|
|
@ -96,11 +102,24 @@ class JavflixScraper(BaseBrowseScraper):
|
||||||
code = cm.group(1).upper()
|
code = cm.group(1).upper()
|
||||||
tags.append(RawTag(external_id=f"javcode:{code}", name=code, slug=cm.group(1).lower()))
|
tags.append(RawTag(external_id=f"javcode:{code}", name=code, slug=cm.group(1).lower()))
|
||||||
|
|
||||||
|
performers: list[RawPerformer] = []
|
||||||
|
ab = _ACTORS_BLOCK_RE.search(detail_html)
|
||||||
|
if ab:
|
||||||
|
seen_p: set[str] = set()
|
||||||
|
for am in _ACTOR_RE.finditer(ab.group(1)):
|
||||||
|
a_slug = am.group(1)
|
||||||
|
a_name = html.unescape(am.group(2)).strip()
|
||||||
|
if a_slug in seen_p or not a_name:
|
||||||
|
continue
|
||||||
|
seen_p.add(a_slug)
|
||||||
|
performers.append(RawPerformer(external_id=f"{self.sitetag}:actor:{a_slug}", name=a_name))
|
||||||
|
|
||||||
return RawScene(
|
return RawScene(
|
||||||
external_id=f"{self.sitetag}:{scene_url}",
|
external_id=f"{self.sitetag}:{scene_url}",
|
||||||
title=title,
|
title=title,
|
||||||
release_date=release_date,
|
release_date=release_date,
|
||||||
url=scene_url,
|
url=scene_url,
|
||||||
|
performers=performers,
|
||||||
tags=tags,
|
tags=tags,
|
||||||
playback_sources=[
|
playback_sources=[
|
||||||
RawPlaybackSource(
|
RawPlaybackSource(
|
||||||
|
|
|
||||||
|
|
@ -26,6 +26,8 @@ import logging
|
||||||
import re
|
import re
|
||||||
from datetime import date
|
from datetime import date
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
|
||||||
from app.connectors.base import (
|
from app.connectors.base import (
|
||||||
RawPlaybackSource,
|
RawPlaybackSource,
|
||||||
RawScene,
|
RawScene,
|
||||||
|
|
@ -45,6 +47,7 @@ _BASE = "https://latestpornvideo.com"
|
||||||
_ARTICLE_RE = re.compile(r'<article[^>]*\bclass="([^"]+)"', re.IGNORECASE)
|
_ARTICLE_RE = re.compile(r'<article[^>]*\bclass="([^"]+)"', re.IGNORECASE)
|
||||||
_LINK_RE = re.compile(r'<a\s+href="([^"]+)"\s+title="([^"]+)"', re.IGNORECASE)
|
_LINK_RE = re.compile(r'<a\s+href="([^"]+)"\s+title="([^"]+)"', re.IGNORECASE)
|
||||||
_THUMB_RE = re.compile(r'data-main-thumb="([^"]+)"', re.IGNORECASE)
|
_THUMB_RE = re.compile(r'data-main-thumb="([^"]+)"', re.IGNORECASE)
|
||||||
|
_TICDN_RE = re.compile(r"^https?://(?:[a-z0-9-]+\.)*ticdn\.net/snaps/", re.IGNORECASE)
|
||||||
_CLASS_TAG_RE = re.compile(r"\btag-([a-z0-9-]+)")
|
_CLASS_TAG_RE = re.compile(r"\btag-([a-z0-9-]+)")
|
||||||
_CLASS_CAT_RE = re.compile(r"\bcategory-([a-z0-9-]+)")
|
_CLASS_CAT_RE = re.compile(r"\bcategory-([a-z0-9-]+)")
|
||||||
# Nazwa thumba: `<Studio>-YYYY-MM-DD-<rest>-cover.jpg`.
|
# Nazwa thumba: `<Studio>-YYYY-MM-DD-<rest>-cover.jpg`.
|
||||||
|
|
@ -52,7 +55,54 @@ _THUMB_NAME_RE = re.compile(r"/([A-Za-z0-9][A-Za-z0-9-]*?)-(\d{4})-(\d{2})-(\d{2
|
||||||
# Tytuł: `<Studio> YY MM DD <rest>` (np. "MySexMobile 20 10 23 Abella Danger").
|
# Tytuł: `<Studio> YY MM DD <rest>` (np. "MySexMobile 20 10 23 Abella Danger").
|
||||||
_TITLE_DATE_RE = re.compile(r"^(.*?)\s*\b(\d{2})\s+(\d{2})\s+(\d{2})\b")
|
_TITLE_DATE_RE = re.compile(r"^(.*?)\s*\b(\d{2})\s+(\d{2})\s+(\d{2})\b")
|
||||||
# Karty homepage zawsze siedzą w kategorii "latest-porn-videos", to nie jest tag.
|
# Karty homepage zawsze siedzą w kategorii "latest-porn-videos", to nie jest tag.
|
||||||
_CAT_SKIP = {"latest-porn-videos", "uncategorized", ""}
|
# "mixed-porn" i "porn-movie": od 09.2026 serwis daje KAZDEMU wpisowi tylko te kategorie,
|
||||||
|
# wiec nic nie mowia (zgloszenie 3204ee85, 2026-10-12: "tylko mixed porn").
|
||||||
|
_CAT_SKIP = {"latest-porn-videos", "uncategorized", "", "mixed-porn", "porn-movie"}
|
||||||
|
|
||||||
|
|
||||||
|
_PIXHOST_SHOW_RE = re.compile(r"^https?://(?:www\.)?pixhost\.to/show/\d+/[^/?#]+$", re.IGNORECASE)
|
||||||
|
_OG_IMAGE_RE = re.compile(r"""property=["']og:image["'][^>]*content=["']([^"']+)""", re.IGNORECASE)
|
||||||
|
_LULU_EMBED_RE = re.compile(r"https?://(?:www\.)?lulu[a-z0-9]*\.[a-z]+/(?:e|d|v)/([a-z0-9]+)", re.IGNORECASE)
|
||||||
|
|
||||||
|
|
||||||
|
def _pixhost_removed(url: str) -> bool:
|
||||||
|
"""pixhost przekierowuje usuniety obrazek na baner /image/removed.png."""
|
||||||
|
try:
|
||||||
|
r = httpx.head(url, headers={"User-Agent": "Mozilla/5.0"}, timeout=10.0, follow_redirects=False)
|
||||||
|
except Exception:
|
||||||
|
return False
|
||||||
|
return r.status_code in (301, 302, 303, 307, 308) and "removed" in (r.headers.get("location") or "")
|
||||||
|
|
||||||
|
|
||||||
|
def _lulu_poster(page_url: str, timeout: float) -> str | None:
|
||||||
|
"""Plakat z odtwarzacza lulu osadzonego we wpisie (img.lulucdn.com/<id>.jpg)."""
|
||||||
|
try:
|
||||||
|
page = browser_get(page_url, timeout=timeout).text
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
m = _LULU_EMBED_RE.search(page)
|
||||||
|
return f"https://img.lulucdn.com/{m.group(1)}.jpg" if m else None
|
||||||
|
|
||||||
|
|
||||||
|
def better_thumb(thumb: str | None, page_url: str, *, timeout: float = 30.0) -> str | None:
|
||||||
|
"""Miniatura, ktora da sie pokazac (zgloszenia eb20f1c0/4d8d5daf, 2026-10-12).
|
||||||
|
|
||||||
|
- `pixhost.to/show/...` to strona podgladu (HTML), nie obrazek: bierzemy jej og:image.
|
||||||
|
- Usuniety obrazek pixhost i brak miniatury: plakat z odtwarzacza lulu z wpisu.
|
||||||
|
Dodatkowe zapytania tylko w tych przypadkach (ok. 10% wpisow).
|
||||||
|
"""
|
||||||
|
if thumb and _PIXHOST_SHOW_RE.match(thumb):
|
||||||
|
try:
|
||||||
|
show = httpx.get(thumb, headers={"User-Agent": "Mozilla/5.0"}, timeout=10.0).text
|
||||||
|
m = _OG_IMAGE_RE.search(show)
|
||||||
|
thumb = html.unescape(m.group(1)) if m else None
|
||||||
|
except Exception:
|
||||||
|
thumb = None
|
||||||
|
if thumb and "pixhost." in thumb and _pixhost_removed(thumb):
|
||||||
|
thumb = None
|
||||||
|
if not thumb:
|
||||||
|
return _lulu_poster(page_url, timeout)
|
||||||
|
return thumb
|
||||||
|
|
||||||
|
|
||||||
def _name_from_slug(slug: str) -> str:
|
def _name_from_slug(slug: str) -> str:
|
||||||
|
|
@ -103,6 +153,10 @@ class LatestPornVideoScraper(BaseBrowseScraper):
|
||||||
|
|
||||||
thumb_m = _THUMB_RE.search(window)
|
thumb_m = _THUMB_RE.search(window)
|
||||||
thumb = thumb_m.group(1) if thumb_m else None
|
thumb = thumb_m.group(1) if thumb_m else None
|
||||||
|
if thumb:
|
||||||
|
# ticdn.net (stary serwer miniatur DoodStream) nie istnieje, te same pliki
|
||||||
|
# leza na imgds.net (2026-10-15, 8 tys. martwych kafelkow).
|
||||||
|
thumb = _TICDN_RE.sub("https://imgds.net/snaps/", thumb)
|
||||||
|
|
||||||
# Studio + release_date z nazwy thumba (`<Studio>-YYYY-MM-DD-`).
|
# Studio + release_date z nazwy thumba (`<Studio>-YYYY-MM-DD-`).
|
||||||
studio: RawStudio | None = None
|
studio: RawStudio | None = None
|
||||||
|
|
@ -148,6 +202,9 @@ class LatestPornVideoScraper(BaseBrowseScraper):
|
||||||
seen_tag.add(sl)
|
seen_tag.add(sl)
|
||||||
tags.append(RawTag(external_id=f"{self.sitetag}:tag:{sl}", name=_name_from_slug(sl), slug=sl))
|
tags.append(RawTag(external_id=f"{self.sitetag}:tag:{sl}", name=_name_from_slug(sl), slug=sl))
|
||||||
|
|
||||||
|
# Po odczycie studia i daty z nazwy pliku, bo zapas (plakat lulu) tej nazwy nie ma.
|
||||||
|
thumb = better_thumb(thumb, scene_url, timeout=self._timeout)
|
||||||
|
|
||||||
fingerprints = []
|
fingerprints = []
|
||||||
if thumb:
|
if thumb:
|
||||||
ph = compute_thumbnail_phash(thumb, referer=_BASE + "/")
|
ph = compute_thumbnail_phash(thumb, referer=_BASE + "/")
|
||||||
|
|
|
||||||
|
|
@ -34,7 +34,7 @@ from app.connectors.direct_scrapers._browse_base import (
|
||||||
BaseBrowseScraper,
|
BaseBrowseScraper,
|
||||||
compute_thumbnail_phash,
|
compute_thumbnail_phash,
|
||||||
)
|
)
|
||||||
from app.extractors import browser_get
|
from app.extractors.tubes.perverzija import fetch_perverzija
|
||||||
from app.normalize.text import slugify
|
from app.normalize.text import slugify
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
@ -68,13 +68,17 @@ class PerverzijaScraper(BaseBrowseScraper):
|
||||||
def crawl_page(self, page: int) -> list[RawScene] | None:
|
def crawl_page(self, page: int) -> list[RawScene] | None:
|
||||||
url = self._listing_url(page)
|
url = self._listing_url(page)
|
||||||
try:
|
try:
|
||||||
res = browser_get(url, timeout=self._timeout)
|
res = fetch_perverzija(url, timeout=self._timeout)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.warning("perverzija REST fetch failed (page %d): %s", page, e)
|
log.warning("perverzija REST fetch failed (page %d): %s", page, e)
|
||||||
return None
|
return None
|
||||||
# WP zwraca 400 (rest_post_invalid_page_number) za ostatnią stroną → exhausted.
|
# WP zwraca 400 (rest_post_invalid_page_number) za ostatnią stroną → exhausted.
|
||||||
if res.status_code != 200:
|
# Inny kod (403 Cloudflare, 5xx) to awaria, nie koniec katalogu.
|
||||||
|
if res.status_code == 400:
|
||||||
return []
|
return []
|
||||||
|
if res.status_code != 200:
|
||||||
|
log.warning("perverzija REST HTTP %s (page %d)", res.status_code, page)
|
||||||
|
return None
|
||||||
try:
|
try:
|
||||||
posts = json.loads(res.text)
|
posts = json.loads(res.text)
|
||||||
except (json.JSONDecodeError, ValueError):
|
except (json.JSONDecodeError, ValueError):
|
||||||
|
|
|
||||||
|
|
@ -32,6 +32,21 @@ _SCENE_URL_RE = re.compile(r'href="(https://watchporn\.to/video/\d+/[a-z0-9\-]+/
|
||||||
_MODEL_RE = re.compile(r'href="https://watchporn\.to/models/([a-z0-9\-]+)/"[^>]*>([^<]+)', re.IGNORECASE)
|
_MODEL_RE = re.compile(r'href="https://watchporn\.to/models/([a-z0-9\-]+)/"[^>]*>([^<]+)', re.IGNORECASE)
|
||||||
_TAG_RE = re.compile(r'href="https://watchporn\.to/tags/([a-z0-9\-]+)/"[^>]*>([^<]+)', re.IGNORECASE)
|
_TAG_RE = re.compile(r'href="https://watchporn\.to/tags/([a-z0-9\-]+)/"[^>]*>([^<]+)', re.IGNORECASE)
|
||||||
_CAT_RE = re.compile(r'href="https://watchporn\.to/categories/([a-z0-9\-]+)/"[^>]*>([^<]+)', re.IGNORECASE)
|
_CAT_RE = re.compile(r'href="https://watchporn\.to/categories/([a-z0-9\-]+)/"[^>]*>([^<]+)', re.IGNORECASE)
|
||||||
|
# Nowy motyw od ok. 2026-10-10 (prefiks klas `wp-`). Obsada sceny siedzi TYLKO w bloku
|
||||||
|
# `<div class="wp-castrow">`, nazwa w `<span class="wp-cast__n">` (nie w tekscie linku),
|
||||||
|
# a karty "related" ponizej maja wlasne linki `/models/` z nazwa w tekscie. Stary globalny
|
||||||
|
# regex gubil prawdziwa obsade (pusty tekst linku) i dokladal performerow z kart related:
|
||||||
|
# srednio 12,5 na scene zamiast 2 (10 do 13.10, 193 sceny). Kategorie to teraz gatunki
|
||||||
|
# (chipy `wp-vchip`), studio jest w linku `/sites/<slug>/` i w JSON-LD "publisher".
|
||||||
|
_NEW_THEME_MARK = 'class="wp-vi__panel"'
|
||||||
|
_CASTROW_RE = re.compile(r'<div class="wp-castrow">(.*?)</div>', re.IGNORECASE | re.DOTALL)
|
||||||
|
_CAST_RE = re.compile(
|
||||||
|
r'href="https://watchporn\.to/models/([a-z0-9\-]+)/"[^>]*>.*?<span class="wp-cast__n">([^<]+)</span>',
|
||||||
|
re.IGNORECASE | re.DOTALL,
|
||||||
|
)
|
||||||
|
_CHIPS_RE = re.compile(r'<div class="wp-chips">(.*?)</div>', re.IGNORECASE | re.DOTALL)
|
||||||
|
_SITE_RE = re.compile(r'href="https://watchporn\.to/sites/([a-z0-9\-]+)/"', re.IGNORECASE)
|
||||||
|
_PUBLISHER_RE = re.compile(r'"publisher"\s*:\s*\{[^}]*?"name"\s*:\s*"([^"]+)"', re.DOTALL)
|
||||||
_DUR_RE = re.compile(r'"duration"\s*:\s*"([^"]+)"')
|
_DUR_RE = re.compile(r'"duration"\s*:\s*"([^"]+)"')
|
||||||
_UPLOAD_RE = re.compile(r'"uploadDate"\s*:\s*"([^"]+)"')
|
_UPLOAD_RE = re.compile(r'"uploadDate"\s*:\s*"([^"]+)"')
|
||||||
_VIDEO_ID_RE = re.compile(r"/video/(\d+)/")
|
_VIDEO_ID_RE = re.compile(r"/video/(\d+)/")
|
||||||
|
|
@ -74,9 +89,18 @@ class WatchPornScraper(BaseBrowseScraper):
|
||||||
um = _UPLOAD_RE.search(detail_html)
|
um = _UPLOAD_RE.search(detail_html)
|
||||||
release_date = _parse_iso_date(um.group(1)) if um else None
|
release_date = _parse_iso_date(um.group(1)) if um else None
|
||||||
|
|
||||||
|
new_theme = _NEW_THEME_MARK in detail_html
|
||||||
|
|
||||||
performers: list[RawPerformer] = []
|
performers: list[RawPerformer] = []
|
||||||
seen_p: set[str] = set()
|
seen_p: set[str] = set()
|
||||||
for m in _MODEL_RE.finditer(detail_html):
|
if new_theme:
|
||||||
|
# Tylko blok obsady; brak bloku = scena bez obsady, NIE wracamy do globalnego
|
||||||
|
# regexu, bo ten zbiera karty related.
|
||||||
|
crm = _CASTROW_RE.search(detail_html)
|
||||||
|
cast_iter = _CAST_RE.finditer(crm.group(1)) if crm else iter(())
|
||||||
|
else:
|
||||||
|
cast_iter = _MODEL_RE.finditer(detail_html)
|
||||||
|
for m in cast_iter:
|
||||||
slug = m.group(1)
|
slug = m.group(1)
|
||||||
name = html.unescape(m.group(2)).strip()
|
name = html.unescape(m.group(2)).strip()
|
||||||
if slug.isdigit() or slug in seen_p or not name:
|
if slug.isdigit() or slug in seen_p or not name:
|
||||||
|
|
@ -86,19 +110,38 @@ class WatchPornScraper(BaseBrowseScraper):
|
||||||
|
|
||||||
tags: list[RawTag] = []
|
tags: list[RawTag] = []
|
||||||
seen_t: set[str] = set()
|
seen_t: set[str] = set()
|
||||||
for m in _TAG_RE.finditer(detail_html):
|
|
||||||
slug = m.group(1)
|
def _add_tag(slug: str, raw_name: str) -> None:
|
||||||
name = html.unescape(m.group(2)).strip()
|
name = html.unescape(raw_name).strip().lstrip("#").strip()
|
||||||
if slug in seen_t or not name:
|
if slug in seen_t or not name:
|
||||||
continue
|
return
|
||||||
seen_t.add(slug)
|
seen_t.add(slug)
|
||||||
tags.append(RawTag(external_id=f"{self.sitetag}:tag:{slug}", name=name, slug=slug))
|
tags.append(RawTag(external_id=f"{self.sitetag}:tag:{slug}", name=name, slug=slug))
|
||||||
|
|
||||||
|
for m in _TAG_RE.finditer(detail_html):
|
||||||
|
_add_tag(m.group(1), m.group(2))
|
||||||
|
if new_theme:
|
||||||
|
# Kategorie-chipy sceny (Blowjob, MILF...) to gatunki, ida do tagow.
|
||||||
|
chm = _CHIPS_RE.search(detail_html)
|
||||||
|
if chm:
|
||||||
|
for m in _CAT_RE.finditer(chm.group(1)):
|
||||||
|
_add_tag(m.group(1), m.group(2))
|
||||||
|
|
||||||
thumbnail_url = (
|
thumbnail_url = (
|
||||||
meta_content(detail_html, property="og:image") or _derive_thumb(scene_url)
|
meta_content(detail_html, property="og:image") or _derive_thumb(scene_url)
|
||||||
)
|
)
|
||||||
|
|
||||||
studio: RawStudio | None = None
|
studio: RawStudio | None = None
|
||||||
|
if new_theme:
|
||||||
|
sm = _SITE_RE.search(detail_html)
|
||||||
|
if sm:
|
||||||
|
pm = _PUBLISHER_RE.search(detail_html)
|
||||||
|
cname = html.unescape(pm.group(1)).strip() if pm else sm.group(1).replace("-", " ").title()
|
||||||
|
if cname:
|
||||||
|
studio = RawStudio(
|
||||||
|
external_id=f"{self.sitetag}:studio:{slugify(cname)}", name=cname, slug=slugify(cname)
|
||||||
|
)
|
||||||
|
else:
|
||||||
cm = _CAT_RE.search(detail_html)
|
cm = _CAT_RE.search(detail_html)
|
||||||
if cm:
|
if cm:
|
||||||
cname = html.unescape(cm.group(2)).strip()
|
cname = html.unescape(cm.group(2)).strip()
|
||||||
|
|
|
||||||
|
|
@ -171,8 +171,15 @@ _VID_TITLE_RE = re.compile(
|
||||||
r'<h1[^>]*\bclass="vid__t[^"]*"[^>]*>\s*(?:<span[^>]*>)?\s*(.+?)\s*(?:</span>)?\s*</h1>',
|
r'<h1[^>]*\bclass="vid__t[^"]*"[^>]*>\s*(?:<span[^>]*>)?\s*(.+?)\s*(?:</span>)?\s*</h1>',
|
||||||
re.IGNORECASE | re.DOTALL,
|
re.IGNORECASE | re.DOTALL,
|
||||||
)
|
)
|
||||||
|
# Motyw 2026-10 losuje przedrostek w h1 ("Watch ...", "Now Playing: ...", "Play ...", na
|
||||||
|
# pandamovies "... in the Panda Database"). Czysty tytul stoi w okruszkach, bierzemy go
|
||||||
|
# stamtad, a h1 zostaje do roku i studia (i jako zapas bez okruszkow). "Play" nie jest
|
||||||
|
# zdejmowane z h1, bo bywa czescia tytulu ("Play Time!", "Play With Me").
|
||||||
|
_BC_CURRENT_RE = re.compile(
|
||||||
|
r'<span[^>]*\bclass="bc__c"[^>]*\baria-current="page"[^>]*>([^<]+)</span>', re.IGNORECASE
|
||||||
|
)
|
||||||
_VID_TITLE_PARTS_RE = re.compile(
|
_VID_TITLE_PARTS_RE = re.compile(
|
||||||
r"^(?:Watch\s+)?(?P<title>.+?)"
|
r"^(?:Watch\s+|Now\s+Playing:\s*)?(?P<title>.+?)"
|
||||||
r"(?:\s+(?P<year>(?:19|20)\d{2})\s+by\s+(?P<studio>.+?))?"
|
r"(?:\s+(?P<year>(?:19|20)\d{2})\s+by\s+(?P<studio>.+?))?"
|
||||||
r"(?:\s+Porn\s+Movie\s+Online\s+Free)?\s*$",
|
r"(?:\s+Porn\s+Movie\s+Online\s+Free)?\s*$",
|
||||||
re.IGNORECASE | re.DOTALL,
|
re.IGNORECASE | re.DOTALL,
|
||||||
|
|
@ -184,9 +191,22 @@ _ST_DURATION_RE = re.compile(
|
||||||
_ST_DATE_RE = re.compile(
|
_ST_DATE_RE = re.compile(
|
||||||
r'class="st st--date"[^>]*>.*?<time\s+datetime="(\d{4}-\d{2}-\d{2})', re.IGNORECASE | re.DOTALL
|
r'class="st st--date"[^>]*>.*?<time\s+datetime="(\d{4}-\d{2}-\d{2})', re.IGNORECASE | re.DOTALL
|
||||||
)
|
)
|
||||||
_YEAR_LOOSE_RE = re.compile(r'/(?:year|release-year)/(\d{4})/"', re.IGNORECASE)
|
# pandamovies.org 2026-10: chip `/release-year/2026` bez koncowego ukosnika.
|
||||||
|
_YEAR_LOOSE_RE = re.compile(r'/(?:year|release-year)/(\d{4})/?"', re.IGNORECASE)
|
||||||
|
# Motyw "vid" (mangoporn, pandamovies, streamporn.li, 2026-10): studio w chipie bez
|
||||||
|
# rel="tag", pod /studios/, /movies/studios/ albo /director/, czasem bez ukosnika.
|
||||||
|
_STUDIO_CHIP_RE = re.compile(
|
||||||
|
r'<a\s+class="chip"\s+href="https?://[^/]+/(?:movies/)?(?:studios?|director)/([a-z0-9-]+)/?"[^>]*>([^<]+)</a>',
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
# Ten sam motyw nie ma juz sekcji obsady z rel="tag"; obsada filmu jest w JSON-LD
|
||||||
|
# (`"actor":[{"name":..., "url":".../actor/<slug>"}]`). Linki do aktorow w HTML bywaja
|
||||||
|
# tez w widzetach bocznych, wiec bierzemy JSON-LD, ktory opisuje tylko ten film.
|
||||||
|
_JSONLD_ACTORS_RE = re.compile(r'"actor"\s*:\s*(\[[^\]]*\])')
|
||||||
|
# mangoporn ma gatunki pod /movies/genre/, pandamovies bez koncowego ukosnika, a chip
|
||||||
|
# kategorii /genres/porn-movies/ to nie gatunek (byl jedynym tagiem filmow mangoporn).
|
||||||
_GENRE_CHIP_RE = re.compile(
|
_GENRE_CHIP_RE = re.compile(
|
||||||
r'<a\s+class="chip"\s+href="https?://[^/]+/genres?/([a-z0-9-]+)/"[^>]*>([^<]+)</a>',
|
r'<a\s+class="chip"\s+href="https?://[^/]+/(?:movies/)?genres?/(?!porn-movies/?")([a-z0-9-]+)/?"[^>]*>([^<]+)</a>',
|
||||||
re.IGNORECASE,
|
re.IGNORECASE,
|
||||||
)
|
)
|
||||||
_DATA_SERVERS_RE = re.compile(r'data-servers="([^"]*)"', re.IGNORECASE)
|
_DATA_SERVERS_RE = re.compile(r'data-servers="([^"]*)"', re.IGNORECASE)
|
||||||
|
|
@ -341,6 +361,30 @@ class DooplayConnector(BaseMovieConnector):
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _jsonld_performers(html: str, source_name: str) -> list[RawPerformer]:
|
||||||
|
"""Obsada z JSON-LD filmu (motyw "vid" 2026-10). Id jak dawniej: slug z adresu aktora,
|
||||||
|
wiec te same osoby nie dubluja sie wzgledem filmow sprzed zmiany motywu."""
|
||||||
|
m = _JSONLD_ACTORS_RE.search(html)
|
||||||
|
if not m:
|
||||||
|
return []
|
||||||
|
try:
|
||||||
|
people = json.loads(m.group(1))
|
||||||
|
except ValueError:
|
||||||
|
return []
|
||||||
|
out: list[RawPerformer] = []
|
||||||
|
seen: set[str] = set()
|
||||||
|
for p in people:
|
||||||
|
if not isinstance(p, dict):
|
||||||
|
continue
|
||||||
|
name = _decode_html((p.get("name") or "").strip())
|
||||||
|
slug = (p.get("url") or "").rstrip("/").rsplit("/", 1)[-1].lower()
|
||||||
|
if not name or not slug or slug in seen:
|
||||||
|
continue
|
||||||
|
seen.add(slug)
|
||||||
|
out.append(RawPerformer(external_id=f"{source_name}:{slug}", name=name))
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
def _parse_dooplay_detail(
|
def _parse_dooplay_detail(
|
||||||
*, slug: str, html: str, source_name: str, base_url: str, page_url: str | None = None
|
*, slug: str, html: str, source_name: str, base_url: str, page_url: str | None = None
|
||||||
) -> RawMovie | None:
|
) -> RawMovie | None:
|
||||||
|
|
@ -359,6 +403,10 @@ def _parse_dooplay_detail(
|
||||||
raw_title = _decode_html(_strip_tags(m_vid.group(1))).strip()
|
raw_title = _decode_html(_strip_tags(m_vid.group(1))).strip()
|
||||||
m_parts = _VID_TITLE_PARTS_RE.match(raw_title)
|
m_parts = _VID_TITLE_PARTS_RE.match(raw_title)
|
||||||
title = (m_parts.group("title") if m_parts else raw_title).strip()
|
title = (m_parts.group("title") if m_parts else raw_title).strip()
|
||||||
|
title = re.sub(r"\s+in\s+the\s+Panda\s+Database$", "", title, flags=re.IGNORECASE)
|
||||||
|
m_bc = _BC_CURRENT_RE.search(html)
|
||||||
|
if m_bc and _decode_html(m_bc.group(1)).strip():
|
||||||
|
title = _decode_html(m_bc.group(1)).strip()
|
||||||
if m_parts and m_parts.group("year"):
|
if m_parts and m_parts.group("year"):
|
||||||
vid_year = int(m_parts.group("year"))
|
vid_year = int(m_parts.group("year"))
|
||||||
if m_parts and m_parts.group("studio"):
|
if m_parts and m_parts.group("studio"):
|
||||||
|
|
@ -370,7 +418,7 @@ def _parse_dooplay_detail(
|
||||||
release_year = int(m_year.group(1)) if m_year else vid_year
|
release_year = int(m_year.group(1)) if m_year else vid_year
|
||||||
|
|
||||||
studio: RawStudio | None = None
|
studio: RawStudio | None = None
|
||||||
m_studio = _STUDIO_RE.search(html)
|
m_studio = _STUDIO_RE.search(html) or _STUDIO_CHIP_RE.search(html)
|
||||||
if m_studio:
|
if m_studio:
|
||||||
studio_slug = m_studio.group(1)
|
studio_slug = m_studio.group(1)
|
||||||
studio_name = _decode_html(m_studio.group(2).strip())
|
studio_name = _decode_html(m_studio.group(2).strip())
|
||||||
|
|
@ -457,6 +505,8 @@ def _parse_dooplay_detail(
|
||||||
)
|
)
|
||||||
for m in _PORNSTAR_RE.finditer(html)
|
for m in _PORNSTAR_RE.finditer(html)
|
||||||
]
|
]
|
||||||
|
if not performers:
|
||||||
|
performers = _jsonld_performers(html, source_name)
|
||||||
|
|
||||||
tags = [
|
tags = [
|
||||||
RawTag(
|
RawTag(
|
||||||
|
|
@ -614,7 +664,10 @@ class StreampornVipConnector(DooplayConnector):
|
||||||
|
|
||||||
class PandamoviesConnector(DooplayConnector):
|
class PandamoviesConnector(DooplayConnector):
|
||||||
name = "pandamovies"
|
name = "pandamovies"
|
||||||
base_url = "https://pandamovies.pw"
|
# 2026-08 pandamovies.pw -> 302 pandamovies.org; konektor odrzuca linki spoza swojej
|
||||||
|
# domeny, wiec od 20.08 ingest dawal 0 filmow. Id filmu to slug, przeprowadzka nie robi
|
||||||
|
# duplikatow (to samo co streamporn.vip -> .li).
|
||||||
|
base_url = "https://pandamovies.org"
|
||||||
|
|
||||||
|
|
||||||
class MangopornConnector(DooplayConnector):
|
class MangopornConnector(DooplayConnector):
|
||||||
|
|
|
||||||
235
app/connectors/xtapes.py
Normal file
235
app/connectors/xtapes.py
Normal file
|
|
@ -0,0 +1,235 @@
|
||||||
|
"""xtapes.la, konektor filmow (kategoria "Full Movies"). Dodany 2026-10-15.
|
||||||
|
|
||||||
|
WordPress z otwartym REST API, wiec liste bierzemy z `/wp-json/wp/v2/posts` z filtrem
|
||||||
|
kategorii, a nie z HTML-a. Ocena przed dodaniem (2026-10-15): 1846 filmow, 3 do 5 nowych
|
||||||
|
dziennie, czesto w dniu premiery; z 30 najnowszych 10 nie mielismy wcale, reszte z
|
||||||
|
yourdailypornmovies/paradisehill/mangoporn (xtapes dokleja im kolejny link).
|
||||||
|
|
||||||
|
Metadane z REST (`_embed=wp:term`):
|
||||||
|
- tytul "TYTUL (RRRR) - Full Movie" -> tytul i rok,
|
||||||
|
- studio to kategoria w nawiasie kwadratowym ("[EvilAngel]", "[Brazzers Movies]"),
|
||||||
|
- pozostale kategorie to gatunki (bez "Full Movies"),
|
||||||
|
- obsada to tagi (`post_tag`),
|
||||||
|
- opis z `content` (czesto pusty).
|
||||||
|
Ze strony filmu: odtwarzacze (iframe), dlugosc (`itemprop="duration"`), okladka (og:image).
|
||||||
|
|
||||||
|
Odtwarzacze: sextapes.to (P.A.C.K.E.R. + JWPlayer, HLS typu StreamWish) i maxcloud.fit.
|
||||||
|
maxcloud to posrednik (lista serwerow w base64, za nia kolejny serwis z blokada devtools),
|
||||||
|
wiec go pomijamy. Link sextapes jest zwiazany z IP rozwiazujacego (z VPS 403 z innego IP),
|
||||||
|
dlatego rozwiazuje go telefon (packerHoster.ts), a backend go nie ekstrahuje (playback.py).
|
||||||
|
|
||||||
|
Dostep: VPS dostaje od Cloudflare 403 na xtapes.la (REST i strony), przez Bright Data 200.
|
||||||
|
Proxy tylko do HTML/JSON, nie do wideo. sextapes.to z VPS odpowiada bez proxy.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import html as html_mod
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import re
|
||||||
|
import time
|
||||||
|
from collections.abc import Iterator
|
||||||
|
from datetime import datetime
|
||||||
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
|
from app.config import get_settings
|
||||||
|
from app.connectors.base import (
|
||||||
|
BaseMovieConnector,
|
||||||
|
RawMovie,
|
||||||
|
RawPerformer,
|
||||||
|
RawPlaybackSource,
|
||||||
|
RawStudio,
|
||||||
|
RawTag,
|
||||||
|
)
|
||||||
|
from app.extractors import browser_get
|
||||||
|
from app.models.source import SourceKind
|
||||||
|
from app.normalize.text import slugify
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
_BASE = "https://xtapes.la"
|
||||||
|
#: Id kategorii "Full Movies" (sprawdzone 2026-10-15; slugi kategorii sa zaciemnione liczbami).
|
||||||
|
_MOVIES_CATEGORY = 1766
|
||||||
|
_PER_PAGE = 30
|
||||||
|
|
||||||
|
_TITLE_RE = re.compile(
|
||||||
|
r"^(?P<title>.+?)\s*\((?P<year>(?:19|20)\d{2})\)\s*(?:[-\u2013\u2014]\s*Full\s+Movie)?\s*$",
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
_FULL_MOVIE_TAIL_RE = re.compile(r"\s*[-\u2013\u2014]\s*Full\s+Movie\s*$", re.IGNORECASE)
|
||||||
|
_STUDIO_CAT_RE = re.compile(r"^\[(?P<name>.+)\]$")
|
||||||
|
_IFRAME_RE = re.compile(r"""<iframe[^>]+src=["']((?:https?:)?//[^"']+)["']""", re.IGNORECASE)
|
||||||
|
_DURATION_RE = re.compile(r'itemprop="duration"\s+content="P(?:\d+D)?T?(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?"')
|
||||||
|
_OG_IMAGE_RE = re.compile(r"""property=["']og:image["']\s+content=["']([^"']+)""", re.IGNORECASE)
|
||||||
|
_TAGS_RE = re.compile(r"<[^>]+>")
|
||||||
|
|
||||||
|
#: Odtwarzacze, ktorych nie zapisujemy: posrednik bez wlasnego wideo i reklamy.
|
||||||
|
_SKIP_HOSTS = ("maxcloud", "realsrv", "magsrv", "exosrv", "juicyads", "google", "doubleclick")
|
||||||
|
|
||||||
|
|
||||||
|
def _host_label(url: str) -> str | None:
|
||||||
|
host = (urlparse(url).hostname or "").lower().removeprefix("www.")
|
||||||
|
if not host:
|
||||||
|
return None
|
||||||
|
parts = host.split(".")
|
||||||
|
return parts[-2] if len(parts) >= 2 else parts[0]
|
||||||
|
|
||||||
|
|
||||||
|
def parse_title(raw: str) -> tuple[str, int | None]:
|
||||||
|
"""'Tushy Raw V87 (2026) - Full Movie' -> ('Tushy Raw V87', 2026)."""
|
||||||
|
t = html_mod.unescape(raw).strip()
|
||||||
|
m = _TITLE_RE.match(t)
|
||||||
|
if m:
|
||||||
|
return m.group("title").strip(), int(m.group("year"))
|
||||||
|
return _FULL_MOVIE_TAIL_RE.sub("", t).strip(), None
|
||||||
|
|
||||||
|
|
||||||
|
def parse_duration(page: str) -> int | None:
|
||||||
|
m = _DURATION_RE.search(page)
|
||||||
|
if not m or not any(m.groups()):
|
||||||
|
return None
|
||||||
|
h, mi, s = (int(x) if x else 0 for x in m.groups())
|
||||||
|
total = h * 3600 + mi * 60 + s
|
||||||
|
return total or None
|
||||||
|
|
||||||
|
|
||||||
|
class XtapesConnector(BaseMovieConnector):
|
||||||
|
kind = SourceKind.scraper
|
||||||
|
name = "xtapes"
|
||||||
|
base_url = _BASE
|
||||||
|
|
||||||
|
_MAX_PAGES_DELTA = 3
|
||||||
|
_MAX_PAGES_FULL = 20
|
||||||
|
|
||||||
|
def __init__(self, *, timeout: float = 30.0) -> None:
|
||||||
|
self._timeout = timeout
|
||||||
|
self._proxy = get_settings().brightdata_proxy_url
|
||||||
|
|
||||||
|
def close(self) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def _get(self, url: str):
|
||||||
|
"""xtapes.la od razu przez proxy (Cloudflare daje VPS-owi 403), bez proxy tylko gdy
|
||||||
|
go nie skonfigurowano (lokalnie z domowego IP strona odpowiada wprost)."""
|
||||||
|
r = browser_get(url, timeout=self._timeout, follow_redirects=True, proxy=self._proxy)
|
||||||
|
if r.status_code >= 400:
|
||||||
|
raise RuntimeError(f"{r.status_code} for {url}")
|
||||||
|
return r
|
||||||
|
|
||||||
|
def fetch_movies(
|
||||||
|
self, *, since: datetime | None = None, limit: int | None = None
|
||||||
|
) -> Iterator[RawMovie]:
|
||||||
|
seen = 0
|
||||||
|
max_pages = self._MAX_PAGES_DELTA if since is not None else self._MAX_PAGES_FULL
|
||||||
|
for page in range(1, max_pages + 1):
|
||||||
|
url = (
|
||||||
|
f"{_BASE}/wp-json/wp/v2/posts?categories={_MOVIES_CATEGORY}"
|
||||||
|
f"&per_page={_PER_PAGE}&page={page}&_embed=wp:term"
|
||||||
|
"&_fields=id,date,slug,link,title,content,_links,_embedded"
|
||||||
|
)
|
||||||
|
if since is not None:
|
||||||
|
# Tylko nowsze od ostatniego udanego przebiegu: strona filmu idzie przez proxy
|
||||||
|
# (~9 s), a 3 pelne strony REST to 90 filmow, ponad limit 6 min na konektor.
|
||||||
|
url += f"&after={since.strftime('%Y-%m-%dT%H:%M:%S')}"
|
||||||
|
posts = None
|
||||||
|
for attempt in range(3):
|
||||||
|
try:
|
||||||
|
posts = json.loads(self._get(url).text)
|
||||||
|
break
|
||||||
|
except Exception as e:
|
||||||
|
# Za ostatnia strona WP oddaje 400 (rest_post_invalid_page_number), to koniec.
|
||||||
|
# Inne bledy (proxy, Cloudflare) bywaja chwilowe: pierwszy pelny przebieg
|
||||||
|
# 2026-10-15 skonczyl sie "sukcesem" z zerem filmow po jednym takim bledzie.
|
||||||
|
if "400 for" in str(e):
|
||||||
|
return
|
||||||
|
log.warning("%s REST page=%d proba %d: %s", self.name, page, attempt + 1, e)
|
||||||
|
time.sleep(3)
|
||||||
|
if posts is None:
|
||||||
|
return
|
||||||
|
if not isinstance(posts, list) or not posts:
|
||||||
|
return
|
||||||
|
for post in posts:
|
||||||
|
try:
|
||||||
|
movie = self._movie_from_post(post)
|
||||||
|
except Exception as e:
|
||||||
|
log.warning("%s post %s failed: %s", self.name, post.get("link"), e)
|
||||||
|
continue
|
||||||
|
if movie is None:
|
||||||
|
continue
|
||||||
|
yield movie
|
||||||
|
seen += 1
|
||||||
|
if limit is not None and seen >= limit:
|
||||||
|
return
|
||||||
|
|
||||||
|
def _movie_from_post(self, post: dict) -> RawMovie | None:
|
||||||
|
link = (post.get("link") or "").strip()
|
||||||
|
title, year = parse_title((post.get("title") or {}).get("rendered", ""))
|
||||||
|
if not link or not title:
|
||||||
|
return None
|
||||||
|
|
||||||
|
studio: RawStudio | None = None
|
||||||
|
tags: list[RawTag] = []
|
||||||
|
performers: list[RawPerformer] = []
|
||||||
|
for group in (post.get("_embedded") or {}).get("wp:term") or []:
|
||||||
|
for term in group or []:
|
||||||
|
name = html_mod.unescape(term.get("name") or "").strip()
|
||||||
|
tax = term.get("taxonomy")
|
||||||
|
if not name:
|
||||||
|
continue
|
||||||
|
if tax == "category":
|
||||||
|
sm = _STUDIO_CAT_RE.match(name)
|
||||||
|
if sm:
|
||||||
|
sname = sm.group("name").strip()
|
||||||
|
if studio is None and slugify(sname):
|
||||||
|
studio = RawStudio(
|
||||||
|
external_id=f"{self.name}:studio:{slugify(sname)}",
|
||||||
|
name=sname,
|
||||||
|
slug=slugify(sname),
|
||||||
|
)
|
||||||
|
elif name.lower() != "full movies" and slugify(name):
|
||||||
|
tags.append(
|
||||||
|
RawTag(external_id=f"{self.name}:tag:{slugify(name)}", name=name, slug=slugify(name))
|
||||||
|
)
|
||||||
|
elif tax == "post_tag" and slugify(name):
|
||||||
|
performers.append(
|
||||||
|
RawPerformer(external_id=f"{self.name}:performer:{slugify(name)}", name=name)
|
||||||
|
)
|
||||||
|
|
||||||
|
page = self._get(link).text
|
||||||
|
|
||||||
|
# Playback najpierw: bez odtwarzacza film jest bezuzyteczny, nie tworzymy sieroty.
|
||||||
|
playback: list[RawPlaybackSource] = []
|
||||||
|
seen_host: set[str] = set()
|
||||||
|
for m in _IFRAME_RE.finditer(page):
|
||||||
|
embed = m.group(1)
|
||||||
|
if embed.startswith("//"):
|
||||||
|
embed = "https:" + embed
|
||||||
|
host = _host_label(embed)
|
||||||
|
if not host or host in seen_host or any(s in embed for s in _SKIP_HOSTS):
|
||||||
|
continue
|
||||||
|
seen_host.add(host)
|
||||||
|
playback.append(
|
||||||
|
RawPlaybackSource(origin=f"{self.name}:{host}", page_url=embed, embed_url=embed)
|
||||||
|
)
|
||||||
|
if not playback:
|
||||||
|
return None
|
||||||
|
|
||||||
|
desc = _TAGS_RE.sub(" ", (post.get("content") or {}).get("rendered") or "")
|
||||||
|
desc = re.sub(r"\s+", " ", html_mod.unescape(desc)).strip() or None
|
||||||
|
om = _OG_IMAGE_RE.search(page)
|
||||||
|
poster = html_mod.unescape(om.group(1)).strip() if om else None
|
||||||
|
|
||||||
|
return RawMovie(
|
||||||
|
external_id=post.get("slug") or urlparse(link).path.strip("/").split("/")[-1],
|
||||||
|
title=title,
|
||||||
|
description=desc,
|
||||||
|
release_year=year,
|
||||||
|
duration_sec=parse_duration(page),
|
||||||
|
url=link,
|
||||||
|
poster_url=poster,
|
||||||
|
studio=studio,
|
||||||
|
performers=performers,
|
||||||
|
tags=tags,
|
||||||
|
playback_sources=playback,
|
||||||
|
raw={"source": self.name, "url": link, "wp_id": post.get("id")},
|
||||||
|
)
|
||||||
|
|
@ -35,6 +35,8 @@ from app.extractors.tubes import (
|
||||||
hdporngg,
|
hdporngg,
|
||||||
hqfap,
|
hqfap,
|
||||||
perverzija,
|
perverzija,
|
||||||
|
porndish,
|
||||||
|
fpoxxx,
|
||||||
hqporner,
|
hqporner,
|
||||||
fullporner,
|
fullporner,
|
||||||
hentaigasm,
|
hentaigasm,
|
||||||
|
|
@ -106,7 +108,8 @@ _REGISTRY: dict[str, Callable[[str], list[StreamSource] | None]] = {
|
||||||
# fpoxxx, KVS, plain get_file + license. 2026-06-01 (task #20): get_file 302 →
|
# fpoxxx, KVS, plain get_file + license. 2026-06-01 (task #20): get_file 302 →
|
||||||
# `videos3.fpo.xxx/remote_control.php?acctoken=<base64>`, zdekodowany acctoken
|
# `videos3.fpo.xxx/remote_control.php?acctoken=<base64>`, zdekodowany acctoken
|
||||||
# zawiera WBITY IP serwera-resolvera → definitywnie IP-bound. WebView only.
|
# zawiera WBITY IP serwera-resolvera → definitywnie IP-bound. WebView only.
|
||||||
"fpoxxx": _vps_blocked_fallback.extract,
|
# Skasowany film = HosterDead zamiast strony z reklama w WebView (2026-10-15).
|
||||||
|
"fpoxxx": fpoxxx.extract,
|
||||||
# sxyland, embeduje playmogo.com/e/<id> (= klon DoodStream: doodcdn.io + pass_md5
|
# sxyland, embeduje playmogo.com/e/<id> (= klon DoodStream: doodcdn.io + pass_md5
|
||||||
# + niewidzialny CF Turnstile; Chrome-DevTools verify 2026-06-08, bug-report 827a50a1).
|
# + niewidzialny CF Turnstile; Chrome-DevTools verify 2026-06-08, bug-report 827a50a1).
|
||||||
# Strona sxyland NIE jest Turnstile-gated (VPS curl wyciąga iframe URL z HTML), więc
|
# Strona sxyland NIE jest Turnstile-gated (VPS curl wyciąga iframe URL z HTML), więc
|
||||||
|
|
@ -177,7 +180,8 @@ _REGISTRY: dict[str, Callable[[str], list[StreamSource] | None]] = {
|
||||||
"fullvideosporn": fullvideosporn.extract,
|
"fullvideosporn": fullvideosporn.extract,
|
||||||
"siskavideo": _embed_iframe.extract,
|
"siskavideo": _embed_iframe.extract,
|
||||||
"porn4dayspw": _embed_iframe.extract,
|
"porn4dayspw": _embed_iframe.extract,
|
||||||
"porndishcom": _embed_iframe.extract,
|
# Wlasny ekstraktor: obejscie cache serwisu, ktory oddawal cudze wpisy (2026-10-15).
|
||||||
|
"porndishcom": porndish.extract,
|
||||||
# xxxfreewatch, DELISTED 2026-05-18. 790 solo-orphan scen, 0% match, CF-walled z VPS.
|
# xxxfreewatch, DELISTED 2026-05-18. 790 solo-orphan scen, 0% match, CF-walled z VPS.
|
||||||
"latestleaksco": _embed_iframe.extract,
|
"latestleaksco": _embed_iframe.extract,
|
||||||
"mypornerleakcom": _embed_iframe.extract,
|
"mypornerleakcom": _embed_iframe.extract,
|
||||||
|
|
|
||||||
|
|
@ -141,12 +141,29 @@ def _extract_ldjson_only(html: str, page_url: str) -> int | None:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _extract_itemprop_only(html: str, page_url: str) -> int | None:
|
||||||
|
"""mypornerleak: `<meta itemprop="duration" content="P0DT0H12M13S">` w <head> to
|
||||||
|
dlugosc TEJ sceny; karty powiazane maja tylko `<span class="duration">40:33</span>`,
|
||||||
|
ktorych tu nie czytamy. Sprawdzone 2026-10-14 na stronach z nowego listingu REST
|
||||||
|
(ingest przez wp-json nie niesie dlugosci, stad 76 tys. scen bez niej)."""
|
||||||
|
if (m := _ITEMPROP_DURATION_RE.search(html)):
|
||||||
|
v = m.group(1).strip()
|
||||||
|
if v.upper().startswith("P") and (parsed := _parse_iso8601(v)) is not None:
|
||||||
|
if 0 < parsed <= _MAX_SANE_DURATION_SEC:
|
||||||
|
return parsed
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
#: Origin → wzorzec. Dopisuj TYLKO to, co sprawdziłeś na próbce prawdziwych stron;
|
#: Origin → wzorzec. Dopisuj TYLKO to, co sprawdziłeś na próbce prawdziwych stron;
|
||||||
#: porntrex i mypornerleak celowo tu nie ma, patrz `app/models/duration_backfill_queue.py`.
|
#: porntrex celowo tu nie ma, patrz `app/models/duration_backfill_queue.py`.
|
||||||
|
#: perverzija: ingest z wp-json nie ma dlugosci, strona ma JSON-LD VideoObject
|
||||||
|
#: `"duration":"PT73M2S"` (2026-10-14, 21 tys. scen bez dlugosci).
|
||||||
ORIGIN_DURATION_RESOLVERS = {
|
ORIGIN_DURATION_RESOLVERS = {
|
||||||
"tube:xhamstercom": _extract_xhamster,
|
"tube:xhamstercom": _extract_xhamster,
|
||||||
"tube:xvideoscom": _extract_ldjson_only,
|
"tube:xvideoscom": _extract_ldjson_only,
|
||||||
"tube:xnxxcom": _extract_ldjson_only,
|
"tube:xnxxcom": _extract_ldjson_only,
|
||||||
|
"tube:perverzijacom": _extract_ldjson_only,
|
||||||
|
"tube:mypornerleakcom": _extract_itemprop_only,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -113,6 +113,10 @@ _HOSTER_FALLBACK_URL_RE = re.compile(
|
||||||
# Sygnatury "video not found" / "deleted" które hostery wstawiają w HTML embed page.
|
# Sygnatury "video not found" / "deleted" które hostery wstawiają w HTML embed page.
|
||||||
# Gdy widzimy te markery, to wiemy że link jest martwy, raise HosterDead, caller w
|
# Gdy widzimy te markery, to wiemy że link jest martwy, raise HosterDead, caller w
|
||||||
# playback.py oznaczy playback_source.dead_at.
|
# playback.py oznaczy playback_source.dead_at.
|
||||||
|
_NOWPLAY_INNER_RE = re.compile(
|
||||||
|
r"""<iframe[^>]+src=["']((?:https?:)?//[^"']*nowplay\.[a-z]+/emb[^"']*)["']""", re.IGNORECASE
|
||||||
|
)
|
||||||
|
|
||||||
_HOSTER_DEAD_PATTERNS = (
|
_HOSTER_DEAD_PATTERNS = (
|
||||||
"Video not found",
|
"Video not found",
|
||||||
"video not found",
|
"video not found",
|
||||||
|
|
@ -266,6 +270,15 @@ def extract_stream_from_hoster(
|
||||||
return None
|
return None
|
||||||
html = r.text
|
html = r.text
|
||||||
|
|
||||||
|
# nowplay.to (sxyland od 10.2026): gdy plik skasowano, strona emb nie ma packera, tylko
|
||||||
|
# ramke do INNEGO wideo nowplay (57-minutowy zastepczy film, wspolny dla wszystkich
|
||||||
|
# skasowanych). Bez tego telefon dostawal hoster i gral cudzy film w WebView.
|
||||||
|
if "nowplay." in iframe_url.lower() and "eval(function(p,a,c,k,e" not in html:
|
||||||
|
inner = _NOWPLAY_INNER_RE.findall(html)
|
||||||
|
own = iframe_url.split("//", 1)[-1].rstrip("/")
|
||||||
|
if inner and all(i.split("//", 1)[-1].rstrip("/") != own for i in inner):
|
||||||
|
raise HosterDead(f"hoster {iframe_url}: plik usuniety, nowplay podstawia zastepczy film")
|
||||||
|
|
||||||
if any(p in html for p in _HOSTER_DEAD_PATTERNS):
|
if any(p in html for p in _HOSTER_DEAD_PATTERNS):
|
||||||
raise HosterDead(f"hoster {iframe_url} reports video deleted/not found")
|
raise HosterDead(f"hoster {iframe_url} reports video deleted/not found")
|
||||||
|
|
||||||
|
|
@ -273,7 +286,9 @@ def extract_stream_from_hoster(
|
||||||
"""Iterate matches, pomiń preroll/ad URLs. Zwraca pierwszy clean lub None."""
|
"""Iterate matches, pomiń preroll/ad URLs. Zwraca pierwszy clean lub None."""
|
||||||
for m in pattern.finditer(text):
|
for m in pattern.finditer(text):
|
||||||
url = m.group(group)
|
url = m.group(group)
|
||||||
if not _looks_like_ad(url):
|
# Link "Download" do nitroflare/rapidgator konczy sie na .mp4, a to strona
|
||||||
|
# hostingu plikow (mangoporn 2026-10: 95 bledow ExoPlayera przez proxy).
|
||||||
|
if not _looks_like_ad(url) and not _FILE_HOSTER_RE.search(url):
|
||||||
return url
|
return url
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
@ -356,7 +371,7 @@ def _try_ytdlp_hoster(iframe_url: str, *, timeout: float) -> str | None:
|
||||||
def _looks_like_video_url(u: str | None) -> bool:
|
def _looks_like_video_url(u: str | None) -> bool:
|
||||||
if not u:
|
if not u:
|
||||||
return False
|
return False
|
||||||
if _looks_like_ad(u):
|
if _looks_like_ad(u) or _FILE_HOSTER_RE.search(u):
|
||||||
return False
|
return False
|
||||||
low = u.lower()
|
low = u.lower()
|
||||||
# Standardowe formaty video. yt-dlp generic czasem zwraca page URL jako
|
# Standardowe formaty video. yt-dlp generic czasem zwraca page URL jako
|
||||||
|
|
|
||||||
153
app/extractors/hosters/xtremestream.py
Normal file
153
app/extractors/hosters/xtremestream.py
Normal file
|
|
@ -0,0 +1,153 @@
|
||||||
|
"""xtremestream (player perverzija): manifest HLS z segmentami AES-128 skladany po stronie backendu.
|
||||||
|
|
||||||
|
Od ok. 2026-10-12 player (`ftk_engine.js`, zaciemniony, z blokada devtools) nie gra juz
|
||||||
|
z `xs1.php`: segmenty pod starymi adresami `/cdn/down/<id>/<q>/<q><i>.html` oddaja 404,
|
||||||
|
a prawdziwe leza pod `/cdn/down/<id>/<q>_enc/<q><i>.html` i sa zaszyfrowane AES-128-CBC,
|
||||||
|
IV = numer segmentu (16 bajtow big-endian), dopelnienie PKCS7, czyli dokladnie tak jak
|
||||||
|
standardowe `#EXT-X-KEY:METHOD=AES-128` bez atrybutu IV (sprawdzone 2026-10-14: openssl
|
||||||
|
z kluczem i IV=0 daje poprawny MPEG-TS, ffprobe widzi h264 i aac).
|
||||||
|
|
||||||
|
Klucz (16 B) oddaje `proto_api.php?data=<id>&op=key&vt=<vt>` z naglowkiem `X-XS-P: 1`.
|
||||||
|
`vt` to token ze strony playera (`XSFTK.boot(id, vt, player)`), zwiazany z IP i UA
|
||||||
|
pobierajacego (ten sam vt z innego IP = 403 `forbidden-vt`), ale klucz jest jeden na film,
|
||||||
|
niezalezny od vt (sprawdzone z dwoch IP). Segmenty i stary manifest `xs1.php` (nadal
|
||||||
|
serwowany, z dobrymi dlugosciami i rotacja hostow cdn[i % n]) nie sa zwiazane z IP.
|
||||||
|
|
||||||
|
Dlatego telefon nic nie musi robic: backend bierze `xs1.php`, przepisuje sciezki na `_enc`,
|
||||||
|
dokleja `#EXT-X-KEY` z URI wzglednym `key.bin`, a `/proxy/hls/<token>/key.bin` oddaje klucz
|
||||||
|
pobrany na wlasny vt. Telefon ciagnie segmenty prosto z CDN (przez VPS leci ~50 kB manifestu
|
||||||
|
i 16 B klucza). Czesc katalogu (starszy storage) ma nadal zwykle segmenty: wtedy manifest
|
||||||
|
idzie bez klucza, tak jak dotad.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import re
|
||||||
|
import time
|
||||||
|
from urllib.parse import parse_qs, quote, urlparse
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
_UA = (
|
||||||
|
"Mozilla/5.0 (Linux; Android 14; Pixel 7) AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||||
|
"Chrome/128.0.0.0 Mobile Safari/537.36"
|
||||||
|
)
|
||||||
|
_PLAYER_RE = re.compile(
|
||||||
|
r"https?://(?P<host>[a-z0-9\-]+\.xtremestream\.[a-z]+)/player/index\.php\?data=(?P<id>[0-9a-f]{16,64})",
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
_BOOT_RE = re.compile(r'XSFTK\.boot\("[0-9a-f]+",\s*"([^"]+)"')
|
||||||
|
# `https://pervl7.xspcdn01.click/cdn/down/<id>/720/7200.html` -> grupy: prefiks, jakosc, numer
|
||||||
|
_SEG_RE = re.compile(r"^(https?://[^/]+/cdn/down/[0-9a-f]+/)(\d+p?)/\2(\d+)\.html\s*$")
|
||||||
|
_DEFAULT_QUALITY = "720"
|
||||||
|
_KEY_TTL_SEC = 6 * 3600
|
||||||
|
_key_cache: dict[str, tuple[bytes, float]] = {}
|
||||||
|
|
||||||
|
|
||||||
|
def parse_player(player_url: str) -> tuple[str, str] | None:
|
||||||
|
m = _PLAYER_RE.search(player_url)
|
||||||
|
return (m.group("host"), m.group("id")) if m else None
|
||||||
|
|
||||||
|
|
||||||
|
def quality_of(player_url: str) -> str:
|
||||||
|
q = parse_qs(urlparse(player_url).query).get("q", [_DEFAULT_QUALITY])[0]
|
||||||
|
return q if re.fullmatch(r"\d{3,4}", q) else _DEFAULT_QUALITY
|
||||||
|
|
||||||
|
|
||||||
|
def _to_enc(line: str) -> str:
|
||||||
|
m = _SEG_RE.match(line)
|
||||||
|
if not m:
|
||||||
|
return line
|
||||||
|
return f"{m.group(1)}{m.group(2)}_enc/{m.group(2)}{m.group(3)}.html"
|
||||||
|
|
||||||
|
|
||||||
|
def _reachable(url: str, timeout: float) -> bool:
|
||||||
|
try:
|
||||||
|
r = httpx.get(url, headers={"User-Agent": _UA, "Range": "bytes=0-0"}, timeout=timeout)
|
||||||
|
return r.status_code in (200, 206)
|
||||||
|
except httpx.HTTPError:
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def fetch_key(player_url: str, *, timeout: float = 20.0) -> bytes | None:
|
||||||
|
"""16-bajtowy klucz AES filmu, z pamiecia podreczna na 6 h (klucz nie zalezy od vt)."""
|
||||||
|
parsed = parse_player(player_url)
|
||||||
|
if not parsed:
|
||||||
|
return None
|
||||||
|
host, vid = parsed
|
||||||
|
cached = _key_cache.get(vid)
|
||||||
|
if cached and time.time() - cached[1] < _KEY_TTL_SEC:
|
||||||
|
return cached[0]
|
||||||
|
index_url = f"https://{host}/player/index.php?data={vid}"
|
||||||
|
try:
|
||||||
|
page = httpx.get(
|
||||||
|
index_url,
|
||||||
|
headers={"User-Agent": _UA, "Referer": "https://tube.perverzija.com/"},
|
||||||
|
timeout=timeout,
|
||||||
|
follow_redirects=True,
|
||||||
|
)
|
||||||
|
m = _BOOT_RE.search(page.text)
|
||||||
|
if not m:
|
||||||
|
log.info("xtremestream: brak tokenu vt na %s (http %s)", index_url, page.status_code)
|
||||||
|
return None
|
||||||
|
vt = m.group(1)
|
||||||
|
# UA musi byc ten sam co przy stronie playera, vt jest pod niego liczony.
|
||||||
|
k = httpx.get(
|
||||||
|
f"https://{host}/player/proto_api.php?data={vid}&op=key&vt={quote(vt, safe='')}",
|
||||||
|
headers={"User-Agent": _UA, "X-XS-P": "1", "Referer": index_url},
|
||||||
|
timeout=timeout,
|
||||||
|
)
|
||||||
|
except httpx.HTTPError as e:
|
||||||
|
log.info("xtremestream: pobranie klucza padlo dla %s: %s", vid, e)
|
||||||
|
return None
|
||||||
|
if k.status_code != 200 or len(k.content) != 16:
|
||||||
|
log.info("xtremestream: klucz dla %s http %s, %d B", vid, k.status_code, len(k.content))
|
||||||
|
return None
|
||||||
|
_key_cache[vid] = (k.content, time.time())
|
||||||
|
return k.content
|
||||||
|
|
||||||
|
|
||||||
|
def build_manifest(player_url: str, *, key_uri: str = "key.bin", timeout: float = 20.0) -> str | None:
|
||||||
|
"""Manifest HLS do odegrania przez telefon: segmenty `_enc` z CDN i klucz pod `key_uri`
|
||||||
|
(wzgledny wobec adresu manifestu). None, gdy xs1.php nie oddaje manifestu albo zaden
|
||||||
|
wariant segmentow (zaszyfrowany ani zwykly) nie odpowiada."""
|
||||||
|
parsed = parse_player(player_url)
|
||||||
|
if not parsed:
|
||||||
|
return None
|
||||||
|
host, vid = parsed
|
||||||
|
q = quality_of(player_url)
|
||||||
|
index_url = f"https://{host}/player/index.php?data={vid}"
|
||||||
|
try:
|
||||||
|
m = httpx.get(
|
||||||
|
f"https://{host}/player/xs1.php?data={vid}&q={q}",
|
||||||
|
headers={"User-Agent": _UA, "Referer": index_url},
|
||||||
|
timeout=timeout,
|
||||||
|
)
|
||||||
|
except httpx.HTTPError as e:
|
||||||
|
log.info("xtremestream: xs1.php padlo dla %s: %s", vid, e)
|
||||||
|
return None
|
||||||
|
text = m.text
|
||||||
|
if m.status_code != 200 or not text.lstrip(" \t\r\n").startswith("#EXTM3U"):
|
||||||
|
log.info("xtremestream: xs1.php dla %s http %s, nie-manifest (%d B)", vid, m.status_code, len(m.content))
|
||||||
|
return None
|
||||||
|
lines = text.splitlines()
|
||||||
|
segments = [l for l in lines if l.startswith("http")]
|
||||||
|
if not segments:
|
||||||
|
return None
|
||||||
|
first_enc = _to_enc(segments[0])
|
||||||
|
if first_enc == segments[0] or not _reachable(first_enc, timeout):
|
||||||
|
# Starszy storage: zwykle segmenty, manifest bez zmian. Jesli i te nie odpowiadaja,
|
||||||
|
# film jest martwy po stronie hostera.
|
||||||
|
return text if _reachable(segments[0], timeout) else None
|
||||||
|
out: list[str] = []
|
||||||
|
key_done = False
|
||||||
|
for line in lines:
|
||||||
|
out.append(_to_enc(line) if line.startswith("http") else line)
|
||||||
|
if not key_done and line.startswith("#EXT-X-MEDIA-SEQUENCE"):
|
||||||
|
out.append(f'#EXT-X-KEY:METHOD=AES-128,URI="{key_uri}"')
|
||||||
|
key_done = True
|
||||||
|
if not key_done:
|
||||||
|
out.insert(1, f'#EXT-X-KEY:METHOD=AES-128,URI="{key_uri}"')
|
||||||
|
return "\n".join(out) + "\n"
|
||||||
|
|
@ -26,6 +26,7 @@ from urllib.parse import urlparse
|
||||||
from app.extractors._fetch import fetch_tube_html
|
from app.extractors._fetch import fetch_tube_html
|
||||||
from app.extractors._models import StreamSource
|
from app.extractors._models import StreamSource
|
||||||
from app.extractors.hoster import extract_stream_from_hoster
|
from app.extractors.hoster import extract_stream_from_hoster
|
||||||
|
from app.extractors._models import HosterDead
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
@ -253,6 +254,11 @@ def _is_player_iframe(url: str) -> bool:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
_PREVIEW_ATTR_RE = re.compile(
|
||||||
|
r"""\sdata-(?:trailer|preview)[\w-]*\s*=\s*(?:"[^"]*"|'[^']*')""", re.IGNORECASE
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _extract_direct_stream_urls(page_html: str, page_url: str) -> list[StreamSource]:
|
def _extract_direct_stream_urls(page_html: str, page_url: str) -> list[StreamSource]:
|
||||||
"""Stage 0.5: scan page for direct mp4/m3u8 URLs (porn4days→iceyfile/gounlimited
|
"""Stage 0.5: scan page for direct mp4/m3u8 URLs (porn4days→iceyfile/gounlimited
|
||||||
download links, xmoviesforyou native player). Returns deduplicated sources
|
download links, xmoviesforyou native player). Returns deduplicated sources
|
||||||
|
|
@ -272,6 +278,11 @@ def _extract_direct_stream_urls(page_html: str, page_url: str) -> list[StreamSou
|
||||||
page_host = (urlparse(page_url).hostname or "").lstrip("www.")
|
page_host = (urlparse(page_url).hostname or "").lstrip("www.")
|
||||||
page_referer = f"https://{page_host}/" if page_host else page_url
|
page_referer = f"https://{page_host}/" if page_host else page_url
|
||||||
|
|
||||||
|
# Kafelki "podobne" maja zwiastuny w data-trailer/data-preview; sxyland oddawal
|
||||||
|
# 25-sekundowy cudzy trailer (gammacdn .../tr_..._720p.mp4) jako film sceny, losowo,
|
||||||
|
# bo kafelki zmieniaja sie przy kazdym pobraniu (zgloszenie 23269128, 2026-10-12).
|
||||||
|
page_html = _PREVIEW_ATTR_RE.sub("", page_html)
|
||||||
|
|
||||||
for m in _DIRECT_MP4_RE.finditer(page_html):
|
for m in _DIRECT_MP4_RE.finditer(page_html):
|
||||||
url = m.group("url")
|
url = m.group("url")
|
||||||
if url in seen:
|
if url in seen:
|
||||||
|
|
@ -290,6 +301,8 @@ def _extract_direct_stream_urls(page_html: str, page_url: str) -> list[StreamSou
|
||||||
low = url.lower()
|
low = url.lower()
|
||||||
if "videos_screenshots/" in low or "/preview_" in low:
|
if "videos_screenshots/" in low or "/preview_" in low:
|
||||||
continue
|
continue
|
||||||
|
if "/trailers/" in low or "//trailers-" in low or "/trailer/" in low:
|
||||||
|
continue
|
||||||
if "/get_file/" in low and "?" not in url:
|
if "/get_file/" in low and "?" not in url:
|
||||||
continue
|
continue
|
||||||
# Quality wykrycie z nazwy pliku
|
# Quality wykrycie z nazwy pliku
|
||||||
|
|
@ -322,7 +335,11 @@ def extract(
|
||||||
page_url: str,
|
page_url: str,
|
||||||
*,
|
*,
|
||||||
timeout: float = 60.0,
|
timeout: float = 60.0,
|
||||||
|
page_html: str | None = None,
|
||||||
) -> list[StreamSource] | None:
|
) -> list[StreamSource] | None:
|
||||||
|
# `page_html` podaje ekstraktor, ktory musial pobrac strone po swojemu (porndish:
|
||||||
|
# obejscie zepsutego cache serwisu i kontrola, czy to wlasciwy wpis).
|
||||||
|
if page_html is None:
|
||||||
page_html = fetch_tube_html(page_url, timeout=timeout)
|
page_html = fetch_tube_html(page_url, timeout=timeout)
|
||||||
|
|
||||||
# Stage 0.5: direct .mp4/.m3u8 URLs on page (porn4days→iceyfile, niektóre xmoviesforyou
|
# Stage 0.5: direct .mp4/.m3u8 URLs on page (porn4days→iceyfile, niektóre xmoviesforyou
|
||||||
|
|
@ -482,11 +499,20 @@ def extract(
|
||||||
# URL z embed page, ale ani VPS proxy ani mobile go nie pobierze (No route to host
|
# URL z embed page, ale ani VPS proxy ani mobile go nie pobierze (No route to host
|
||||||
# albo 403). Bez iframe fallbacku mobile dostaje 503 i utyka. Z fallbackiem chain:
|
# albo 403). Bez iframe fallbacku mobile dostaje 503 i utyka. Z fallbackiem chain:
|
||||||
# direct → proxy → iframe1 (WebView) → iframe2 (WebView) → page (WebView).
|
# direct → proxy → iframe1 (WebView) → iframe2 (WebView) → page (WebView).
|
||||||
|
# Ramki, o ktorych hoster mowi wprost "plik usuniety" w sposob pewny (na razie tylko
|
||||||
|
# zastepczy film nowplay). Ogolnego HosterDead tu nie liczymy: strony DoodStream maja
|
||||||
|
# ukryty blok "not found" w kazdym HTML-u.
|
||||||
|
dead_iframes: set[str] = set()
|
||||||
for idx, iframe_url in enumerate(iframe_urls):
|
for idx, iframe_url in enumerate(iframe_urls):
|
||||||
try:
|
try:
|
||||||
stream_url = extract_stream_from_hoster(
|
stream_url = extract_stream_from_hoster(
|
||||||
iframe_url, referer=referer, timeout=timeout,
|
iframe_url, referer=referer, timeout=timeout,
|
||||||
)
|
)
|
||||||
|
except HosterDead as e:
|
||||||
|
if "nowplay." in iframe_url.lower():
|
||||||
|
dead_iframes.add(iframe_url)
|
||||||
|
log.info("embed_iframe: %s", e)
|
||||||
|
continue
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.warning("embed_iframe: extract failed for %s: %s", iframe_url, e)
|
log.warning("embed_iframe: extract failed for %s: %s", iframe_url, e)
|
||||||
continue
|
continue
|
||||||
|
|
@ -508,7 +534,7 @@ def extract(
|
||||||
# jako hoster fallback. WebView załaduje embed page z własną sesją/IP/cookies,
|
# jako hoster fallback. WebView załaduje embed page z własną sesją/IP/cookies,
|
||||||
# może pobrać video gdy VPS proxy blocked.
|
# może pobrać video gdy VPS proxy blocked.
|
||||||
for u in iframe_urls:
|
for u in iframe_urls:
|
||||||
if u == iframe_url:
|
if u == iframe_url or u in dead_iframes:
|
||||||
continue
|
continue
|
||||||
host_label = (urlparse(u).hostname or "").lstrip("www.").split(".")[0] or None
|
host_label = (urlparse(u).hostname or "").lstrip("www.").split(".")[0] or None
|
||||||
sources.append(StreamSource(link=u, type="hoster", quality=host_label))
|
sources.append(StreamSource(link=u, type="hoster", quality=host_label))
|
||||||
|
|
@ -525,8 +551,12 @@ def extract(
|
||||||
# vs "luluvid"). Na końcu page_url jako safety-net, gdy wszystkie iframe'y są
|
# vs "luluvid"). Na końcu page_url jako safety-net, gdy wszystkie iframe'y są
|
||||||
# martwe (np. sxyland → xtapes.porn 301→camcaps.to 403), mobile WebView otworzy
|
# martwe (np. sxyland → xtapes.porn 301→camcaps.to 403), mobile WebView otworzy
|
||||||
# główną stronę aggregator tube'a, gdzie user może wybrać alternatywny player.
|
# główną stronę aggregator tube'a, gdzie user może wybrać alternatywny player.
|
||||||
|
if iframe_urls and all(u in dead_iframes for u in iframe_urls):
|
||||||
|
raise HosterDead(f"wszystkie odtwarzacze na {page_url} maja usuniety plik")
|
||||||
sources = []
|
sources = []
|
||||||
for u in iframe_urls:
|
for u in iframe_urls:
|
||||||
|
if u in dead_iframes:
|
||||||
|
continue
|
||||||
host_label = (urlparse(u).hostname or "").lstrip("www.").split(".")[0] or None
|
host_label = (urlparse(u).hostname or "").lstrip("www.").split(".")[0] or None
|
||||||
sources.append(StreamSource(link=u, type="hoster", quality=host_label))
|
sources.append(StreamSource(link=u, type="hoster", quality=host_label))
|
||||||
page_host_label = (urlparse(page_url).hostname or "").lstrip("www.").split(".")[0] or "page"
|
page_host_label = (urlparse(page_url).hostname or "").lstrip("www.").split(".")[0] or "page"
|
||||||
|
|
|
||||||
|
|
@ -24,7 +24,7 @@ import time
|
||||||
import urllib.parse as _up
|
import urllib.parse as _up
|
||||||
|
|
||||||
from app.extractors._fetch import _DEFAULT_IMPERSONATE, _DEFAULT_UA, _HAS_CURL_CFFI
|
from app.extractors._fetch import _DEFAULT_IMPERSONATE, _DEFAULT_UA, _HAS_CURL_CFFI
|
||||||
from app.extractors._models import StreamSource
|
from app.extractors._models import StreamSource, TubePageError
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
@ -134,6 +134,11 @@ def resolve_kvs(page_url: str, *, base_url: str, timeout: float = 60.0) -> list[
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.info("kvs: page fetch failed %s: %s", page_url, e)
|
log.info("kvs: page fetch failed %s: %s", page_url, e)
|
||||||
return None
|
return None
|
||||||
|
if resp.status_code in (404, 410):
|
||||||
|
# Usuniety film: TubePageError -> API oznacza zrodlo martwe (410). Wczesniej
|
||||||
|
# zwracalismy None = "chwilowy blad" (503) bez konca: ok. polowa z 39 tys. zrodel
|
||||||
|
# yespornvip to strony 404, pokazywane jako zywe (zgloszenie 5e81eb76, 2026-10-15).
|
||||||
|
raise TubePageError(resp.status_code, page_url)
|
||||||
if not html:
|
if not html:
|
||||||
log.info("kvs: empty page %s", page_url)
|
log.info("kvs: empty page %s", page_url)
|
||||||
return None
|
return None
|
||||||
|
|
|
||||||
31
app/extractors/tubes/fpoxxx.py
Normal file
31
app/extractors/tubes/fpoxxx.py
Normal file
|
|
@ -0,0 +1,31 @@
|
||||||
|
"""fpo.xxx: kontrola usunietego filmu przed oddaniem strony do WebView.
|
||||||
|
|
||||||
|
Odtwarzanie idzie na telefonie (`fpoxxxResolver.ts`, KVS license_code z IP telefonu). Gdy
|
||||||
|
telefon nic nie znajdzie, backend oddawal sama strone jako hoster. Na stronie skasowanego
|
||||||
|
filmu (`<div class="no-player">`, "Deleted by the owner request", bez license_code) jedynym
|
||||||
|
wideo jest reklama tsyndicate, ktora WebView wylapywal i puszczal jako scene (zgloszenie
|
||||||
|
1ee12943, 2026-10-12: "fpoxxx daje reklamy zamiast scen"; ok. 10% zrodel fpoxxx to takie
|
||||||
|
strony). Tu taka strona to HosterDead, wiec zrodlo znika zamiast grac reklame.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
|
||||||
|
from app.extractors._fetch import fetch_tube_html
|
||||||
|
from app.extractors._models import HosterDead, StreamSource, TubePageError
|
||||||
|
from app.extractors.tubes import _vps_blocked_fallback
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def extract(page_url: str, *, timeout: float = 60.0) -> list[StreamSource]:
|
||||||
|
try:
|
||||||
|
html = fetch_tube_html(page_url, timeout=min(timeout, 20.0))
|
||||||
|
except TubePageError:
|
||||||
|
raise
|
||||||
|
except Exception as e:
|
||||||
|
log.info("fpoxxx: strona %s niepobrana (%s), oddaje do WebView", page_url, e)
|
||||||
|
return _vps_blocked_fallback.extract(page_url, timeout=timeout)
|
||||||
|
if 'class="no-player"' in html and "license_code" not in html:
|
||||||
|
raise HosterDead(f"fpoxxx {page_url}: film usuniety (no-player)")
|
||||||
|
return _vps_blocked_fallback.extract(page_url, timeout=timeout)
|
||||||
|
|
@ -14,6 +14,13 @@ ale nie trzeba go składać ręcznie — bierzemy iframe taki, jaki przyszedł.
|
||||||
`admin-ajax.php` dostaje challenge Cloudflare (403, „Just a moment..."), dokładnie jak
|
`admin-ajax.php` dostaje challenge Cloudflare (403, „Just a moment..."), dokładnie jak
|
||||||
`POST /api/video-info` przy sexu. Bez klucza proxy extractor zwraca None.
|
`POST /api/video-info` przy sexu. Bez klucza proxy extractor zwraca None.
|
||||||
|
|
||||||
|
**2026-10-15 (zgloszenie 6ab3a973, zadna scena nie grala):** strona zmienila zapis na
|
||||||
|
`{action: 'get_player_contents',a: '79669'}` (spacje), a odpowiedz admin-ajax ma 5 playerow
|
||||||
|
(`?dt_embed=rtmp|hls|hls-mp4|r2-mp4` i `newjav.php`) z `&#038;` w src. Strony playerow
|
||||||
|
trzymaja adresy w JSON-ie (ukosniki zapisane jako backslash i slash). javprovider daje 404 dla nowych odcinkow, gdvid
|
||||||
|
(`/hls/output/.../mp4/<jakosc>/...mp4`, bez tokenu) gra wszedzie. POST przechodzi dzis z VPS-a
|
||||||
|
bez proxy, Bright Data zostaje tylko na wypadek 403/Cloudflare.
|
||||||
|
|
||||||
**Preferujemy `gdvid.info` nad `javprovider.com`**: pierwszy oddaje plik BEZ tokenu,
|
**Preferujemy `gdvid.info` nad `javprovider.com`**: pierwszy oddaje plik BEZ tokenu,
|
||||||
więc URL nigdy nie wygasa; drugi ma `?st=...&e=<epoch>`. Oba zweryfikowane z VPS-a:
|
więc URL nigdy nie wygasa; drugi ma `?st=...&e=<epoch>`. Oba zweryfikowane z VPS-a:
|
||||||
206 na Range, `ftypisom` w pierwszym KB (faststart), działają BEZ Referera — stąd
|
206 na Range, `ftypisom` w pierwszym KB (faststart), działają BEZ Referera — stąd
|
||||||
|
|
@ -21,6 +28,7 @@ więc URL nigdy nie wygasa; drugi ma `?st=...&e=<epoch>`. Oba zweryfikowane z VP
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import html
|
||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
import re
|
import re
|
||||||
|
|
@ -32,11 +40,13 @@ from app.extractors._models import StreamSource
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
_BASE = "https://hentaimama.io"
|
_BASE = "https://hentaimama.io"
|
||||||
_POST_ID_RE = re.compile(r"action:'get_player_contents',\s*a:'(\d+)'")
|
_POST_ID_RE = re.compile(r"""action:\s*['"]get_player_contents['"],\s*a:\s*['"](\d+)['"]""")
|
||||||
|
_BODY_POST_ID_RE = re.compile(r"\bpostid-(\d+)\b")
|
||||||
_IFRAME_SRC_RE = re.compile(r'<iframe[^>]+src="([^"]+)"', re.IGNORECASE)
|
_IFRAME_SRC_RE = re.compile(r'<iframe[^>]+src="([^"]+)"', re.IGNORECASE)
|
||||||
_MP4_RE = re.compile(r'https?://[^\s"\'<>]+\.mp4[^\s"\'<>]*')
|
_VIDEO_RE = re.compile(r"""https?://[^\s"'<>]+\.(?:mp4|m3u8)[^\s"'<>]*""")
|
||||||
#: Host bez tokenu idzie pierwszy — jego URL nie wygasa.
|
_QUALITY_RE = re.compile(r"/mp4/(\d{3,4})p/")
|
||||||
_HOST_RANK = ("gdvid", "javprovider")
|
#: Host bez tokenu idzie pierwszy, jego URL nie wygasa. javprovider na koniec (404 na nowych).
|
||||||
|
_HOST_RANK = ("gdvid.info/hls/", "gdvid", "hentaidoge", "javprovider")
|
||||||
|
|
||||||
|
|
||||||
def _rank(url: str) -> int:
|
def _rank(url: str) -> int:
|
||||||
|
|
@ -46,42 +56,55 @@ def _rank(url: str) -> int:
|
||||||
return len(_HOST_RANK)
|
return len(_HOST_RANK)
|
||||||
|
|
||||||
|
|
||||||
def extract(page_url: str, *, timeout: float = 60.0) -> list[StreamSource] | None:
|
def _quality(url: str) -> int:
|
||||||
proxy = get_settings().brightdata_proxy_url
|
m = _QUALITY_RE.search(url)
|
||||||
if not proxy:
|
return int(m.group(1)) if m else 0
|
||||||
log.info("hentaimama: brak BRIGHTDATA_PROXY_URL — CF zablokuje POST admin-ajax")
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
|
def _player_contents(page_url: str, post_id: str, timeout: float):
|
||||||
|
"""POST get_player_contents: najpierw bez proxy, Bright Data tylko przy blokadzie."""
|
||||||
|
from curl_cffi import requests as cr
|
||||||
|
|
||||||
|
proxy = get_settings().brightdata_proxy_url
|
||||||
|
attempts = [None] + ([proxy] if proxy else [])
|
||||||
|
for px in attempts:
|
||||||
|
session = cr.Session(
|
||||||
|
impersonate="chrome120",
|
||||||
|
proxies={"http": px, "https": px} if px else None,
|
||||||
|
verify=False,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
# Strona odcinka najpierw, bez jej ciasteczek CF potrafi odrzucic POST-a.
|
||||||
|
session.get(page_url, timeout=timeout)
|
||||||
|
res = session.post(
|
||||||
|
f"{_BASE}/wp-admin/admin-ajax.php",
|
||||||
|
data={"action": "get_player_contents", "a": post_id},
|
||||||
|
headers={"Referer": page_url, "X-Requested-With": "XMLHttpRequest"},
|
||||||
|
timeout=timeout,
|
||||||
|
)
|
||||||
|
if res.status_code == 200:
|
||||||
|
return session, json.loads(res.text)
|
||||||
|
log.info("hentaimama: admin-ajax %s (proxy=%s) dla %s", res.status_code, bool(px), page_url)
|
||||||
|
except Exception as e:
|
||||||
|
log.info("hentaimama: get_player_contents (proxy=%s) %s: %s", bool(px), page_url, e)
|
||||||
|
return None, None
|
||||||
|
|
||||||
|
|
||||||
|
def extract(page_url: str, *, timeout: float = 60.0) -> list[StreamSource] | None:
|
||||||
html_text = fetch_tube_html(page_url, timeout=timeout)
|
html_text = fetch_tube_html(page_url, timeout=timeout)
|
||||||
pm = _POST_ID_RE.search(html_text)
|
pm = _POST_ID_RE.search(html_text) or _BODY_POST_ID_RE.search(html_text)
|
||||||
if not pm:
|
if not pm:
|
||||||
log.info("hentaimama: brak id posta na %s", page_url)
|
log.info("hentaimama: brak id posta na %s", page_url)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
from curl_cffi import requests as cr
|
session, fields = _player_contents(page_url, pm.group(1), timeout)
|
||||||
|
if session is None:
|
||||||
try:
|
|
||||||
session = cr.Session(
|
|
||||||
impersonate="chrome120",
|
|
||||||
proxies={"http": proxy, "https": proxy},
|
|
||||||
verify=False,
|
|
||||||
)
|
|
||||||
# Strona odcinka najpierw — bez jej ciasteczek CF odrzuca POST-a.
|
|
||||||
session.get(page_url, timeout=timeout)
|
|
||||||
res = session.post(
|
|
||||||
f"{_BASE}/wp-admin/admin-ajax.php",
|
|
||||||
data={"action": "get_player_contents", "a": pm.group(1)},
|
|
||||||
headers={"Referer": page_url, "X-Requested-With": "XMLHttpRequest"},
|
|
||||||
timeout=timeout,
|
|
||||||
)
|
|
||||||
fields = json.loads(res.text)
|
|
||||||
except Exception as e:
|
|
||||||
log.info("hentaimama: get_player_contents failed %s: %s", page_url, e)
|
|
||||||
return None
|
return None
|
||||||
|
|
||||||
players: list[str] = []
|
players: list[str] = []
|
||||||
for field in fields if isinstance(fields, list) else []:
|
for field in fields if isinstance(fields, list) else []:
|
||||||
for src in _IFRAME_SRC_RE.findall(str(field)):
|
for src in _IFRAME_SRC_RE.findall(str(field)):
|
||||||
|
src = html.unescape(html.unescape(src))
|
||||||
if src.startswith("//"):
|
if src.startswith("//"):
|
||||||
src = "https:" + src
|
src = "https:" + src
|
||||||
if src.startswith("http") and src not in players:
|
if src.startswith("http") and src not in players:
|
||||||
|
|
@ -91,29 +114,29 @@ def extract(page_url: str, *, timeout: float = 60.0) -> list[StreamSource] | Non
|
||||||
seen: set[str] = set()
|
seen: set[str] = set()
|
||||||
for player_url in players:
|
for player_url in players:
|
||||||
try:
|
try:
|
||||||
body = session.get(
|
body = session.get(player_url, headers={"Referer": page_url}, timeout=timeout).text
|
||||||
player_url, headers={"Referer": page_url}, timeout=timeout
|
|
||||||
).text
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.info("hentaimama: player fetch failed %s: %s", player_url, e)
|
log.info("hentaimama: player fetch failed %s: %s", player_url, e)
|
||||||
continue
|
continue
|
||||||
for link in _MP4_RE.findall(body):
|
body = body.replace(chr(92) + "/", "/")
|
||||||
if link in seen:
|
for link in _VIDEO_RE.findall(body):
|
||||||
|
if link in seen or "thumbnail" in link or link.endswith(".vtt"):
|
||||||
continue
|
continue
|
||||||
seen.add(link)
|
seen.add(link)
|
||||||
|
q = _quality(link)
|
||||||
out.append(
|
out.append(
|
||||||
StreamSource(
|
StreamSource(
|
||||||
link=link,
|
link=link,
|
||||||
type="mp4",
|
type="m3u8" if ".m3u8" in link else "mp4",
|
||||||
quality=None, # serwis nie podaje rozdzielczości
|
quality=f"{q}p" if q else None,
|
||||||
referer=_BASE + "/",
|
referer=_BASE + "/",
|
||||||
# Zweryfikowane: 206 bez Referera, faststart → telefon gra direct.
|
# Zweryfikowane: 206 bez Referera, faststart, telefon gra direct.
|
||||||
raw={"mobile_direct_ok": True},
|
raw={"mobile_direct_ok": True},
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
if not out:
|
if not out:
|
||||||
log.info("hentaimama: nie znalazłem mp4 dla %s", page_url)
|
log.info("hentaimama: nie znalazlem wideo dla %s", page_url)
|
||||||
return None
|
return None
|
||||||
out.sort(key=lambda s: _rank(s.link))
|
out.sort(key=lambda s: (_rank(s.link), s.type != "mp4", -_quality(s.link)))
|
||||||
return out
|
return out
|
||||||
|
|
|
||||||
|
|
@ -26,11 +26,32 @@ from __future__ import annotations
|
||||||
import logging
|
import logging
|
||||||
import re
|
import re
|
||||||
|
|
||||||
|
from app.config import get_settings
|
||||||
from app.extractors._models import StreamSource
|
from app.extractors._models import StreamSource
|
||||||
from app.extractors import browser_get
|
from app.extractors import browser_get
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def fetch_perverzija(url: str, *, timeout: float):
|
||||||
|
"""GET strony perverzija, przy blokadzie Cloudflare przez Bright Data.
|
||||||
|
|
||||||
|
2026-10-15 ok. 04:31 UTC Cloudflare zaczal odpowiadac 403 z challenge'em na IPv4 VPS-a
|
||||||
|
(IPv6 dalej 200), dla stron scen i dla REST-a listingu. Ekstraktor nie patrzyl na status,
|
||||||
|
szedl w zapas embed-iframe, ten tez dostawal 403, API oddawalo 503 i perverzija
|
||||||
|
przestala grac wszystkim (zgloszenie 1c662356). Proxy (ryczalt, ISP) tylko dla HTML-a,
|
||||||
|
player xtremestream i segmenty ida jak dotad.
|
||||||
|
"""
|
||||||
|
res = browser_get(url, timeout=timeout)
|
||||||
|
if res.status_code != 403:
|
||||||
|
return res
|
||||||
|
proxy = get_settings().brightdata_proxy_url
|
||||||
|
if not proxy:
|
||||||
|
log.warning("perverzija: 403 (Cloudflare) na %s, brak proxy", url)
|
||||||
|
return res
|
||||||
|
log.info("perverzija: 403 na %s, ponawiam przez proxy", url)
|
||||||
|
return browser_get(url, timeout=timeout, proxy=proxy)
|
||||||
|
|
||||||
# iframe playera: https://pervl4.xtremestream.xyz/player/index.php?data=<32 hex>
|
# iframe playera: https://pervl4.xtremestream.xyz/player/index.php?data=<32 hex>
|
||||||
_PLAYER_IFRAME_RE = re.compile(
|
_PLAYER_IFRAME_RE = re.compile(
|
||||||
r"https?://(?P<host>[a-z0-9\-]+\.xtremestream\.[a-z]+)/player/index\.php\?data=(?P<id>[0-9a-f]{16,64})",
|
r"https?://(?P<host>[a-z0-9\-]+\.xtremestream\.[a-z]+)/player/index\.php\?data=(?P<id>[0-9a-f]{16,64})",
|
||||||
|
|
@ -44,11 +65,15 @@ _QUALITIES = ("720", "480")
|
||||||
|
|
||||||
def extract(page_url: str, *, timeout: float = 60.0) -> list[StreamSource] | None:
|
def extract(page_url: str, *, timeout: float = 60.0) -> list[StreamSource] | None:
|
||||||
try:
|
try:
|
||||||
res = browser_get(page_url, timeout=timeout)
|
res = fetch_perverzija(page_url, timeout=timeout)
|
||||||
html = res.text if hasattr(res, "text") else str(res)
|
html = res.text if hasattr(res, "text") else str(res)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.info("perverzija page fetch failed %s: %s", page_url, e)
|
log.info("perverzija page fetch failed %s: %s", page_url, e)
|
||||||
return None
|
return None
|
||||||
|
if getattr(res, "status_code", 200) == 403:
|
||||||
|
# Zapas embed-iframe pobiera strone jeszcze raz bez proxy i tez dostaje 403.
|
||||||
|
log.warning("perverzija: strona %s nadal 403", page_url)
|
||||||
|
return None
|
||||||
|
|
||||||
m = _PLAYER_IFRAME_RE.search(html)
|
m = _PLAYER_IFRAME_RE.search(html)
|
||||||
if not m:
|
if not m:
|
||||||
|
|
@ -68,11 +93,16 @@ def extract(page_url: str, *, timeout: float = 60.0) -> list[StreamSource] | Non
|
||||||
host, vid = m.group("host"), m.group("id")
|
host, vid = m.group("host"), m.group("id")
|
||||||
player_url = f"https://{host}/player/index.php?data={vid}"
|
player_url = f"https://{host}/player/index.php?data={vid}"
|
||||||
|
|
||||||
|
# Od 2026-10-12 segmenty sa szyfrowane (AES-128, klucz spod tokenu zwiazanego z IP),
|
||||||
|
# wiec manifest sklada backend w `/proxy/hls` (producent `xtremestream`, patrz
|
||||||
|
# extractors/hosters/xtremestream.py). Link to adres playera z jakoscia w zapytaniu;
|
||||||
|
# producent bierze z niego id i `q`. `mobile_direct_ok` zostaje: segmenty NIE sa
|
||||||
|
# IP-bound (test z dwoch IP 2026-10-14), przez VPS leci manifest i 16 B klucza.
|
||||||
out: list[StreamSource] = []
|
out: list[StreamSource] = []
|
||||||
for q in _QUALITIES:
|
for q in _QUALITIES:
|
||||||
out.append(
|
out.append(
|
||||||
StreamSource(
|
StreamSource(
|
||||||
link=f"https://{host}/player/xs1.php?data={vid}&q={q}",
|
link=f"{player_url}&q={q}",
|
||||||
quality=f"{q}p",
|
quality=f"{q}p",
|
||||||
type="m3u8",
|
type="m3u8",
|
||||||
referer=player_url,
|
referer=player_url,
|
||||||
|
|
@ -80,11 +110,7 @@ def extract(page_url: str, *, timeout: float = 60.0) -> list[StreamSource] | Non
|
||||||
"extractor": "perverzija",
|
"extractor": "perverzija",
|
||||||
"player": player_url,
|
"player": player_url,
|
||||||
"q": q,
|
"q": q,
|
||||||
# Segmenty NIE są IP-bound i nie potrzebują Referera: segment pobrany
|
"manifest_producer": "xtremestream",
|
||||||
# z domowego łącza po manifeście wziętym na VPS oddał 200 i 76 kB
|
|
||||||
# zaczynające się od 0x47 (sync MPEG-TS), test 2026-08-20. Dzięki temu
|
|
||||||
# przez VPS idzie sam manifest (~16 kB) przez /proxy/hls, a segmenty
|
|
||||||
# telefon ciągnie prosto z xspcdn. Bez tej flagi całe wideo szło proxy.
|
|
||||||
"mobile_direct_ok": True,
|
"mobile_direct_ok": True,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
|
||||||
52
app/extractors/tubes/porndish.py
Normal file
52
app/extractors/tubes/porndish.py
Normal file
|
|
@ -0,0 +1,52 @@
|
||||||
|
"""porndish.com: strona sceny z obejsciem zepsutego cache serwisu, potem embed-iframe.
|
||||||
|
|
||||||
|
Zgloszenie 88085cda (2026-10-12, "nic z porndish nie dziala"): cache strony porndish
|
||||||
|
(`x-cache-handler: autoptimize-pro-cache`, `cf-cache-status: HIT`) oddaje dla kazdego
|
||||||
|
`/porn/<slug>/` CUDZY wpis. Nowsze adresy dostawaly "[DadCrush] Carly Kiss", starsze
|
||||||
|
"[MyLifeInMiami] ...", wiec kazda z ~40 tys. scen porndish grala jeden z dwoch tych samych
|
||||||
|
filmow. Telemetria liczyla to jako sukces (wideo ruszalo, tylko nie to).
|
||||||
|
|
||||||
|
Dowolny parametr w zapytaniu omija cache (`BYPASS`), wiec dokladamy `?nc=<godzina>`
|
||||||
|
(jeden wariant na godzine, zeby nie mnozyc wpisow w ich cache). Do tego kontrola: kanoniczny
|
||||||
|
adres strony musi miec ten sam slug co zadany, inaczej nie oddajemy niczego (502), bo
|
||||||
|
cudzy film jest gorszy niz brak filmu.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import re
|
||||||
|
import time
|
||||||
|
from urllib.parse import urlsplit
|
||||||
|
|
||||||
|
from app.extractors._fetch import fetch_tube_html
|
||||||
|
from app.extractors._models import StreamSource, TubePageError
|
||||||
|
from app.extractors.tubes import _embed_iframe
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
_CANONICAL_RE = re.compile(
|
||||||
|
r"""<link[^>]+rel=["']canonical["'][^>]+href=["']([^"']+)["']"""
|
||||||
|
r"""|<meta[^>]+property=["']og:url["'][^>]+content=["']([^"']+)["']""",
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _slug(url: str) -> str:
|
||||||
|
return urlsplit(url).path.rstrip("/").rsplit("/", 1)[-1].lower()
|
||||||
|
|
||||||
|
|
||||||
|
def extract(page_url: str, *, timeout: float = 60.0) -> list[StreamSource] | None:
|
||||||
|
sep = "&" if "?" in page_url else "?"
|
||||||
|
busted = f"{page_url}{sep}nc={int(time.time() // 3600)}"
|
||||||
|
html = fetch_tube_html(busted, timeout=timeout)
|
||||||
|
|
||||||
|
want = _slug(page_url)
|
||||||
|
m = _CANONICAL_RE.search(html)
|
||||||
|
got = _slug(m.group(1) or m.group(2)) if m else ""
|
||||||
|
if want and got and got != want:
|
||||||
|
log.warning("porndish: %s oddal cudzy wpis (%s)", page_url, got)
|
||||||
|
raise TubePageError(502, page_url)
|
||||||
|
|
||||||
|
# Adres z `nc` takze dla zapasu "cala strona w WebView": bez niego telefon tez dostalby
|
||||||
|
# cudzy wpis z cache.
|
||||||
|
return _embed_iframe.extract(busted, timeout=timeout, page_html=html)
|
||||||
|
|
@ -12,11 +12,14 @@ Wyjściem jest zakładka „Downloads", która niesie:
|
||||||
`postId` siedzi w klasie `<body>` (`postid-69951`), więc nie trzeba parsować zakładki —
|
`postId` siedzi w klasie `<body>` (`postid-69951`), więc nie trzeba parsować zakładki —
|
||||||
składamy URL sami dla obu jakości.
|
składamy URL sami dla obu jakości.
|
||||||
|
|
||||||
**Oddajemy `download.php`, a NIE docelowego pliku.** Endpoint przekierowuje na
|
**Oddajemy docelowy plik, nie `download.php` (zmiana 2026-10-15).** Endpoint przekierowuje
|
||||||
`download.xanimeporn.com/...?s=<sig>&e=<epoch>`, czyli URL z tokenem i wygasaniem.
|
na `download.xanimeporn.com/videos/<q>/...mp4?s=<sig>&e=<epoch>` z `Content-Disposition:
|
||||||
Zwracając `download.php` zostawiamy generowanie tokenu na moment odtwarzania: telefon
|
attachment`. Wczesniej oddawalismy `download.php`, zeby telefon sam bral swiezy podpis, ale
|
||||||
idzie za przekierowaniem i dostaje świeży podpis, więc nic nie wietrzeje. To ta sama
|
apka po `.php` w adresie uznawala link za strone playera i otwierala go w WebView, ktory
|
||||||
lekcja co przy watchporn, gdzie zapisany statyczny URL po czasie padał.
|
POBIERAL plik do Pobranych zamiast grac (zgloszenie 54b17aed; od dodania serwisu ani jedno
|
||||||
|
odtworzenie). Podpis zyje ok. 2 h i nie jest zwiazany z IP (adres z VPS-a gra z domowego),
|
||||||
|
a resolve dzieje sie przy kliknieciu, wiec rozwiazujemy przekierowanie tutaj. Gdy sie nie
|
||||||
|
uda, zostaje `download.php` jako zapas.
|
||||||
|
|
||||||
Zweryfikowane z VPS-a: 1080p 222 MB i 720p 84 MB, oba 206 na Range z `ftypisom`/`ftypmp42`
|
Zweryfikowane z VPS-a: 1080p 222 MB i 720p 84 MB, oba 206 na Range z `ftypisom`/`ftypmp42`
|
||||||
w pierwszym KB (faststart). Wymaga Referera i UA, oba dokłada `playback.py`. Bez
|
w pierwszym KB (faststart). Wymaga Referera i UA, oba dokłada `playback.py`. Bez
|
||||||
|
|
@ -27,6 +30,8 @@ from __future__ import annotations
|
||||||
import logging
|
import logging
|
||||||
import re
|
import re
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
|
||||||
from app.extractors._fetch import fetch_tube_html
|
from app.extractors._fetch import fetch_tube_html
|
||||||
from app.extractors._models import StreamSource
|
from app.extractors._models import StreamSource
|
||||||
|
|
||||||
|
|
@ -36,6 +41,7 @@ _BASE = "https://xanimeporn.com"
|
||||||
_POST_ID_RE = re.compile(r"\bpostid-(\d+)\b")
|
_POST_ID_RE = re.compile(r"\bpostid-(\d+)\b")
|
||||||
#: Serwis publikuje dokładnie te dwie jakości; kolejność = preferencja.
|
#: Serwis publikuje dokładnie te dwie jakości; kolejność = preferencja.
|
||||||
_QUALITIES = ("1080p", "720p")
|
_QUALITIES = ("1080p", "720p")
|
||||||
|
_UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36"
|
||||||
|
|
||||||
|
|
||||||
def extract(page_url: str, *, timeout: float = 60.0) -> list[StreamSource] | None:
|
def extract(page_url: str, *, timeout: float = 60.0) -> list[StreamSource] | None:
|
||||||
|
|
@ -46,15 +52,29 @@ def extract(page_url: str, *, timeout: float = 60.0) -> list[StreamSource] | Non
|
||||||
return None
|
return None
|
||||||
post_id = m.group(1)
|
post_id = m.group(1)
|
||||||
|
|
||||||
return [
|
out: list[StreamSource] = []
|
||||||
|
for q in _QUALITIES:
|
||||||
|
link = f"https://videos.xanimeporn.com/download.php?id={post_id}&quality={q}"
|
||||||
|
try:
|
||||||
|
res = httpx.get(
|
||||||
|
link,
|
||||||
|
headers={"User-Agent": _UA, "Referer": _BASE + "/"},
|
||||||
|
follow_redirects=False,
|
||||||
|
timeout=15.0,
|
||||||
|
)
|
||||||
|
location = res.headers.get("location") or ""
|
||||||
|
if res.status_code in (301, 302, 303, 307, 308) and location.startswith("http"):
|
||||||
|
link = location
|
||||||
|
except Exception as e:
|
||||||
|
log.info("xanimeporn: przekierowanie %s %s nie wyszlo: %s", post_id, q, e)
|
||||||
|
out.append(
|
||||||
StreamSource(
|
StreamSource(
|
||||||
link=f"https://videos.xanimeporn.com/download.php?id={post_id}&quality={q}",
|
link=link,
|
||||||
type="mp4",
|
type="mp4",
|
||||||
quality=q,
|
quality=q,
|
||||||
referer=_BASE + "/",
|
referer=_BASE + "/",
|
||||||
# Zweryfikowane 206 + faststart z VPS-a; telefon sam podąża za 302 i
|
# Zweryfikowane 206 + faststart, podpis nie jest zwiazany z IP.
|
||||||
# dostaje własny, świeży token.
|
|
||||||
raw={"mobile_direct_ok": True},
|
raw={"mobile_direct_ok": True},
|
||||||
)
|
)
|
||||||
for q in _QUALITIES
|
)
|
||||||
]
|
return out
|
||||||
|
|
|
||||||
|
|
@ -135,9 +135,9 @@ def version() -> dict[str, str | int | None]:
|
||||||
# expo-video przy zamykaniu z ostatnich (Sentry 147164135, zgłoszenie 2026-09-15).
|
# expo-video przy zamykaniu z ostatnich (Sentry 147164135, zgłoszenie 2026-09-15).
|
||||||
# Podbijaj RAZEM z `version` przy każdym wydaniu APK.
|
# Podbijaj RAZEM z `version` przy każdym wydaniu APK.
|
||||||
return {
|
return {
|
||||||
"version": "0.4.2",
|
"version": "0.4.4",
|
||||||
"runtime_version": "1.4",
|
"runtime_version": "1.4",
|
||||||
"apk_build": 16,
|
"apk_build": 18,
|
||||||
"apk_url": apk_url,
|
"apk_url": apk_url,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,6 @@
|
||||||
from app.models.base import Base
|
from app.models.base import Base
|
||||||
from app.models.blacklist import BlacklistedPerformer, BlacklistedStudio, BlacklistedTag
|
from app.models.blacklist import BlacklistedPerformer, BlacklistedStudio, BlacklistedTag
|
||||||
|
from app.models.browse_run import BrowseRun
|
||||||
from app.models.bug_report import BugReport
|
from app.models.bug_report import BugReport
|
||||||
from app.models.clip import Clip, ClipTag
|
from app.models.clip import Clip, ClipTag
|
||||||
from app.models.clip_feed import ClipAffinity, ClipBlacklist, ClipFavorite, ClipFeedEvent, ClipSeen
|
from app.models.clip_feed import ClipAffinity, ClipBlacklist, ClipFavorite, ClipFeedEvent, ClipSeen
|
||||||
|
|
@ -44,6 +45,7 @@ from app.models.tag_vertical_count import TagVerticalCount
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"Base",
|
"Base",
|
||||||
|
"BrowseRun",
|
||||||
"BlacklistedPerformer",
|
"BlacklistedPerformer",
|
||||||
"BlacklistedStudio",
|
"BlacklistedStudio",
|
||||||
"BlacklistedTag",
|
"BlacklistedTag",
|
||||||
|
|
|
||||||
34
app/models/browse_run.py
Normal file
34
app/models/browse_run.py
Normal file
|
|
@ -0,0 +1,34 @@
|
||||||
|
"""Przebieg browse-latest per scraper (sitetag), zeby watchdog odroznial trzy rozne cisze:
|
||||||
|
listing pusty (fetch albo parser listingu padl), listing pelny ale wszystko znane (zly
|
||||||
|
endpoint, rotacja karuzeli) i zwykly brak nowych publikacji. ingest_runs tego nie daje,
|
||||||
|
bo wszystkie tuby dziela jeden Source "tube-scraper".
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import uuid
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
from sqlalchemy import Boolean, DateTime, Float, Index, Integer, String, Text, func
|
||||||
|
from sqlalchemy.dialects.postgresql import UUID
|
||||||
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
|
||||||
|
from app.models.base import Base
|
||||||
|
|
||||||
|
|
||||||
|
class BrowseRun(Base):
|
||||||
|
__tablename__ = "browse_runs"
|
||||||
|
__table_args__ = (Index("ix_browse_runs_sitetag_started", "sitetag", "started_at"),)
|
||||||
|
|
||||||
|
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||||
|
sitetag: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||||
|
started_at: Mapped[datetime] = mapped_column(
|
||||||
|
DateTime(timezone=True), server_default=func.now(), nullable=False
|
||||||
|
)
|
||||||
|
elapsed_s: Mapped[float | None] = mapped_column(Float, nullable=True)
|
||||||
|
seen: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||||
|
new: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||||
|
updated: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||||
|
skipped: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||||
|
errors: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||||
|
failed: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
||||||
|
note: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
|
|
@ -32,8 +32,13 @@ def find_movie_candidates(
|
||||||
net dla SEO tytułów (np. mangoporn często ma "Watch Cece adult..." vs paradisehill
|
net dla SEO tytułów (np. mangoporn często ma "Watch Cece adult..." vs paradisehill
|
||||||
"Cece", trigram złapie, ale gdyby nie, studio+year wystarczają).
|
"Cece", trigram złapie, ale gdyby nie, studio+year wystarczają).
|
||||||
"""
|
"""
|
||||||
from sqlalchemy import literal
|
from sqlalchemy import func, literal
|
||||||
|
|
||||||
|
# Kolejnosc po podobienstwie tytulu: bez niej LIMIT bral dowolne 50 z sumy trigramu i
|
||||||
|
# bloku studio+rok, a duze studio ma kilkaset filmow w trzech latach. Wtedy wierny
|
||||||
|
# duplikat (ten sam tytul, rok, studio i obsada) wypadal z puli i powstawal nowy film
|
||||||
|
# (2026-10-16: 13 z 228 nowych filmow z pandamovies, po scaleniu blizniaczych studiow
|
||||||
|
# bloki urosly).
|
||||||
q = session.execute(
|
q = session.execute(
|
||||||
select(Movie)
|
select(Movie)
|
||||||
.where(
|
.where(
|
||||||
|
|
@ -45,6 +50,7 @@ def find_movie_candidates(
|
||||||
_studio_year_block(studio_id, release_year),
|
_studio_year_block(studio_id, release_year),
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
.order_by(func.similarity(Movie.title_normalized, literal(title_normalized)).desc())
|
||||||
.limit(limit)
|
.limit(limit)
|
||||||
).scalars()
|
).scalars()
|
||||||
return list(q)
|
return list(q)
|
||||||
|
|
|
||||||
|
|
@ -340,7 +340,14 @@ def _sync_playback_sources(
|
||||||
session: Session, *, movie_id: uuid.UUID, norm: NormalizedMovie
|
session: Session, *, movie_id: uuid.UUID, norm: NormalizedMovie
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Upsert po (origin, page_url), unique constraint w tabeli."""
|
"""Upsert po (origin, page_url), unique constraint w tabeli."""
|
||||||
|
from app.extractors.hosters import seekplayer_engine
|
||||||
|
|
||||||
for pb in norm.playback_sources:
|
for pb in norm.playback_sources:
|
||||||
|
# Rodzina seekplayer (easyvidplayer, embedseek, upns, player4me) nie serwuje wideo
|
||||||
|
# od 08.2026 i zostala wygaszona hurtem 27.08, ale ingest dopisywal nowe takie linki
|
||||||
|
# (~1700 do 10.2026, zgloszenie e9fc12e8): czarny ekran albo minuta czekania.
|
||||||
|
if seekplayer_engine.matches(pb.embed_url or pb.page_url or ""):
|
||||||
|
continue
|
||||||
existing = session.execute(
|
existing = session.execute(
|
||||||
select(MoviePlaybackSource).where(
|
select(MoviePlaybackSource).where(
|
||||||
MoviePlaybackSource.origin == pb.origin,
|
MoviePlaybackSource.origin == pb.origin,
|
||||||
|
|
|
||||||
|
|
@ -16,6 +16,7 @@ Threshold (lower niż scenes, title dla movies ma niższy false-positive rate):
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import re
|
||||||
import uuid
|
import uuid
|
||||||
from collections.abc import Iterable
|
from collections.abc import Iterable
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
|
|
@ -24,6 +25,7 @@ from rapidfuzz import fuzz
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from app.models.movie import Movie, MoviePerformer
|
from app.models.movie import Movie, MoviePerformer
|
||||||
|
from app.models.studio import Studio
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
|
|
@ -58,6 +60,31 @@ AUTO_THRESHOLD = 0.85
|
||||||
REVIEW_THRESHOLD = 0.65
|
REVIEW_THRESHOLD = 0.65
|
||||||
|
|
||||||
|
|
||||||
|
_ROMAN_RE = re.compile(r"^(x{0,3})(ix|iv|v?i{0,3})$")
|
||||||
|
_ROMAN_VALUES = {"i": 1, "v": 5, "x": 10}
|
||||||
|
|
||||||
|
|
||||||
|
def _roman_value(token: str) -> int | None:
|
||||||
|
"""Wartosc cyfry rzymskiej II..XXXIX; pojedyncze litery (i, v, x) to zwykle slowa,
|
||||||
|
a "xx"/"xxx" w tych tytulach to slowo ("Star Wars XXX"), nie liczba."""
|
||||||
|
if len(token) < 2 or not _ROMAN_RE.match(token) or set(token) == {"x"}:
|
||||||
|
return None
|
||||||
|
total, prev = 0, 0
|
||||||
|
for ch in reversed(token):
|
||||||
|
v = _ROMAN_VALUES[ch]
|
||||||
|
total = total - v if v < prev else total + v
|
||||||
|
prev = max(prev, v)
|
||||||
|
return total if 2 <= total <= 39 else None
|
||||||
|
|
||||||
|
|
||||||
|
def _roman_to_arabic(title_normalized: str) -> str:
|
||||||
|
out = []
|
||||||
|
for t in title_normalized.split():
|
||||||
|
v = _roman_value(t)
|
||||||
|
out.append(str(v) if v is not None else t)
|
||||||
|
return " ".join(out)
|
||||||
|
|
||||||
|
|
||||||
def _series_number(title_normalized: str) -> int | None:
|
def _series_number(title_normalized: str) -> int | None:
|
||||||
"""Ostatnia samodzielna liczba w tytule, z pominieciem lat (1900-2099).
|
"""Ostatnia samodzielna liczba w tytule, z pominieciem lat (1900-2099).
|
||||||
|
|
||||||
|
|
@ -221,6 +248,14 @@ def _slowa(title_normalized: str) -> list[str]:
|
||||||
return [t for t in out if t not in _SEO_WYPELNIACZE]
|
return [t for t in out if t not in _SEO_WYPELNIACZE]
|
||||||
|
|
||||||
|
|
||||||
|
def _same_studio_name(session: Session, a_id: uuid.UUID, b_id: uuid.UUID) -> bool:
|
||||||
|
a, b = session.get(Studio, a_id), session.get(Studio, b_id)
|
||||||
|
if a is None or b is None or not a.name_normalized or not b.name_normalized:
|
||||||
|
return False
|
||||||
|
ka, kb = a.name_normalized.replace(" ", ""), b.name_normalized.replace(" ", "")
|
||||||
|
return len(ka) >= 4 and ka == kb
|
||||||
|
|
||||||
|
|
||||||
def score_movie_candidate(
|
def score_movie_candidate(
|
||||||
session: Session,
|
session: Session,
|
||||||
*,
|
*,
|
||||||
|
|
@ -233,11 +268,12 @@ def score_movie_candidate(
|
||||||
"""Liczy score dopasowania incoming → candidate. Wszystko w [0,1]."""
|
"""Liczy score dopasowania incoming → candidate. Wszystko w [0,1]."""
|
||||||
breakdown = MovieScoreBreakdown(reasons={})
|
breakdown = MovieScoreBreakdown(reasons={})
|
||||||
|
|
||||||
# Title (always)
|
# Title (always). Cyfry rzymskie jako arabskie: "hardcore gangbangs viii" i "... 8" to
|
||||||
|
# ten sam numer serii (zgloszenie f8c9486e, xtapes vs inne zrodla).
|
||||||
series_reject = False
|
series_reject = False
|
||||||
if candidate.title_normalized and norm_title_normalized:
|
if candidate.title_normalized and norm_title_normalized:
|
||||||
breakdown.title, series_reject, powody = _title_signal(
|
breakdown.title, series_reject, powody = _title_signal(
|
||||||
candidate.title_normalized, norm_title_normalized
|
_roman_to_arabic(candidate.title_normalized), _roman_to_arabic(norm_title_normalized)
|
||||||
)
|
)
|
||||||
breakdown.reasons.update(powody)
|
breakdown.reasons.update(powody)
|
||||||
|
|
||||||
|
|
@ -253,9 +289,12 @@ def score_movie_candidate(
|
||||||
else:
|
else:
|
||||||
breakdown.year = 0.0
|
breakdown.year = 0.0
|
||||||
|
|
||||||
# Studio match, binary
|
# Studio match, binary. Rozne wiersze o tej samej nazwie bez spacji ("EvilAngel" z xtapes
|
||||||
|
# i "Evil Angel" z TPDB) to to samo studio; w bazie jest ok. 2 tys. takich par.
|
||||||
if candidate.studio_id and norm_studio_id:
|
if candidate.studio_id and norm_studio_id:
|
||||||
breakdown.studio_match = candidate.studio_id == norm_studio_id
|
breakdown.studio_match = candidate.studio_id == norm_studio_id or _same_studio_name(
|
||||||
|
session, candidate.studio_id, norm_studio_id
|
||||||
|
)
|
||||||
|
|
||||||
# Cast Jaccard, gdy obie strony mają performerów
|
# Cast Jaccard, gdy obie strony mają performerów
|
||||||
norm_perf_set = {str(pid) for pid in norm_performer_ids if pid is not None}
|
norm_perf_set = {str(pid) for pid in norm_performer_ids if pid is not None}
|
||||||
|
|
|
||||||
|
|
@ -4,7 +4,7 @@ from __future__ import annotations
|
||||||
import uuid
|
import uuid
|
||||||
from datetime import date, timedelta
|
from datetime import date, timedelta
|
||||||
|
|
||||||
from sqlalchemy import and_, or_, select, text
|
from sqlalchemy import and_, func, select, text
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from app.config import get_settings
|
from app.config import get_settings
|
||||||
|
|
@ -165,12 +165,29 @@ def find_blocking_candidates(
|
||||||
- jeśli mamy studio + date → studio_id == X AND date BETWEEN ±window_days
|
- jeśli mamy studio + date → studio_id == X AND date BETWEEN ±window_days
|
||||||
- jeśli mamy tylko date → date BETWEEN ±window_days
|
- jeśli mamy tylko date → date BETWEEN ±window_days
|
||||||
- jeśli mamy tylko studio → studio_id == X
|
- jeśli mamy tylko studio → studio_id == X
|
||||||
- dodatkowo, jeśli `title_normalized` podany, OR-uj exact title match
|
- dodatkowo, jeśli `title_normalized` podany, sceny o identycznym tytule
|
||||||
(przydaje się gdy date/studio brakuje)
|
|
||||||
|
Sceny o identycznym tytule ida osobnym zapytaniem i zawsze trafiaja do puli. Do
|
||||||
|
2026-10-16 byly OR-owane z blokiem pod jednym LIMIT bez kolejnosci, a samo okno dat
|
||||||
|
to ok. 45 tys. scen, wiec dokladny duplikat potrafil wypasc (ten sam blad co przy
|
||||||
|
filmach, find_movie_candidates). Blok studio+data idzie od najblizszej daty (12 ms
|
||||||
|
dla najwiekszego studia); kolejnosc po podobienstwie tytulu kosztowala 0,4-3 s na
|
||||||
|
scene, za drogo na ingest.
|
||||||
"""
|
"""
|
||||||
if window_days is None:
|
if window_days is None:
|
||||||
window_days = get_settings().date_window_days
|
window_days = get_settings().date_window_days
|
||||||
|
|
||||||
|
out: list[Scene] = []
|
||||||
|
if title_normalized:
|
||||||
|
out = list(
|
||||||
|
session.execute(
|
||||||
|
select(Scene)
|
||||||
|
.where(Scene.title_normalized == title_normalized)
|
||||||
|
.order_by(Scene.created_at.desc())
|
||||||
|
.limit(limit)
|
||||||
|
).scalars().all()
|
||||||
|
)
|
||||||
|
|
||||||
conds = []
|
conds = []
|
||||||
if studio_id is not None and release_date is not None:
|
if studio_id is not None and release_date is not None:
|
||||||
conds.append(
|
conds.append(
|
||||||
|
|
@ -191,11 +208,15 @@ def find_blocking_candidates(
|
||||||
elif studio_id is not None:
|
elif studio_id is not None:
|
||||||
conds.append(Scene.studio_id == studio_id)
|
conds.append(Scene.studio_id == studio_id)
|
||||||
|
|
||||||
if title_normalized:
|
|
||||||
conds.append(Scene.title_normalized == title_normalized)
|
|
||||||
|
|
||||||
if not conds:
|
if not conds:
|
||||||
return []
|
return out
|
||||||
|
|
||||||
stmt = select(Scene).where(or_(*conds)).limit(limit)
|
stmt = select(Scene).where(*conds)
|
||||||
return list(session.execute(stmt).scalars().all())
|
if studio_id is not None and release_date is not None:
|
||||||
|
stmt = stmt.order_by(func.abs(Scene.release_date - release_date))
|
||||||
|
seen = {s.id for s in out}
|
||||||
|
for s in session.execute(stmt.limit(limit)).scalars().all():
|
||||||
|
if s.id not in seen:
|
||||||
|
seen.add(s.id)
|
||||||
|
out.append(s)
|
||||||
|
return out
|
||||||
|
|
|
||||||
|
|
@ -11,7 +11,7 @@ from __future__ import annotations
|
||||||
import logging
|
import logging
|
||||||
import uuid
|
import uuid
|
||||||
|
|
||||||
from sqlalchemy import select
|
from sqlalchemy import func, select
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from app.models.studio import Studio, StudioAlias, StudioExternalRef
|
from app.models.studio import Studio, StudioAlias, StudioExternalRef
|
||||||
|
|
@ -21,6 +21,24 @@ from app.normalize.text import slugify
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def _compact_name_twin(session: Session, name_normalized: str) -> Studio | None:
|
||||||
|
"""Jedno studio o tej samej nazwie bez spacji ("evilangel" = "evil angel").
|
||||||
|
|
||||||
|
xtapes podaje studia sklejone ("[EvilAngel]", "[TeamSkeet]"), a reszta zrodel ze
|
||||||
|
spacja; osobne studio psulo dopasowanie filmow (studio "rozne" obnizalo ocene, np.
|
||||||
|
"Hardcore Gangbangs 8" i "... VIII", zgloszenie f8c9486e). Tylko przy jednym trafieniu.
|
||||||
|
"""
|
||||||
|
compact = (name_normalized or "").replace(" ", "")
|
||||||
|
if len(compact) < 4:
|
||||||
|
return None
|
||||||
|
rows = session.execute(
|
||||||
|
select(Studio)
|
||||||
|
.where(func.replace(Studio.name_normalized, " ", "") == compact)
|
||||||
|
.limit(2)
|
||||||
|
).scalars().all()
|
||||||
|
return rows[0] if len(rows) == 1 else None
|
||||||
|
|
||||||
|
|
||||||
def resolve_studio(
|
def resolve_studio(
|
||||||
session: Session,
|
session: Session,
|
||||||
*,
|
*,
|
||||||
|
|
@ -43,6 +61,8 @@ def resolve_studio(
|
||||||
studio = session.execute(
|
studio = session.execute(
|
||||||
select(Studio).where(Studio.name_normalized == norm.name_normalized)
|
select(Studio).where(Studio.name_normalized == norm.name_normalized)
|
||||||
).scalar_one_or_none()
|
).scalar_one_or_none()
|
||||||
|
if studio is None:
|
||||||
|
studio = _compact_name_twin(session, norm.name_normalized)
|
||||||
if studio is None:
|
if studio is None:
|
||||||
studio = Studio(
|
studio = Studio(
|
||||||
name=norm.name,
|
name=norm.name,
|
||||||
|
|
|
||||||
|
|
@ -21,12 +21,37 @@ import time
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
|
|
||||||
from app.connectors.direct_scrapers import ALL_BROWSE_SCRAPERS, SCRAPER_SOURCE_NAME
|
from app.connectors.direct_scrapers import ALL_BROWSE_SCRAPERS, SCRAPER_SOURCE_NAME
|
||||||
|
from app.db import session_scope
|
||||||
|
from app.models.browse_run import BrowseRun
|
||||||
from app.models.source import SourceKind
|
from app.models.source import SourceKind
|
||||||
from app.scheduler.performer_driven import _ingest_iter_into_run
|
from app.scheduler.performer_driven import _ingest_iter_into_run
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def _record_run(sitetag: str, counters: dict | None, elapsed: float, failed: bool, note: str | None) -> None:
|
||||||
|
"""Jeden wiersz browse_runs per scraper i przebieg; czyta go watchdog listingu.
|
||||||
|
Blad zapisu nie moze zatrzymac browse (log i dalej)."""
|
||||||
|
c = counters or {}
|
||||||
|
try:
|
||||||
|
with session_scope() as s:
|
||||||
|
s.add(
|
||||||
|
BrowseRun(
|
||||||
|
sitetag=sitetag,
|
||||||
|
elapsed_s=round(elapsed, 1),
|
||||||
|
seen=int(c.get("seen", 0)),
|
||||||
|
new=int(c.get("new", 0)),
|
||||||
|
updated=int(c.get("updated", 0)),
|
||||||
|
skipped=int(c.get("skipped", 0)),
|
||||||
|
errors=int(c.get("errors", 0)),
|
||||||
|
failed=failed,
|
||||||
|
note=(note or None) and note[:500],
|
||||||
|
)
|
||||||
|
)
|
||||||
|
except Exception: # pragma: no cover - tabela moze nie istniec przed migracja 0056
|
||||||
|
log.exception("browse-latest: zapis browse_runs dla %s nie powiodl sie", sitetag)
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class BrowseCounters:
|
class BrowseCounters:
|
||||||
scrapers_run: int = 0
|
scrapers_run: int = 0
|
||||||
|
|
@ -62,9 +87,11 @@ def run_browse_latest(*, max_pages: int = 5) -> BrowseCounters:
|
||||||
"browse-latest: %s done in %.1fs counters=%s",
|
"browse-latest: %s done in %.1fs counters=%s",
|
||||||
scraper.sitetag, elapsed, c,
|
scraper.sitetag, elapsed, c,
|
||||||
)
|
)
|
||||||
|
_record_run(scraper.sitetag, c, elapsed, failed=False, note=None)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
counters.total_errors += 1
|
counters.total_errors += 1
|
||||||
log.exception("browse-latest: %s failed: %s", scraper.sitetag, e)
|
log.exception("browse-latest: %s failed: %s", scraper.sitetag, e)
|
||||||
|
_record_run(scraper.sitetag, None, time.time() - t0, failed=True, note=f"{type(e).__name__}: {e}")
|
||||||
|
|
||||||
log.info("browse-latest done: %s", counters)
|
log.info("browse-latest done: %s", counters)
|
||||||
return counters
|
return counters
|
||||||
|
|
|
||||||
|
|
@ -74,44 +74,96 @@ def plan_duration_backfill(limit: int | None = None) -> int:
|
||||||
|
|
||||||
|
|
||||||
def _fetch_one(row: tuple) -> tuple:
|
def _fetch_one(row: tuple) -> tuple:
|
||||||
"""(scene_id, duration|None). Sieć i parsowanie, bez dotykania bazy."""
|
"""(scene_id, duration|None, gone). Sieć i parsowanie, bez dotykania bazy.
|
||||||
|
|
||||||
|
`gone` = strona odpowiedziała 404/410, więc kolejne próby nic nie dadzą i wiersz
|
||||||
|
można zamknąć od razu zamiast odwiedzać martwy adres trzy razy. Błąd sieci, 5xx
|
||||||
|
albo 403 (limit, geo) zostają zwykłą nieudaną próbą.
|
||||||
|
"""
|
||||||
scene_id, page_url, origin = row
|
scene_id, page_url, origin = row
|
||||||
from app.extractors._fetch import browser_get
|
from app.extractors._fetch import browser_get
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
if origin == "tube:perverzijacom":
|
||||||
|
# Cloudflare blokuje IPv4 VPS-a (403, od 2026-10-14/15); bez proxy kazda proba
|
||||||
|
# palila sie i wiersz zamykal po trzech bez dlugosci.
|
||||||
|
from app.extractors.tubes.perverzija import fetch_perverzija
|
||||||
|
|
||||||
|
r = fetch_perverzija(page_url, timeout=_FETCH_TIMEOUT)
|
||||||
|
else:
|
||||||
r = browser_get(page_url, timeout=_FETCH_TIMEOUT, follow_redirects=True)
|
r = browser_get(page_url, timeout=_FETCH_TIMEOUT, follow_redirects=True)
|
||||||
|
if r.status_code in (404, 410):
|
||||||
|
return (scene_id, None, True)
|
||||||
if r.status_code >= 400:
|
if r.status_code >= 400:
|
||||||
return (scene_id, None)
|
return (scene_id, None, False)
|
||||||
return (scene_id, extract_duration_for_origin(origin, r.text, page_url))
|
return (scene_id, extract_duration_for_origin(origin, r.text, page_url), False)
|
||||||
except Exception as e: # pojedyncza strona nie może wywrócić paczki
|
except Exception as e: # pojedyncza strona nie może wywrócić paczki
|
||||||
log.debug("duration-backfill: %s padlo (%s)", page_url, e)
|
log.debug("duration-backfill: %s padlo (%s)", page_url, e)
|
||||||
return (scene_id, None)
|
return (scene_id, None, False)
|
||||||
|
|
||||||
|
|
||||||
|
def _take_batch(session, *, batch: int) -> list[tuple]:
|
||||||
|
"""Paczka po rowno z kazdego originu, ktory ma jeszcze cos w kolejce.
|
||||||
|
|
||||||
|
Kolejnosc dopisania (`created_at`) jako jedyny klucz oznaczalaby, ze origin dopisany
|
||||||
|
pozniej (perverzija, mypornerleak, 2026-10-14) czeka, az 900 tys. wierszy xhamster,
|
||||||
|
xvideos i xnxx sie wyczerpie. Zamiast tego limit dzielimy miedzy originy z otwartymi
|
||||||
|
wierszami (najmniejsze kolejki dostaja swoj udzial w calosci, reszta dzieli resztę),
|
||||||
|
a wewnatrz originu zostaje stary porzadek: najpierw nieprobowane, potem najstarsze.
|
||||||
|
"""
|
||||||
|
remaining = session.execute(
|
||||||
|
text(
|
||||||
|
"""
|
||||||
|
SELECT origin, count(*) AS n
|
||||||
|
FROM duration_backfill_queue
|
||||||
|
WHERE done_at IS NULL AND attempts < :max_attempts
|
||||||
|
GROUP BY origin
|
||||||
|
"""
|
||||||
|
),
|
||||||
|
{"max_attempts": _MAX_ATTEMPTS},
|
||||||
|
).fetchall()
|
||||||
|
pending = sorted(((int(n), o) for o, n in remaining if n), key=lambda t: t[0])
|
||||||
|
rows: list[tuple] = []
|
||||||
|
left = batch
|
||||||
|
for i, (n, origin) in enumerate(pending):
|
||||||
|
quota = min(n, max(1, left // (len(pending) - i)))
|
||||||
|
if quota <= 0:
|
||||||
|
break
|
||||||
|
rows += [
|
||||||
|
tuple(r)
|
||||||
|
for r in session.execute(
|
||||||
|
text(
|
||||||
|
"""
|
||||||
|
SELECT scene_id, page_url, origin
|
||||||
|
FROM duration_backfill_queue
|
||||||
|
WHERE origin = :origin AND done_at IS NULL AND attempts < :max_attempts
|
||||||
|
ORDER BY attempts, created_at
|
||||||
|
LIMIT :quota
|
||||||
|
"""
|
||||||
|
),
|
||||||
|
{"origin": origin, "max_attempts": _MAX_ATTEMPTS, "quota": quota},
|
||||||
|
).fetchall()
|
||||||
|
]
|
||||||
|
left -= quota
|
||||||
|
if left <= 0:
|
||||||
|
break
|
||||||
|
return rows
|
||||||
|
|
||||||
|
|
||||||
def run_duration_backfill(*, batch: int = _DEFAULT_BATCH) -> dict[str, int]:
|
def run_duration_backfill(*, batch: int = _DEFAULT_BATCH) -> dict[str, int]:
|
||||||
"""Bierze paczkę z kolejki, dolicza długość, zapisuje. Pusta kolejka = no-op."""
|
"""Bierze paczkę z kolejki, dolicza długość, zapisuje. Pusta kolejka = no-op."""
|
||||||
with session_scope() as s:
|
with session_scope() as s:
|
||||||
rows = s.execute(
|
rows = _take_batch(s, batch=batch)
|
||||||
text(
|
|
||||||
"""
|
|
||||||
SELECT scene_id, page_url, origin
|
|
||||||
FROM duration_backfill_queue
|
|
||||||
WHERE done_at IS NULL AND attempts < :max_attempts
|
|
||||||
ORDER BY attempts, created_at
|
|
||||||
LIMIT :batch
|
|
||||||
"""
|
|
||||||
),
|
|
||||||
{"max_attempts": _MAX_ATTEMPTS, "batch": batch},
|
|
||||||
).fetchall()
|
|
||||||
|
|
||||||
if not rows:
|
if not rows:
|
||||||
return {"taken": 0, "filled": 0, "missed": 0}
|
return {"taken": 0, "filled": 0, "missed": 0, "gone": 0}
|
||||||
|
|
||||||
with ThreadPoolExecutor(max_workers=_CONCURRENCY) as pool:
|
with ThreadPoolExecutor(max_workers=_CONCURRENCY) as pool:
|
||||||
results = list(pool.map(_fetch_one, [tuple(r) for r in rows]))
|
results = list(pool.map(_fetch_one, [tuple(r) for r in rows]))
|
||||||
|
|
||||||
filled = [(sid, dur) for sid, dur in results if dur]
|
filled = [(sid, dur) for sid, dur, _gone in results if dur]
|
||||||
missed = [sid for sid, dur in results if not dur]
|
gone = [sid for sid, dur, is_gone in results if not dur and is_gone]
|
||||||
|
missed = [sid for sid, dur, is_gone in results if not dur and not is_gone]
|
||||||
|
|
||||||
with session_scope() as s:
|
with session_scope() as s:
|
||||||
for scene_id, dur in filled:
|
for scene_id, dur in filled:
|
||||||
|
|
@ -130,6 +182,17 @@ def run_duration_backfill(*, batch: int = _DEFAULT_BATCH) -> dict[str, int]:
|
||||||
),
|
),
|
||||||
{"id": scene_id},
|
{"id": scene_id},
|
||||||
)
|
)
|
||||||
|
if gone:
|
||||||
|
s.execute(
|
||||||
|
text(
|
||||||
|
"""
|
||||||
|
UPDATE duration_backfill_queue
|
||||||
|
SET attempts = :max_attempts, done_at = now()
|
||||||
|
WHERE scene_id = ANY(:ids)
|
||||||
|
"""
|
||||||
|
),
|
||||||
|
{"ids": gone, "max_attempts": _MAX_ATTEMPTS},
|
||||||
|
)
|
||||||
if missed:
|
if missed:
|
||||||
# Wyczerpane próby zamykamy, żeby kolejka nie kręciła się w kółko na martwych.
|
# Wyczerpane próby zamykamy, żeby kolejka nie kręciła się w kółko na martwych.
|
||||||
s.execute(
|
s.execute(
|
||||||
|
|
@ -145,4 +208,9 @@ def run_duration_backfill(*, batch: int = _DEFAULT_BATCH) -> dict[str, int]:
|
||||||
{"ids": missed, "max_attempts": _MAX_ATTEMPTS},
|
{"ids": missed, "max_attempts": _MAX_ATTEMPTS},
|
||||||
)
|
)
|
||||||
|
|
||||||
return {"taken": len(rows), "filled": len(filled), "missed": len(missed)}
|
return {
|
||||||
|
"taken": len(rows),
|
||||||
|
"filled": len(filled),
|
||||||
|
"missed": len(missed),
|
||||||
|
"gone": len(gone),
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -61,12 +61,59 @@ def _bucket(age_h: float) -> tuple[str, str]:
|
||||||
|
|
||||||
# Serwisy, ktore publikuja rzadziej niz co dwa dni: prog 48 h robil z nich staly alarm
|
# Serwisy, ktore publikuja rzadziej niz co dwa dni: prog 48 h robil z nich staly alarm
|
||||||
# (sprawdzone 2026-10-13: xanimeporn 1 do 3 wpisow na tydzien z RSS, hentaimama wydaje
|
# (sprawdzone 2026-10-13: xanimeporn 1 do 3 wpisow na tydzien z RSS, hentaimama wydaje
|
||||||
# odcinki raz w tygodniu, obie listy w calosci znane w bazie). Prog w godzinach.
|
# odcinki raz w tygodniu, obie listy w calosci znane w bazie). Prog w godzinach, reczny
|
||||||
|
# podlog; do tego kazde zrodlo dostaje prog z wlasnej kadencji (patrz _cadence_threshold_h).
|
||||||
_BROWSE_MAX_AGE_OVERRIDE_H: dict[str, int] = {
|
_BROWSE_MAX_AGE_OVERRIDE_H: dict[str, int] = {
|
||||||
"xanimeporn": 168,
|
"xanimeporn": 168,
|
||||||
"hentaimama": 240,
|
"hentaimama": 240,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# Zrodla swiadomie zamrozone: alarm nic nie wniesie, dopoki ktos nie zmieni stanu rzeczy.
|
||||||
|
# porndoe (2026-10-14): kazdy nowy /watch/ z sitemapy przekierowuje na /videos z kazdego IP
|
||||||
|
# (proxy PL i UK, z ciastkami sesji i po zdjeciu bramki wieku), darmowy katalog stoi na
|
||||||
|
# 11.09; stare 64 tys. scen gra. Kontrola reczna co jakis czas, nie co 6 h w Sentry.
|
||||||
|
_FROZEN_BROWSE: dict[str, str] = {
|
||||||
|
"porndoecom": "2026-10-14 nowe sceny przekierowuja na /videos z kazdego IP",
|
||||||
|
}
|
||||||
|
|
||||||
|
# Prog z kadencji: mediana odstepu (w dobach) miedzy dniami z nowymi pozycjami w ostatnich
|
||||||
|
# 60 dniach, razy 3, nie mniej niz prog klasy i nie wiecej niz 14 dni. Zrodlo codzienne
|
||||||
|
# zostaje przy 48 h, tygodniowe dostaje ok. 14 dni zamiast alarmu co dobe (2026-10-14).
|
||||||
|
_ADAPTIVE_FACTOR = 3.0
|
||||||
|
_ADAPTIVE_CAP_H = 14 * 24
|
||||||
|
_ADAPTIVE_MIN_ACTIVE_DAYS = 5
|
||||||
|
_ADAPTIVE_LOOKBACK_DAYS = 60
|
||||||
|
|
||||||
|
|
||||||
|
def _cadence_threshold_h(session, *, origin: str, table: str, base_h: int) -> int:
|
||||||
|
import statistics
|
||||||
|
|
||||||
|
if table == "movie_playback_sources":
|
||||||
|
where = "(origin = :o OR origin LIKE :p)"
|
||||||
|
params: dict = {"o": origin, "p": f"{origin}:%", "lb": _ADAPTIVE_LOOKBACK_DAYS}
|
||||||
|
else:
|
||||||
|
where = "origin = :o"
|
||||||
|
params = {"o": origin, "lb": _ADAPTIVE_LOOKBACK_DAYS}
|
||||||
|
try:
|
||||||
|
rows = session.execute(
|
||||||
|
text(
|
||||||
|
f"SELECT DISTINCT created_at::date AS d FROM {table} WHERE {where} "
|
||||||
|
"AND created_at > now() - make_interval(days => :lb) ORDER BY d"
|
||||||
|
),
|
||||||
|
params,
|
||||||
|
).all()
|
||||||
|
except Exception: # pragma: no cover
|
||||||
|
log.exception("ingest-watchdog: kadencja %s nieznana, prog klasy", origin)
|
||||||
|
return base_h
|
||||||
|
days = [r[0] for r in rows]
|
||||||
|
if len(days) < _ADAPTIVE_MIN_ACTIVE_DAYS:
|
||||||
|
return base_h
|
||||||
|
gaps = [(b - a).days for a, b in zip(days, days[1:])]
|
||||||
|
if not gaps:
|
||||||
|
return base_h
|
||||||
|
median_gap_h = statistics.median(gaps) * 24.0
|
||||||
|
return int(max(base_h, min(_ADAPTIVE_CAP_H, _ADAPTIVE_FACTOR * median_gap_h)))
|
||||||
|
|
||||||
|
|
||||||
def run_ingest_freshness_watchdog(
|
def run_ingest_freshness_watchdog(
|
||||||
*,
|
*,
|
||||||
|
|
@ -98,6 +145,7 @@ def run_ingest_freshness_watchdog(
|
||||||
[
|
[
|
||||||
(f"tube:{t}", "playback_sources", _BROWSE_MAX_AGE_OVERRIDE_H.get(t, max_age_hours), "browse")
|
(f"tube:{t}", "playback_sources", _BROWSE_MAX_AGE_OVERRIDE_H.get(t, max_age_hours), "browse")
|
||||||
for t in sorted(browse_tags)
|
for t in sorted(browse_tags)
|
||||||
|
if t not in _FROZEN_BROWSE
|
||||||
]
|
]
|
||||||
+ [(f"tube:{t}", "playback_sources", search_max_age_hours, "search")
|
+ [(f"tube:{t}", "playback_sources", search_max_age_hours, "search")
|
||||||
for t in sorted(search_tags)]
|
for t in sorted(search_tags)]
|
||||||
|
|
@ -132,6 +180,10 @@ def run_ingest_freshness_watchdog(
|
||||||
if total < min_history or newest is None:
|
if total < min_history or newest is None:
|
||||||
continue
|
continue
|
||||||
age_h = (now - newest).total_seconds() / 3600.0
|
age_h = (now - newest).total_seconds() / 3600.0
|
||||||
|
if age_h >= threshold:
|
||||||
|
# Dopiero przy przekroczeniu progu klasy liczymy kadencje (jedno zapytanie
|
||||||
|
# per podejrzane zrodlo, nie per wszystkie).
|
||||||
|
threshold = _cadence_threshold_h(s, origin=origin, table=table, base_h=threshold)
|
||||||
if age_h >= threshold:
|
if age_h >= threshold:
|
||||||
label, level = _bucket(age_h)
|
label, level = _bucket(age_h)
|
||||||
stale.append(
|
stale.append(
|
||||||
|
|
|
||||||
|
|
@ -455,6 +455,15 @@ def _job_ingest_watchdog(
|
||||||
log.warning("[scheduler] ingest-watchdog: %d stale origin(s)", len(res["stale"]))
|
log.warning("[scheduler] ingest-watchdog: %d stale origin(s)", len(res["stale"]))
|
||||||
except Exception:
|
except Exception:
|
||||||
log.exception("[scheduler] ingest-watchdog failed")
|
log.exception("[scheduler] ingest-watchdog failed")
|
||||||
|
# Jakosc: listing (browse_runs), parser (performerzy, dlugosc, miniatury), odtwarzanie.
|
||||||
|
# Osobny try, zeby awaria jednego sprawdzenia nie zdjela drugiego.
|
||||||
|
try:
|
||||||
|
from app.scheduler.quality_watchdog import run_quality_watchdogs
|
||||||
|
|
||||||
|
q = run_quality_watchdogs()
|
||||||
|
log.info("[scheduler] quality-watchdog: %s", q["summary"])
|
||||||
|
except Exception:
|
||||||
|
log.exception("[scheduler] quality-watchdog failed")
|
||||||
|
|
||||||
|
|
||||||
def _job_hetzner_monitor() -> None:
|
def _job_hetzner_monitor() -> None:
|
||||||
|
|
|
||||||
304
app/scheduler/quality_watchdog.py
Normal file
304
app/scheduler/quality_watchdog.py
Normal file
|
|
@ -0,0 +1,304 @@
|
||||||
|
"""Watchdogi jakosci zrodel, uzupelnienie freshness z ingest_watchdog.py.
|
||||||
|
|
||||||
|
Freshness widzi tylko "brak nowych pozycji". Cztery klasy awarii, ktore przez to przechodzily
|
||||||
|
(wszystkie z 10.2026): watchporn zmienil motyw i parser dopinal 12 performerow na scene
|
||||||
|
z kart "related" (ingest "zdrowy", new>0); galaxyporn, latestpornvideo i porndish oddaja
|
||||||
|
sceny bez dlugosci (apka filtruje min 60 s, wiec sa niewidoczne, a licznik new rosnie);
|
||||||
|
porndoe oddawal 31 tych samych scen (listing pelny, new=0, nie wiadomo czy to serwis stoi,
|
||||||
|
czy czytamy zla strone); perverzija grala w 50% przy "zywych" zrodlach.
|
||||||
|
|
||||||
|
Trzy sprawdzenia, kazde z wlasnym fingerprintem w Sentry (jedno issue na zrodlo i rodzaj,
|
||||||
|
bez spamu co 6 h):
|
||||||
|
|
||||||
|
- **listing** (browse_runs): ostatnie przebiegi scrapera z seen=0 albo wyjatkiem = listing
|
||||||
|
pusty lub blad; dlugi ciag seen>0 i new=0 u zrodla publikujacego codziennie = czytamy
|
||||||
|
zla strone (rotacja karuzeli, endpoint nie sortowany po dacie).
|
||||||
|
- **parser** (playback_sources + scenes z 24 h wobec 14 dni wczesniej): skok lub zanik
|
||||||
|
performerow na scene, udzial scen bez dlugosci, bez miniatury. Lapie zmiane motywu
|
||||||
|
strony w pierwszej dobie, bez czekania na zgloszenie z apki.
|
||||||
|
- **playback** (playback_events 24 h wobec 7 dni): spadek udanych odtworzen per origin,
|
||||||
|
plus serwis z >= 300 zywymi zrodlami i zerem udanych odtworzen w 7 dni (cicha awaria).
|
||||||
|
|
||||||
|
`to_sentry=False` daje sam odczyt (reczne sprawdzenie bez zasmiecania issue).
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import statistics
|
||||||
|
from collections import defaultdict
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from sqlalchemy import text
|
||||||
|
|
||||||
|
from app.db import session_scope
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# listing
|
||||||
|
_LISTING_EMPTY_STREAK = 3 # tyle ostatnich przebiegow z seen=0/bledem z rzedu
|
||||||
|
_LISTING_ALLKNOWN_STREAK = 8 # tyle przebiegow (48 h przy co 6 h) z seen>0 i new=0
|
||||||
|
_LISTING_ALLKNOWN_MAX_GAP_DAYS = 2.0 # tylko zrodla publikujace co najmniej co 2 dni
|
||||||
|
# parser
|
||||||
|
_PARSER_MIN_RECENT = 10
|
||||||
|
_PARSER_MIN_BASE = 30
|
||||||
|
# playback
|
||||||
|
_PLAYBACK_MIN_ATTEMPTS = 30
|
||||||
|
_PLAYBACK_MIN_BASE_ATTEMPTS = 50
|
||||||
|
_SILENT_MIN_LIVE = 300
|
||||||
|
|
||||||
|
|
||||||
|
def _cadence_gap_days(session, origin: str, lookback_days: int = 60) -> float | None:
|
||||||
|
"""Mediana odstepu (w dobach) miedzy dniami, w ktorych zrodlo dalo nowe pozycje."""
|
||||||
|
rows = session.execute(
|
||||||
|
text(
|
||||||
|
"SELECT DISTINCT created_at::date AS d FROM playback_sources "
|
||||||
|
"WHERE origin = :o AND created_at > now() - make_interval(days => :lb) ORDER BY d"
|
||||||
|
),
|
||||||
|
{"o": origin, "lb": lookback_days},
|
||||||
|
).all()
|
||||||
|
days = [r[0] for r in rows]
|
||||||
|
if len(days) < 5:
|
||||||
|
return None
|
||||||
|
gaps = [(b - a).days for a, b in zip(days, days[1:])]
|
||||||
|
return float(statistics.median(gaps)) if gaps else None
|
||||||
|
|
||||||
|
|
||||||
|
def check_listing_health(session) -> list[dict[str, Any]]:
|
||||||
|
rows = session.execute(
|
||||||
|
text(
|
||||||
|
"SELECT sitetag, started_at, seen, new, skipped, failed, note FROM browse_runs "
|
||||||
|
"WHERE started_at > now() - interval '60 hours' ORDER BY sitetag, started_at DESC"
|
||||||
|
)
|
||||||
|
).all()
|
||||||
|
by_tag: dict[str, list] = defaultdict(list)
|
||||||
|
for r in rows:
|
||||||
|
by_tag[r.sitetag].append(r)
|
||||||
|
out: list[dict[str, Any]] = []
|
||||||
|
for sitetag, runs in by_tag.items():
|
||||||
|
last = runs[:_LISTING_EMPTY_STREAK]
|
||||||
|
if len(last) >= _LISTING_EMPTY_STREAK and all(r.failed or r.seen == 0 for r in last):
|
||||||
|
note = next((r.note for r in last if r.note), None)
|
||||||
|
out.append(
|
||||||
|
{
|
||||||
|
"sitetag": sitetag,
|
||||||
|
"problem": "empty",
|
||||||
|
"runs": len(last),
|
||||||
|
"note": note,
|
||||||
|
"level": "warning",
|
||||||
|
"message": (
|
||||||
|
f"listing-watchdog: {sitetag} listing pusty albo blad w {len(last)} "
|
||||||
|
f"przebiegach z rzedu" + (f" ({note[:120]})" if note else "")
|
||||||
|
),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
last = runs[:_LISTING_ALLKNOWN_STREAK]
|
||||||
|
if len(last) >= _LISTING_ALLKNOWN_STREAK and all(
|
||||||
|
(not r.failed) and r.seen > 0 and r.new == 0 for r in last
|
||||||
|
):
|
||||||
|
gap = _cadence_gap_days(session, f"tube:{sitetag}")
|
||||||
|
if gap is not None and gap <= _LISTING_ALLKNOWN_MAX_GAP_DAYS:
|
||||||
|
seen = sum(r.seen for r in last)
|
||||||
|
out.append(
|
||||||
|
{
|
||||||
|
"sitetag": sitetag,
|
||||||
|
"problem": "all_known",
|
||||||
|
"runs": len(last),
|
||||||
|
"seen": seen,
|
||||||
|
"gap_days": gap,
|
||||||
|
"level": "warning",
|
||||||
|
"message": (
|
||||||
|
f"listing-watchdog: {sitetag} {len(last)} przebiegi z rzedu, {seen} pozycji "
|
||||||
|
f"i 0 nowych, a zrodlo publikuje co {gap:.0f} d: zla strona listingu "
|
||||||
|
f"(rotacja, brak sortu po dacie) albo serwis stanal"
|
||||||
|
),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def check_parser_quality(session) -> list[dict[str, Any]]:
|
||||||
|
rows = session.execute(
|
||||||
|
text(
|
||||||
|
"""
|
||||||
|
WITH w AS (
|
||||||
|
SELECT ps.origin, ps.scene_id, ps.thumbnail_url,
|
||||||
|
(ps.created_at > now() - interval '24 hours') AS recent
|
||||||
|
FROM playback_sources ps
|
||||||
|
WHERE ps.origin LIKE 'tube:%' AND ps.created_at > now() - interval '15 days'
|
||||||
|
)
|
||||||
|
SELECT w.origin, w.recent, count(*) AS n,
|
||||||
|
avg((SELECT count(*) FROM scene_performers sp WHERE sp.scene_id = w.scene_id)) AS perf,
|
||||||
|
avg(CASE WHEN x.duration_sec IS NULL THEN 1.0 ELSE 0.0 END) AS dur_null,
|
||||||
|
avg(CASE WHEN w.thumbnail_url IS NULL THEN 1.0 ELSE 0.0 END) AS thumb_null
|
||||||
|
FROM w JOIN scenes x ON x.id = w.scene_id
|
||||||
|
GROUP BY 1, 2
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
).all()
|
||||||
|
stats: dict[str, dict[bool, Any]] = defaultdict(dict)
|
||||||
|
for r in rows:
|
||||||
|
stats[r.origin][bool(r.recent)] = r
|
||||||
|
out: list[dict[str, Any]] = []
|
||||||
|
for origin, parts in stats.items():
|
||||||
|
rec, base = parts.get(True), parts.get(False)
|
||||||
|
if rec is None or base is None or rec.n < _PARSER_MIN_RECENT or base.n < _PARSER_MIN_BASE:
|
||||||
|
continue
|
||||||
|
pr, pb = float(rec.perf or 0), float(base.perf or 0)
|
||||||
|
dr, db = float(rec.dur_null or 0), float(base.dur_null or 0)
|
||||||
|
tr, tb = float(rec.thumb_null or 0), float(base.thumb_null or 0)
|
||||||
|
probs: list[tuple[str, str]] = []
|
||||||
|
if pb >= 0 and pr >= 2.5 * max(pb, 0.4) and pr - pb >= 3:
|
||||||
|
probs.append(("perf_inflated", f"obsada rozdeta: {pr:.1f} performerow na scene (wczesniej {pb:.1f}), karty related?"))
|
||||||
|
if pb >= 0.8 and pr <= 0.25 * pb:
|
||||||
|
probs.append(("perf_lost", f"obsada zniknela: {pr:.2f} na scene (wczesniej {pb:.2f})"))
|
||||||
|
if dr >= 0.5 and db <= 0.25:
|
||||||
|
probs.append(("duration_lost", f"{dr*100:.0f}% scen bez dlugosci (wczesniej {db*100:.0f}%), apka je ukrywa"))
|
||||||
|
if tr >= 0.5 and tb <= 0.25:
|
||||||
|
probs.append(("thumb_lost", f"{tr*100:.0f}% zrodel bez miniatury (wczesniej {tb*100:.0f}%)"))
|
||||||
|
for metric, desc in probs:
|
||||||
|
out.append(
|
||||||
|
{
|
||||||
|
"origin": origin,
|
||||||
|
"metric": metric,
|
||||||
|
"recent_n": int(rec.n),
|
||||||
|
"base_n": int(base.n),
|
||||||
|
"level": "warning",
|
||||||
|
"message": f"parser-watchdog: {origin} {desc} (24 h: {int(rec.n)} scen, baza {int(base.n)})",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def check_playback_health(session) -> list[dict[str, Any]]:
|
||||||
|
rows = session.execute(
|
||||||
|
text(
|
||||||
|
"""
|
||||||
|
SELECT origin,
|
||||||
|
count(*) FILTER (WHERE created_at > now() - interval '24 hours') AS att24,
|
||||||
|
avg(CASE WHEN status = 'success' THEN 1.0 ELSE 0.0 END)
|
||||||
|
FILTER (WHERE created_at > now() - interval '24 hours') AS ok24,
|
||||||
|
count(*) FILTER (WHERE created_at <= now() - interval '24 hours') AS attb,
|
||||||
|
avg(CASE WHEN status = 'success' THEN 1.0 ELSE 0.0 END)
|
||||||
|
FILTER (WHERE created_at <= now() - interval '24 hours') AS okb
|
||||||
|
FROM playback_events
|
||||||
|
WHERE created_at > now() - interval '8 days'
|
||||||
|
GROUP BY 1
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
).all()
|
||||||
|
out: list[dict[str, Any]] = []
|
||||||
|
for r in rows:
|
||||||
|
att24 = int(r.att24 or 0)
|
||||||
|
if att24 < _PLAYBACK_MIN_ATTEMPTS or r.ok24 is None:
|
||||||
|
continue
|
||||||
|
ok24 = float(r.ok24)
|
||||||
|
okb = float(r.okb) if r.okb is not None else None
|
||||||
|
attb = int(r.attb or 0)
|
||||||
|
bucket = None
|
||||||
|
if ok24 < 0.3:
|
||||||
|
bucket, level = "lt30", "error"
|
||||||
|
elif ok24 < 0.6 and okb is not None and attb >= _PLAYBACK_MIN_BASE_ATTEMPTS and okb - ok24 >= 0.25:
|
||||||
|
bucket, level = "lt60", "warning"
|
||||||
|
if not bucket:
|
||||||
|
continue
|
||||||
|
base_txt = f", tydzien wczesniej {okb*100:.0f}%" if okb is not None else ""
|
||||||
|
out.append(
|
||||||
|
{
|
||||||
|
"origin": r.origin,
|
||||||
|
"bucket": bucket,
|
||||||
|
"ok24": round(ok24, 3),
|
||||||
|
"att24": att24,
|
||||||
|
"level": level,
|
||||||
|
"message": f"playback-watchdog: {r.origin} {ok24*100:.0f}% udanych z {att24} prob w 24 h{base_txt}",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def check_silent_origins(session) -> list[dict[str, Any]]:
|
||||||
|
"""Serwis z wieloma zywymi zrodlami i ZERO udanych odtworzen w 7 dni.
|
||||||
|
|
||||||
|
check_playback_health liczy tylko originy ze zdarzeniami (min. 30 w 24 h). hentaimama
|
||||||
|
i xanimeporn padly tak, ze apka nie dochodzila do odtwarzacza (503 z resolve, pobieranie
|
||||||
|
pliku w WebView), wiec przez miesiac nie mialy ani jednego wiersza i nic nie alarmowalo
|
||||||
|
(2026-10-15). Od tego dnia resolve zapisuje tez bledy serwera, ale serwis, ktorego
|
||||||
|
nikt nie odtwarza z innego powodu, i tak bylby niewidoczny. Prog 300 zrodel: w dniu
|
||||||
|
wdrozenia lapalo dokladnie te dwa serwisy.
|
||||||
|
"""
|
||||||
|
rows = session.execute(
|
||||||
|
text(
|
||||||
|
"""
|
||||||
|
WITH big AS (
|
||||||
|
SELECT origin, count(*) AS live
|
||||||
|
FROM playback_sources
|
||||||
|
WHERE dead_at IS NULL
|
||||||
|
GROUP BY origin
|
||||||
|
HAVING count(*) >= :min_live
|
||||||
|
), ev AS (
|
||||||
|
SELECT origin,
|
||||||
|
count(*) AS att,
|
||||||
|
count(*) FILTER (WHERE status = 'success') AS ok
|
||||||
|
FROM playback_events
|
||||||
|
WHERE created_at > now() - interval '7 days'
|
||||||
|
GROUP BY origin
|
||||||
|
)
|
||||||
|
SELECT big.origin, big.live, coalesce(ev.att, 0) AS att
|
||||||
|
FROM big LEFT JOIN ev ON ev.origin = big.origin
|
||||||
|
WHERE coalesce(ev.ok, 0) = 0
|
||||||
|
"""
|
||||||
|
),
|
||||||
|
{"min_live": _SILENT_MIN_LIVE},
|
||||||
|
).all()
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
"origin": r.origin,
|
||||||
|
"bucket": "zero7d",
|
||||||
|
"live": int(r.live),
|
||||||
|
"att7d": int(r.att),
|
||||||
|
"level": "error",
|
||||||
|
"message": (
|
||||||
|
f"playback-watchdog: {r.origin} 0 udanych odtworzen w 7 dni "
|
||||||
|
f"({int(r.att)} prob, {int(r.live)} zywych zrodel)"
|
||||||
|
),
|
||||||
|
}
|
||||||
|
for r in rows
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _emit(items: list[dict[str, Any]], *, kind: str, key: str, sub: str) -> None:
|
||||||
|
try:
|
||||||
|
import sentry_sdk
|
||||||
|
except Exception: # pragma: no cover
|
||||||
|
return
|
||||||
|
for x in items:
|
||||||
|
try:
|
||||||
|
with sentry_sdk.push_scope() as scope:
|
||||||
|
scope.level = x["level"]
|
||||||
|
scope.set_tag("watchdog_kind", kind)
|
||||||
|
scope.set_tag("watchdog_origin", str(x.get(key)))
|
||||||
|
for k, v in x.items():
|
||||||
|
if k not in ("message", "level"):
|
||||||
|
scope.set_extra(k, v)
|
||||||
|
scope.fingerprint = [kind, str(x.get(key)), str(x.get(sub))]
|
||||||
|
sentry_sdk.capture_message(x["message"])
|
||||||
|
except Exception: # pragma: no cover
|
||||||
|
log.exception("quality-watchdog: Sentry capture failed")
|
||||||
|
|
||||||
|
|
||||||
|
def run_quality_watchdogs(*, to_sentry: bool = True) -> dict[str, Any]:
|
||||||
|
with session_scope() as s:
|
||||||
|
listing = check_listing_health(s)
|
||||||
|
parser = check_parser_quality(s)
|
||||||
|
playback = check_playback_health(s) + check_silent_origins(s)
|
||||||
|
for group, name in ((listing, "listing"), (parser, "parser"), (playback, "playback")):
|
||||||
|
for x in group:
|
||||||
|
log.warning("quality-watchdog [%s]: %s", name, x["message"])
|
||||||
|
if to_sentry:
|
||||||
|
_emit(listing, kind="listing-watchdog", key="sitetag", sub="problem")
|
||||||
|
_emit(parser, kind="parser-watchdog", key="origin", sub="metric")
|
||||||
|
_emit(playback, kind="playback-watchdog", key="origin", sub="bucket")
|
||||||
|
summary = {"listing": len(listing), "parser": len(parser), "playback": len(playback)}
|
||||||
|
if not any(summary.values()):
|
||||||
|
log.info("quality-watchdog: bez uwag")
|
||||||
|
return {"summary": summary, "listing": listing, "parser": parser, "playback": playback}
|
||||||
|
|
@ -27,6 +27,7 @@ import { SafeAreaProvider } from 'react-native-safe-area-context';
|
||||||
import { GoonClient } from './src/api';
|
import { GoonClient } from './src/api';
|
||||||
import { ClientProvider } from './src/ClientContext';
|
import { ClientProvider } from './src/ClientContext';
|
||||||
import { PreferencesProvider } from './src/PreferencesContext';
|
import { PreferencesProvider } from './src/PreferencesContext';
|
||||||
|
import { MovieActionsProvider } from './src/MovieActionsContext';
|
||||||
import { SceneActionsProvider } from './src/SceneActionsContext';
|
import { SceneActionsProvider } from './src/SceneActionsContext';
|
||||||
import { OnboardingModal } from './src/components/OnboardingModal';
|
import { OnboardingModal } from './src/components/OnboardingModal';
|
||||||
import { SurveyModal } from './src/components/SurveyModal';
|
import { SurveyModal } from './src/components/SurveyModal';
|
||||||
|
|
@ -505,6 +506,7 @@ export default function App() {
|
||||||
<ClientProvider client={client}>
|
<ClientProvider client={client}>
|
||||||
<PreferencesProvider>
|
<PreferencesProvider>
|
||||||
<SceneActionsProvider>
|
<SceneActionsProvider>
|
||||||
|
<MovieActionsProvider>
|
||||||
<AppNavigator
|
<AppNavigator
|
||||||
client={client}
|
client={client}
|
||||||
appVersion={APP_VERSION}
|
appVersion={APP_VERSION}
|
||||||
|
|
@ -517,6 +519,7 @@ export default function App() {
|
||||||
<WhatsNewModal />
|
<WhatsNewModal />
|
||||||
<SurveyModal />
|
<SurveyModal />
|
||||||
<OnboardingModal />
|
<OnboardingModal />
|
||||||
|
</MovieActionsProvider>
|
||||||
</SceneActionsProvider>
|
</SceneActionsProvider>
|
||||||
</PreferencesProvider>
|
</PreferencesProvider>
|
||||||
</ClientProvider>
|
</ClientProvider>
|
||||||
|
|
|
||||||
|
|
@ -131,8 +131,8 @@ android {
|
||||||
applicationId 'com.goon.mobile'
|
applicationId 'com.goon.mobile'
|
||||||
minSdkVersion rootProject.ext.minSdkVersion
|
minSdkVersion rootProject.ext.minSdkVersion
|
||||||
targetSdkVersion rootProject.ext.targetSdkVersion
|
targetSdkVersion rootProject.ext.targetSdkVersion
|
||||||
versionCode 17
|
versionCode 18
|
||||||
versionName "0.4.3"
|
versionName "0.4.4"
|
||||||
}
|
}
|
||||||
signingConfigs {
|
signingConfigs {
|
||||||
debug {
|
debug {
|
||||||
|
|
|
||||||
|
|
@ -5,9 +5,14 @@ import android.content.Context
|
||||||
import android.content.pm.ActivityInfo
|
import android.content.pm.ActivityInfo
|
||||||
import android.content.pm.PackageManager
|
import android.content.pm.PackageManager
|
||||||
import android.content.res.Configuration
|
import android.content.res.Configuration
|
||||||
|
import android.graphics.drawable.Drawable
|
||||||
|
import android.graphics.drawable.GradientDrawable
|
||||||
import android.os.Build
|
import android.os.Build
|
||||||
import android.os.Bundle
|
import android.os.Bundle
|
||||||
import android.util.Log
|
import android.util.Log
|
||||||
|
import android.util.TypedValue
|
||||||
|
import android.view.View
|
||||||
|
import java.lang.ref.WeakReference
|
||||||
|
|
||||||
import com.facebook.react.ReactActivity
|
import com.facebook.react.ReactActivity
|
||||||
import com.facebook.react.ReactActivityDelegate
|
import com.facebook.react.ReactActivityDelegate
|
||||||
|
|
@ -37,7 +42,9 @@ import expo.modules.ReactActivityDelegateWrapper
|
||||||
*/
|
*/
|
||||||
open class MainActivity : ReactActivity() {
|
open class MainActivity : ReactActivity() {
|
||||||
override fun onCreate(savedInstanceState: Bundle?) {
|
override fun onCreate(savedInstanceState: Bundle?) {
|
||||||
if (!isTelevision()) {
|
// Tablet (krotszy bok >= 600 dp) zostaje bez blokady: pion na tablecie trzymanym w poziomie
|
||||||
|
// to waski pasek, a zdjecie blokady z JS (src/lib/tablet.ts) i tak przychodzi chwile pozniej.
|
||||||
|
if (!isTelevision() && resources.configuration.smallestScreenWidthDp < 600) {
|
||||||
requestedOrientation = ActivityInfo.SCREEN_ORIENTATION_PORTRAIT
|
requestedOrientation = ActivityInfo.SCREEN_ORIENTATION_PORTRAIT
|
||||||
}
|
}
|
||||||
// Set the theme to AppTheme BEFORE onCreate to support
|
// Set the theme to AppTheme BEFORE onCreate to support
|
||||||
|
|
@ -55,6 +62,40 @@ open class MainActivity : ReactActivity() {
|
||||||
} catch (e: Throwable) {
|
} catch (e: Throwable) {
|
||||||
Log.w("Goon", "Cast niedostepny na tym urzadzeniu: " + e.javaClass.simpleName)
|
Log.w("Goon", "Cast niedostepny na tym urzadzeniu: " + e.javaClass.simpleName)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
installFocusRing()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Obwodka wokol elementu z fokusem pilota (Fire TV, Android TV, klawiatura).
|
||||||
|
//
|
||||||
|
// Pilot chodzi po zwyklych przyciskach RN, ale nic nie pokazywalo, gdzie jest fokus
|
||||||
|
// (zgloszenie 7768ae0c, Fire TV): widoki RN maja wlasne tlo bez stanu "focused", Android TV
|
||||||
|
// nie rysuje systemowego podswietlenia, a RN 0.76 na Androidzie nie przekazuje onFocus/onBlur
|
||||||
|
// do JS (sprawdzone 2026-10-15 na emulatorze: przycisk ma fokus, zdarzenie nie przychodzi).
|
||||||
|
// Dlatego natywnie: przy kazdej zmianie fokusu nowy widok dostaje foreground z obwodka,
|
||||||
|
// a poprzedni odzyskuje swoj. W trybie dotykowym (telefon) nic nie rysujemy.
|
||||||
|
private var ringView: WeakReference<View>? = null
|
||||||
|
private var ringPrevForeground: Drawable? = null
|
||||||
|
|
||||||
|
private fun installFocusRing() {
|
||||||
|
window.decorView.viewTreeObserver.addOnGlobalFocusChangeListener { _, newFocus ->
|
||||||
|
ringView?.get()?.let { it.foreground = ringPrevForeground }
|
||||||
|
ringView = null
|
||||||
|
ringPrevForeground = null
|
||||||
|
if (newFocus == null || newFocus.isInTouchMode) return@addOnGlobalFocusChangeListener
|
||||||
|
ringPrevForeground = newFocus.foreground
|
||||||
|
newFocus.foreground = makeFocusRing()
|
||||||
|
ringView = WeakReference(newFocus)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun makeFocusRing(): Drawable {
|
||||||
|
val dp = TypedValue.applyDimension(TypedValue.COMPLEX_UNIT_DIP, 1f, resources.displayMetrics)
|
||||||
|
return GradientDrawable().apply {
|
||||||
|
shape = GradientDrawable.RECTANGLE
|
||||||
|
cornerRadius = 12 * dp
|
||||||
|
setStroke((3 * dp).toInt(), 0xFFF5EDE0.toInt())
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private fun isTelevision(): Boolean {
|
private fun isTelevision(): Boolean {
|
||||||
|
|
|
||||||
|
|
@ -2,7 +2,7 @@
|
||||||
"expo": {
|
"expo": {
|
||||||
"name": "goon",
|
"name": "goon",
|
||||||
"slug": "goon",
|
"slug": "goon",
|
||||||
"version": "0.4.3",
|
"version": "0.4.4",
|
||||||
"orientation": "portrait",
|
"orientation": "portrait",
|
||||||
"userInterfaceStyle": "automatic",
|
"userInterfaceStyle": "automatic",
|
||||||
"newArchEnabled": false,
|
"newArchEnabled": false,
|
||||||
|
|
|
||||||
168
mobile/src/MovieActionsContext.tsx
Normal file
168
mobile/src/MovieActionsContext.tsx
Normal file
|
|
@ -0,0 +1,168 @@
|
||||||
|
/**
|
||||||
|
* MovieActions: scalanie duplikatow filmow przytrzymaniem, jak przy scenach
|
||||||
|
* (SceneActionsContext). Zgloszenie f8c9486e (2026-10-16): te same filmy z dwoch zrodel,
|
||||||
|
* ktorych automat nie zlaczyl, np. "Hardcore Gangbangs 8" i "Hardcore Gangbangs VIII".
|
||||||
|
*
|
||||||
|
* Przytrzymany film ZOSTAJE, kolejne tapniecia zaznaczaja jego duplikaty, przycisk
|
||||||
|
* w bannerze scala wszystkie naraz (`POST /movies/{keep}/merge/{drop}` w petli).
|
||||||
|
* Kontekst, bo MoviePosterCard jest wspolny dla kilku ekranow.
|
||||||
|
*/
|
||||||
|
import { useQueryClient } from '@tanstack/react-query';
|
||||||
|
import React from 'react';
|
||||||
|
import { ActivityIndicator, Alert, Pressable, StyleSheet, Text, View } from 'react-native';
|
||||||
|
|
||||||
|
import { useClient } from './ClientContext';
|
||||||
|
import { theme } from './theme';
|
||||||
|
import type { MovieOut } from './types';
|
||||||
|
|
||||||
|
interface MovieActionsCtx {
|
||||||
|
keeper: MovieOut | null;
|
||||||
|
isSelecting: boolean;
|
||||||
|
isMarked: (movieId: string) => boolean;
|
||||||
|
openActions: (movie: MovieOut) => void;
|
||||||
|
toggleDuplicate: (movie: MovieOut) => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
const Ctx = React.createContext<MovieActionsCtx | null>(null);
|
||||||
|
|
||||||
|
/** Bez providera (np. w testach) karta dziala jak zwykly przycisk. */
|
||||||
|
export function useMovieActions(): MovieActionsCtx | null {
|
||||||
|
return React.useContext(Ctx);
|
||||||
|
}
|
||||||
|
|
||||||
|
const MOVIE_LIST_KEYS = ['movies', 'performer-movies', 'studio-movies', 'studio-movies-pages', 'favorites-movies'];
|
||||||
|
|
||||||
|
export function MovieActionsProvider({ children }: { children: React.ReactNode }) {
|
||||||
|
const client = useClient();
|
||||||
|
const queryClient = useQueryClient();
|
||||||
|
const [keeper, setKeeper] = React.useState<MovieOut | null>(null);
|
||||||
|
const [marked, setMarked] = React.useState<MovieOut[]>([]);
|
||||||
|
const [merging, setMerging] = React.useState(false);
|
||||||
|
|
||||||
|
const reset = React.useCallback(() => {
|
||||||
|
setKeeper(null);
|
||||||
|
setMarked([]);
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
const openActions = React.useCallback(
|
||||||
|
(movie: MovieOut) => {
|
||||||
|
if (keeper) return;
|
||||||
|
Alert.alert(movie.title, 'Movie actions', [
|
||||||
|
{ text: 'Merge duplicates into this', onPress: () => setKeeper(movie) },
|
||||||
|
{ text: 'Cancel', style: 'cancel' },
|
||||||
|
]);
|
||||||
|
},
|
||||||
|
[keeper],
|
||||||
|
);
|
||||||
|
|
||||||
|
const toggleDuplicate = React.useCallback(
|
||||||
|
(movie: MovieOut) => {
|
||||||
|
if (!keeper || merging || movie.id === keeper.id) return;
|
||||||
|
setMarked((prev) =>
|
||||||
|
prev.some((m) => m.id === movie.id) ? prev.filter((m) => m.id !== movie.id) : [...prev, movie],
|
||||||
|
);
|
||||||
|
},
|
||||||
|
[keeper, merging],
|
||||||
|
);
|
||||||
|
|
||||||
|
const isMarked = React.useCallback((movieId: string) => marked.some((m) => m.id === movieId), [marked]);
|
||||||
|
|
||||||
|
const runMerge = React.useCallback(() => {
|
||||||
|
if (!keeper || marked.length === 0 || merging) return;
|
||||||
|
const preview = marked.slice(0, 3).map((m) => `- ${m.title}`).join('\n');
|
||||||
|
const more = marked.length > 3 ? `\n- and ${marked.length - 3} more` : '';
|
||||||
|
Alert.alert(
|
||||||
|
marked.length === 1 ? 'Merge duplicate?' : `Merge ${marked.length} duplicates?`,
|
||||||
|
`Keeping:\n${keeper.title}\n\nMerging in and removing:\n${preview}${more}`,
|
||||||
|
[
|
||||||
|
{ text: 'Cancel', style: 'cancel' },
|
||||||
|
{
|
||||||
|
text: 'Merge',
|
||||||
|
onPress: async () => {
|
||||||
|
setMerging(true);
|
||||||
|
const failed: string[] = [];
|
||||||
|
for (const dup of marked) {
|
||||||
|
try {
|
||||||
|
await client.mergeDuplicateMovie(keeper.id, dup.id);
|
||||||
|
} catch (e: any) {
|
||||||
|
failed.push(`${dup.title}: ${e?.message || 'unknown error'}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
setMerging(false);
|
||||||
|
for (const k of MOVIE_LIST_KEYS) queryClient.invalidateQueries({ queryKey: [k] });
|
||||||
|
queryClient.invalidateQueries({ queryKey: ['movie', keeper.id] });
|
||||||
|
reset();
|
||||||
|
if (failed.length) {
|
||||||
|
Alert.alert(
|
||||||
|
`Merged ${marked.length - failed.length} of ${marked.length}`,
|
||||||
|
failed.slice(0, 5).join('\n\n'),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
);
|
||||||
|
}, [keeper, marked, merging, client, reset, queryClient]);
|
||||||
|
|
||||||
|
const value = React.useMemo(
|
||||||
|
() => ({ keeper, isSelecting: !!keeper, isMarked, openActions, toggleDuplicate }),
|
||||||
|
[keeper, isMarked, openActions, toggleDuplicate],
|
||||||
|
);
|
||||||
|
|
||||||
|
return (
|
||||||
|
<Ctx.Provider value={value}>
|
||||||
|
{children}
|
||||||
|
{keeper ? (
|
||||||
|
<View style={styles.banner} pointerEvents="box-none">
|
||||||
|
<View style={styles.bannerInner}>
|
||||||
|
<Text style={styles.bannerText} numberOfLines={2}>
|
||||||
|
{marked.length === 0
|
||||||
|
? `Tap every duplicate of "${keeper.title}"`
|
||||||
|
: `${marked.length} marked, keeping "${keeper.title}"`}
|
||||||
|
</Text>
|
||||||
|
{merging ? (
|
||||||
|
<ActivityIndicator color={theme.fg} />
|
||||||
|
) : (
|
||||||
|
<Pressable
|
||||||
|
onPress={runMerge}
|
||||||
|
disabled={marked.length === 0}
|
||||||
|
style={[styles.bannerMerge, marked.length === 0 && styles.bannerMergeOff]}
|
||||||
|
hitSlop={8}
|
||||||
|
>
|
||||||
|
<Text style={styles.bannerMergeText}>
|
||||||
|
{marked.length > 0 ? `Merge ${marked.length}` : 'Merge'}
|
||||||
|
</Text>
|
||||||
|
</Pressable>
|
||||||
|
)}
|
||||||
|
<Pressable onPress={reset} style={styles.bannerCancel} hitSlop={8} disabled={merging}>
|
||||||
|
<Text style={styles.bannerCancelText}>Cancel</Text>
|
||||||
|
</Pressable>
|
||||||
|
</View>
|
||||||
|
</View>
|
||||||
|
) : null}
|
||||||
|
</Ctx.Provider>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const styles = StyleSheet.create({
|
||||||
|
banner: { position: 'absolute', left: 0, right: 0, bottom: 0, padding: 10 },
|
||||||
|
bannerInner: {
|
||||||
|
backgroundColor: theme.accent,
|
||||||
|
borderRadius: 12,
|
||||||
|
padding: 12,
|
||||||
|
flexDirection: 'row',
|
||||||
|
alignItems: 'center',
|
||||||
|
gap: 10,
|
||||||
|
},
|
||||||
|
bannerText: { color: theme.fg, fontSize: 13, fontWeight: '600', flex: 1 },
|
||||||
|
bannerMerge: { backgroundColor: theme.fg, borderRadius: 8, paddingVertical: 6, paddingHorizontal: 12 },
|
||||||
|
bannerMergeOff: { opacity: 0.4 },
|
||||||
|
bannerMergeText: { color: theme.accent, fontWeight: '800', fontSize: 13 },
|
||||||
|
bannerCancel: {
|
||||||
|
backgroundColor: 'rgba(0,0,0,0.3)',
|
||||||
|
borderRadius: 8,
|
||||||
|
paddingVertical: 6,
|
||||||
|
paddingHorizontal: 12,
|
||||||
|
},
|
||||||
|
bannerCancelText: { color: theme.fg, fontWeight: '700', fontSize: 13 },
|
||||||
|
});
|
||||||
|
|
@ -143,6 +143,9 @@ export function SceneActionsProvider({ children }: { children: React.ReactNode }
|
||||||
}
|
}
|
||||||
setMerging(false);
|
setMerging(false);
|
||||||
invalidate();
|
invalidate();
|
||||||
|
// Szczegoly zachowanej sceny tez: bez tego ekran sceny pokazywal zrodla sprzed
|
||||||
|
// scalenia i wygladalo, jakby czesc zginela (zgloszenie 0bd8c127).
|
||||||
|
queryClient.invalidateQueries({ queryKey: ['scene', keeper.id] });
|
||||||
reset();
|
reset();
|
||||||
if (failed.length) {
|
if (failed.length) {
|
||||||
Alert.alert(
|
Alert.alert(
|
||||||
|
|
@ -154,7 +157,7 @@ export function SceneActionsProvider({ children }: { children: React.ReactNode }
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
);
|
);
|
||||||
}, [keeper, marked, merging, client, invalidate, reset]);
|
}, [keeper, marked, merging, client, invalidate, reset, queryClient]);
|
||||||
|
|
||||||
const value = React.useMemo(
|
const value = React.useMemo(
|
||||||
() => ({
|
() => ({
|
||||||
|
|
|
||||||
|
|
@ -314,6 +314,10 @@ export class GoonClient {
|
||||||
return this.request(`/scenes/${keepId}/merge/${dropId}`, { method: 'POST' });
|
return this.request(`/scenes/${keepId}/merge/${dropId}`, { method: 'POST' });
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async mergeDuplicateMovie(keepId: string, dropId: string): Promise<{ keep_id: string; dropped_id: string }> {
|
||||||
|
return this.request(`/movies/${keepId}/merge/${dropId}`, { method: 'POST' });
|
||||||
|
}
|
||||||
|
|
||||||
async markMoviePlaybackDead(movieId: string, playbackId: string): Promise<void> {
|
async markMoviePlaybackDead(movieId: string, playbackId: string): Promise<void> {
|
||||||
await this.request(`/movies/${movieId}/playback/${playbackId}/mark-dead`, {
|
await this.request(`/movies/${movieId}/playback/${playbackId}/mark-dead`, {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
|
|
|
||||||
|
|
@ -35,6 +35,71 @@ export type ChangelogEntry = {
|
||||||
};
|
};
|
||||||
|
|
||||||
export const CHANGELOG: ChangelogEntry[] = [
|
export const CHANGELOG: ChangelogEntry[] = [
|
||||||
|
{
|
||||||
|
id: '2026-10-16b',
|
||||||
|
date: 'October 2026',
|
||||||
|
items: [
|
||||||
|
'Movies of a favorite performer added since your last visit are marked NEW and listed first.',
|
||||||
|
'Merge duplicate movies like scenes: long-press the one to keep, tap its copies, then Merge.',
|
||||||
|
'Movies numbered differently ("8" and "VIII") or from the same studio spelled differently are now merged automatically.',
|
||||||
|
],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: '2026-10-16a',
|
||||||
|
date: 'October 2026',
|
||||||
|
items: [
|
||||||
|
'xtapes movies fall back to a second stream when the first one fails or is blocked on your network.',
|
||||||
|
'TV: setting a PIN works with the remote. An OK key now sits in the keypad, the Next button was below the edge of the screen.',
|
||||||
|
],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: '2026-10-15e',
|
||||||
|
date: 'October 2026',
|
||||||
|
items: [
|
||||||
|
'New movie source: xtapes, with 3 to 5 fresh full movies a day, often on the day they come out. They play in the app player.',
|
||||||
|
],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: '2026-10-15d',
|
||||||
|
date: 'October 2026',
|
||||||
|
items: [
|
||||||
|
'After merging duplicate scenes, the scene page shows all merged sources right away instead of the old list.',
|
||||||
|
'Marking the only source of a scene as broken no longer hides the whole scene when the video still exists on the site; 366 scenes hidden this way are back.',
|
||||||
|
'Site lists filtered by a tag load much faster (fullmovies with a tag: from up to 15 s to under half a second).',
|
||||||
|
],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: '2026-10-15c',
|
||||||
|
date: 'October 2026',
|
||||||
|
items: [
|
||||||
|
'rule34video starts playing more than twice as fast: the app reads the video straight from the page instead of loading it twice.',
|
||||||
|
],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: '2026-10-15b',
|
||||||
|
date: 'October 2026',
|
||||||
|
items: [
|
||||||
|
'Version 0.4.4 is out as a new APK (the app will offer to install it). Fire TV and Android TV: the item selected with the remote now has a clear outline, so you can see where you are. Tablets start straight in landscape instead of a narrow strip.',
|
||||||
|
],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: '2026-10-15a',
|
||||||
|
date: 'October 2026',
|
||||||
|
items: [
|
||||||
|
'hentaimama plays again, and xanimeporn plays in the app instead of downloading the file.',
|
||||||
|
'porndish played one of the same two videos for every scene because of a cache bug on their side. Each scene now plays its own video.',
|
||||||
|
'perverzija works again after their site started blocking us this morning.',
|
||||||
|
'Movies from mangoporn show their DoodStream and Mixdrop links again, and titles like "Watch ... Porn Movie Online Free" are cleaned up.',
|
||||||
|
'Deleted fpoxxx and sxyland videos no longer play an ad or a random clip instead of the scene.',
|
||||||
|
],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: '2026-10-13b',
|
||||||
|
date: 'October 2026',
|
||||||
|
items: [
|
||||||
|
'Version 0.4.3 is out as a new APK (the app will offer to install it). Android TV: the app no longer crashes when it was opened from two places at once, for example the TV home screen and a shortcut. Crash reports from the native side now carry enough detail to be fixed.',
|
||||||
|
],
|
||||||
|
},
|
||||||
{
|
{
|
||||||
id: '2026-10-13a',
|
id: '2026-10-13a',
|
||||||
date: 'October 2026',
|
date: 'October 2026',
|
||||||
|
|
|
||||||
|
|
@ -3,6 +3,7 @@
|
||||||
import { Image } from 'expo-image';
|
import { Image } from 'expo-image';
|
||||||
import React from 'react';
|
import React from 'react';
|
||||||
import { Pressable, StyleSheet, Text, View } from 'react-native';
|
import { Pressable, StyleSheet, Text, View } from 'react-native';
|
||||||
|
import { useMovieActions } from '../MovieActionsContext';
|
||||||
import { fonts, theme } from '../theme';
|
import { fonts, theme } from '../theme';
|
||||||
import type { MovieOut } from '../types';
|
import type { MovieOut } from '../types';
|
||||||
|
|
||||||
|
|
@ -16,6 +17,19 @@ export function MoviePosterCard({
|
||||||
onPress: () => void;
|
onPress: () => void;
|
||||||
}) {
|
}) {
|
||||||
const studio = movie.studio?.name;
|
const studio = movie.studio?.name;
|
||||||
|
// Przytrzymanie: scalanie duplikatow (MovieActionsContext). W trybie wyboru tap
|
||||||
|
// zaznacza i odznacza zamiast otwierac film; karta zachowywana nie reaguje.
|
||||||
|
const actions = useMovieActions();
|
||||||
|
const isKeeper = actions?.keeper?.id === movie.id;
|
||||||
|
const marked = actions?.isMarked(movie.id) ?? false;
|
||||||
|
const selecting = !!actions?.isSelecting;
|
||||||
|
const handlePress = () => {
|
||||||
|
if (selecting) {
|
||||||
|
if (!isKeeper) actions?.toggleDuplicate(movie);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
onPress();
|
||||||
|
};
|
||||||
// Watched indicator (parytet ze ScenesScreen): finished=true → dim posteru +
|
// Watched indicator (parytet ze ScenesScreen): finished=true → dim posteru +
|
||||||
// ✓ badge w prawym górnym. Pośredni progress bez dim'a (user może wrócić).
|
// ✓ badge w prawym górnym. Pośredni progress bez dim'a (user może wrócić).
|
||||||
// bug-report b207ff17 2026-05-26.
|
// bug-report b207ff17 2026-05-26.
|
||||||
|
|
@ -25,8 +39,20 @@ export function MoviePosterCard({
|
||||||
? Math.min(100, Math.round((movie.position_sec / movie.duration_sec) * 100))
|
? Math.min(100, Math.round((movie.position_sec / movie.duration_sec) * 100))
|
||||||
: 0;
|
: 0;
|
||||||
return (
|
return (
|
||||||
<Pressable style={styles.card} onPress={onPress}>
|
<Pressable
|
||||||
<View style={styles.posterWrap}>
|
style={styles.card}
|
||||||
|
onPress={handlePress}
|
||||||
|
onLongPress={actions ? () => actions.openActions(movie) : undefined}
|
||||||
|
delayLongPress={350}
|
||||||
|
>
|
||||||
|
<View
|
||||||
|
style={[
|
||||||
|
styles.posterWrap,
|
||||||
|
selecting && !isKeeper && !marked && styles.posterSelectable,
|
||||||
|
marked && styles.posterMarked,
|
||||||
|
isKeeper && styles.posterKeeper,
|
||||||
|
]}
|
||||||
|
>
|
||||||
{movie.poster_url ? (
|
{movie.poster_url ? (
|
||||||
<Image
|
<Image
|
||||||
source={{ uri: movie.poster_url }}
|
source={{ uri: movie.poster_url }}
|
||||||
|
|
@ -43,7 +69,14 @@ export function MoviePosterCard({
|
||||||
<Text style={styles.ratingText}>{movie.rating.toFixed(1)}</Text>
|
<Text style={styles.ratingText}>{movie.rating.toFixed(1)}</Text>
|
||||||
</View>
|
</View>
|
||||||
) : null}
|
) : null}
|
||||||
{isNew ? (
|
{isKeeper || marked ? (
|
||||||
|
<View style={[styles.dupBadge, isKeeper && styles.keepBadge]}>
|
||||||
|
<Text style={[styles.dupBadgeText, isKeeper && styles.keepBadgeText]}>
|
||||||
|
{isKeeper ? 'KEEP' : 'DUPLICATE'}
|
||||||
|
</Text>
|
||||||
|
</View>
|
||||||
|
) : null}
|
||||||
|
{isNew && !isKeeper && !marked ? (
|
||||||
<View style={styles.newBadge}>
|
<View style={styles.newBadge}>
|
||||||
<Text style={styles.newBadgeText}>NEW</Text>
|
<Text style={styles.newBadgeText}>NEW</Text>
|
||||||
</View>
|
</View>
|
||||||
|
|
@ -103,6 +136,21 @@ const styles = StyleSheet.create({
|
||||||
},
|
},
|
||||||
newBadgeText: { color: theme.fg, fontSize: 9, fontFamily: fonts.mono, fontWeight: '700', letterSpacing: 0.6 },
|
newBadgeText: { color: theme.fg, fontSize: 9, fontFamily: fonts.mono, fontWeight: '700', letterSpacing: 0.6 },
|
||||||
posterDimmed: { opacity: 0.45 },
|
posterDimmed: { opacity: 0.45 },
|
||||||
|
posterSelectable: { borderWidth: 2, borderColor: theme.accentText, opacity: 0.85 },
|
||||||
|
posterMarked: { borderWidth: 2, borderColor: theme.fg, opacity: 0.55 },
|
||||||
|
posterKeeper: { borderWidth: 3, borderColor: theme.accent, opacity: 1 },
|
||||||
|
dupBadge: {
|
||||||
|
position: 'absolute',
|
||||||
|
top: 8,
|
||||||
|
left: 8,
|
||||||
|
backgroundColor: theme.accent,
|
||||||
|
paddingHorizontal: 6,
|
||||||
|
paddingVertical: 2,
|
||||||
|
borderRadius: 4,
|
||||||
|
},
|
||||||
|
dupBadgeText: { color: theme.fg, fontSize: 9, fontWeight: '800', letterSpacing: 0.6 },
|
||||||
|
keepBadge: { backgroundColor: theme.fg },
|
||||||
|
keepBadgeText: { color: theme.accent },
|
||||||
watchedBadge: {
|
watchedBadge: {
|
||||||
position: 'absolute',
|
position: 'absolute',
|
||||||
bottom: 8,
|
bottom: 8,
|
||||||
|
|
|
||||||
|
|
@ -12,9 +12,22 @@
|
||||||
* bez proxy. Ten sam wzorzec co doodstream.ts (`redirect: 'manual'` + Location header,
|
* bez proxy. Ten sam wzorzec co doodstream.ts (`redirect: 'manual'` + Location header,
|
||||||
* potwierdzone że działa na Android RN).
|
* potwierdzone że działa na Android RN).
|
||||||
*/
|
*/
|
||||||
|
import type { StreamLink } from '../types';
|
||||||
|
|
||||||
const UA =
|
const UA =
|
||||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36';
|
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36';
|
||||||
|
|
||||||
|
/** fetch z limitem czasu: bez niego wiszaca strona trzymala ekran "resolving" bez konca. */
|
||||||
|
async function fetchWithTimeout(url: string, init: RequestInit, ms: number): Promise<Response> {
|
||||||
|
const ctrl = new AbortController();
|
||||||
|
const t = setTimeout(() => ctrl.abort(), ms);
|
||||||
|
try {
|
||||||
|
return await fetch(url, { ...init, signal: ctrl.signal });
|
||||||
|
} finally {
|
||||||
|
clearTimeout(t);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/** get_file URL który wymaga in-session resolve (nie nasz proxy URL). */
|
/** get_file URL który wymaga in-session resolve (nie nasz proxy URL). */
|
||||||
export function isGetFileUrl(url: string): boolean {
|
export function isGetFileUrl(url: string): boolean {
|
||||||
return /\/get_file\//.test(url) && !/\/proxy\//.test(url);
|
return /\/get_file\//.test(url) && !/\/proxy\//.test(url);
|
||||||
|
|
@ -49,7 +62,7 @@ export async function resolveGetFilePage(
|
||||||
): Promise<string | null> {
|
): Promise<string | null> {
|
||||||
let html: string;
|
let html: string;
|
||||||
try {
|
try {
|
||||||
const r = await fetch(pageUrl, { headers: { 'User-Agent': UA, Accept: 'text/html' } });
|
const r = await fetchWithTimeout(pageUrl, { headers: { 'User-Agent': UA, Accept: 'text/html' } }, 10000);
|
||||||
if (!r.ok) return null;
|
if (!r.ok) return null;
|
||||||
html = await r.text();
|
html = await r.text();
|
||||||
} catch {
|
} catch {
|
||||||
|
|
@ -118,7 +131,7 @@ export async function resolveGetFile(url: string, referer?: string): Promise<str
|
||||||
for (let hop = 0; hop < 4; hop++) {
|
for (let hop = 0; hop < 4; hop++) {
|
||||||
let r: Response;
|
let r: Response;
|
||||||
try {
|
try {
|
||||||
r = await fetch(cur, {
|
r = await fetchWithTimeout(cur, {
|
||||||
method: 'GET',
|
method: 'GET',
|
||||||
redirect: 'manual',
|
redirect: 'manual',
|
||||||
headers: {
|
headers: {
|
||||||
|
|
@ -126,7 +139,7 @@ export async function resolveGetFile(url: string, referer?: string): Promise<str
|
||||||
Range: 'bytes=0-1',
|
Range: 'bytes=0-1',
|
||||||
...(referer ? { Referer: referer } : {}),
|
...(referer ? { Referer: referer } : {}),
|
||||||
},
|
},
|
||||||
});
|
}, 6000);
|
||||||
} catch {
|
} catch {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
@ -150,3 +163,54 @@ export async function resolveGetFile(url: string, referer?: string): Promise<str
|
||||||
}
|
}
|
||||||
return cur;
|
return cur;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* rule34video od razu na telefonie (zgloszenie 008e261d, 2026-10-12: "dziala, ale strasznie
|
||||||
|
* wolno"). Wczesniej: resolve na serwerze, potem telefon i tak pobieral cala strone jeszcze
|
||||||
|
* raz (242 KB, token v-acctoken jest zwiazany z IP pobierajacego) i robil przeskok przez
|
||||||
|
* get_file, w sumie ok. 3,5 s po odpowiedzi serwera. Tu jedno pobranie strony na telefonie
|
||||||
|
* i wszystkie jakosci z flashvars. Przekierowanie get_file -> boomio-cdn robi sam ExoPlayer
|
||||||
|
* (302, potem 206 bez Referera, sprawdzone z IP domowego).
|
||||||
|
*/
|
||||||
|
export async function resolveRule34Page(pageUrl: string): Promise<StreamLink[]> {
|
||||||
|
let html: string;
|
||||||
|
try {
|
||||||
|
const r = await fetchWithTimeout(pageUrl, { headers: { 'User-Agent': UA, Accept: 'text/html' } }, 10000);
|
||||||
|
if (!r.ok) return [];
|
||||||
|
html = await r.text();
|
||||||
|
} catch {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
const labels = new Map<string, string>();
|
||||||
|
let lm: RegExpExecArray | null;
|
||||||
|
_FLASHVAR_LABEL_RE.lastIndex = 0;
|
||||||
|
while ((lm = _FLASHVAR_LABEL_RE.exec(html)) !== null) labels.set(lm[1], lm[2].trim());
|
||||||
|
let origin = '';
|
||||||
|
try {
|
||||||
|
origin = new URL(pageUrl).origin;
|
||||||
|
} catch {
|
||||||
|
// zostaje pusty
|
||||||
|
}
|
||||||
|
const rank = (q: string) => ({ '1080p': 0, '720p': 1, '480p': 2, '360p': 3 }[q.toLowerCase()] ?? 99);
|
||||||
|
const out: StreamLink[] = [];
|
||||||
|
const seen = new Set<string>();
|
||||||
|
let fm: RegExpExecArray | null;
|
||||||
|
_FLASHVAR_URL_RE.lastIndex = 0;
|
||||||
|
while ((fm = _FLASHVAR_URL_RE.exec(html)) !== null) {
|
||||||
|
const q = labels.get(fm[1]) || '';
|
||||||
|
if (_SKIP_Q.test(q)) continue;
|
||||||
|
let u = fm[2];
|
||||||
|
if (u.startsWith('//')) u = 'https:' + u;
|
||||||
|
if (seen.has(u)) continue;
|
||||||
|
seen.add(u);
|
||||||
|
out.push({
|
||||||
|
stream_url: u,
|
||||||
|
direct_url: u,
|
||||||
|
headers: { 'User-Agent': UA, ...(origin ? { Referer: origin + '/' } : {}) },
|
||||||
|
quality: q || null,
|
||||||
|
type: 'mp4',
|
||||||
|
} as StreamLink);
|
||||||
|
}
|
||||||
|
out.sort((a, b) => rank(a.quality || '') - rank(b.quality || ''));
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -31,6 +31,9 @@ const PACKER_HOSTS = new Set([
|
||||||
'streamwish.com', 'streamwish.to', 'embedwish.com', 'wishembed.pro',
|
'streamwish.com', 'streamwish.to', 'embedwish.com', 'wishembed.pro',
|
||||||
'awish.pro', 'dwish.pro', 'mwish.pro', 'swishsrv.com',
|
'awish.pro', 'dwish.pro', 'mwish.pro', 'swishsrv.com',
|
||||||
'streamwish.site', 'hlswish.com', 'kswplayer.info',
|
'streamwish.site', 'hlswish.com', 'kswplayer.info',
|
||||||
|
// sextapes.to (filmy z xtapes.la, 2026-10-15): ten sam P.A.C.K.E.R.-JWPlayer, HLS na
|
||||||
|
// cdn-centaurus zwiazany z IP rozwiazujacego, wiec rozwiazac musi telefon.
|
||||||
|
'sextapes.to',
|
||||||
]);
|
]);
|
||||||
|
|
||||||
const UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36';
|
const UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36';
|
||||||
|
|
@ -93,6 +96,65 @@ export function unpackPacker(js: string): string | null {
|
||||||
const FILE_RE = /(?:["']?file["']?|sources?)\s*[:=]\s*["'](https?:\/\/[^"']+\.(?:m3u8|mp4|mpd)[^"']*)["']/i;
|
const FILE_RE = /(?:["']?file["']?|sources?)\s*[:=]\s*["'](https?:\/\/[^"']+\.(?:m3u8|mp4|mpd)[^"']*)["']/i;
|
||||||
const AD_RE = /\/(?:preroll|midroll|postroll|preplay|ads?|advert|promo)\d*\.(?:mp4|m3u8|webm)/i;
|
const AD_RE = /\/(?:preroll|midroll|postroll|preplay|ads?|advert|promo)\d*\.(?:mp4|m3u8|webm)/i;
|
||||||
|
|
||||||
|
// Nowszy zapis StreamWish (sextapes.to 2026-10): `var links={"hls4":"/stream/.../master.m3u8",
|
||||||
|
// "hls2":"https://cdn/.../master.m3u8?t=..."}` i `sources:[{file:links.hls4||links.hls2}]`,
|
||||||
|
// wiec FILE_RE nie widzi literalu. Bierzemy pelny adres (hls2), a gdy go nie ma, wzgledny
|
||||||
|
// (hls4) doklejony do domeny embedu.
|
||||||
|
const LINKS_ENTRY_RE = /["']?(hls\d+)["']?\s*:\s*["']([^"']+\.m3u8[^"']*)["']/gi;
|
||||||
|
|
||||||
|
/** Kandydaci z obiektu `links`: najpierw pelne adresy (hls2, CDN), potem wzgledne (hls4,
|
||||||
|
* sciezka na domenie embedu) zamienione na pelne. */
|
||||||
|
function linksObjectCandidates(js: string, embedUrl: string): string[] {
|
||||||
|
const absolute: string[] = [];
|
||||||
|
const relative: string[] = [];
|
||||||
|
let m: RegExpExecArray | null;
|
||||||
|
LINKS_ENTRY_RE.lastIndex = 0;
|
||||||
|
while ((m = LINKS_ENTRY_RE.exec(js)) !== null) {
|
||||||
|
const u = m[2];
|
||||||
|
if (AD_RE.test(u)) continue;
|
||||||
|
if (/^https?:\/\//i.test(u)) absolute.push(u);
|
||||||
|
else if (u.startsWith('/')) {
|
||||||
|
try {
|
||||||
|
relative.push(new URL(u, embedUrl).toString());
|
||||||
|
} catch {
|
||||||
|
// pomijamy
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return [...absolute, ...relative];
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Czy manifest pod adresem odpowiada (krotki GET z limitem czasu). */
|
||||||
|
async function manifestReachable(url: string, referer: string): Promise<boolean> {
|
||||||
|
const ctrl = new AbortController();
|
||||||
|
const t = setTimeout(() => ctrl.abort(), 4000);
|
||||||
|
try {
|
||||||
|
const r = await fetch(url, { headers: { 'User-Agent': UA, Referer: referer }, signal: ctrl.signal });
|
||||||
|
if (!r.ok) return false;
|
||||||
|
return (await r.text()).trimStart().startsWith('#EXTM3U');
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
} finally {
|
||||||
|
clearTimeout(t);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Adres z obiektu `links`, sprawdzony z telefonu. sextapes (2026-10-16): hls2 lezy na
|
||||||
|
* zewnetrznym CDN (premilkyway, cdn-centaurus), ktory potrafi dac 502 albo byc zablokowany
|
||||||
|
* przez filtr reklam w DNS (blad "Failed to connect to ...premilkyway.com/[::]"). Wtedy
|
||||||
|
* bierzemy hls4 z domeny embedu (tam strona sama zaczyna). Segmenty hls4 to TS z doklejonym
|
||||||
|
* na poczatku 70-bajtowym PNG, ExoPlayer i tak znajduje synchronizacje TS.
|
||||||
|
*/
|
||||||
|
async function linksObjectUrl(js: string, embedUrl: string, referer: string): Promise<string | null> {
|
||||||
|
const candidates = linksObjectCandidates(js, embedUrl);
|
||||||
|
if (candidates.length <= 1) return candidates[0] || null;
|
||||||
|
for (const c of candidates) {
|
||||||
|
if (await manifestReachable(c, referer)) return c;
|
||||||
|
}
|
||||||
|
return candidates[0];
|
||||||
|
}
|
||||||
|
|
||||||
export async function resolvePackerHoster(
|
export async function resolvePackerHoster(
|
||||||
embedUrl: string,
|
embedUrl: string,
|
||||||
sourceUrl?: string,
|
sourceUrl?: string,
|
||||||
|
|
@ -137,6 +199,7 @@ export async function resolvePackerHoster(
|
||||||
}
|
}
|
||||||
const um = unpacked.match(FILE_RE);
|
const um = unpacked.match(FILE_RE);
|
||||||
if (um && !AD_RE.test(um[1])) videoUrl = um[1];
|
if (um && !AD_RE.test(um[1])) videoUrl = um[1];
|
||||||
|
if (!videoUrl) videoUrl = await linksObjectUrl(unpacked, url, referer);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!videoUrl) return { error: 'no_video_url' };
|
if (!videoUrl) return { error: 'no_video_url' };
|
||||||
|
|
|
||||||
|
|
@ -223,7 +223,14 @@ export function PerformerScenesScreen() {
|
||||||
[scenesQuery.data?.items, seenSince],
|
[scenesQuery.data?.items, seenSince],
|
||||||
);
|
);
|
||||||
|
|
||||||
const movies = moviesQuery.data?.items ?? [];
|
// Filmy jak sceny: NEW (dodane od ostatniej wizyty w ulubionych) na gore i z odznaka
|
||||||
|
// (zgloszenie f8c9486e: "widac nowe filmy, ale nie widac ktore to sa").
|
||||||
|
const movies = React.useMemo(() => {
|
||||||
|
const items = moviesQuery.data?.items ?? [];
|
||||||
|
if (!seenSince) return items;
|
||||||
|
const isNew = (m: (typeof items)[number]) => !!(m.created_at && m.created_at > seenSince);
|
||||||
|
return [...items.filter(isNew), ...items.filter((m) => !isNew(m))];
|
||||||
|
}, [moviesQuery.data?.items, seenSince]);
|
||||||
const scenesTotal = scenesQuery.data?.total ?? 0;
|
const scenesTotal = scenesQuery.data?.total ?? 0;
|
||||||
const scenesSuffix = scenesQuery.data?.total_capped ? '+' : '';
|
const scenesSuffix = scenesQuery.data?.total_capped ? '+' : '';
|
||||||
const moviesTotal = moviesQuery.data?.total ?? 0;
|
const moviesTotal = moviesQuery.data?.total ?? 0;
|
||||||
|
|
@ -340,6 +347,7 @@ export function PerformerScenesScreen() {
|
||||||
return (
|
return (
|
||||||
<MoviePosterCard
|
<MoviePosterCard
|
||||||
movie={item}
|
movie={item}
|
||||||
|
isNew={!!(seenSince && item.created_at && item.created_at > seenSince)}
|
||||||
onPress={() => navigation.navigate('MovieDetail', { id: item.id })}
|
onPress={() => navigation.navigate('MovieDetail', { id: item.id })}
|
||||||
/>
|
/>
|
||||||
);
|
);
|
||||||
|
|
|
||||||
|
|
@ -44,7 +44,21 @@ export function PinEntry({ title, error, onSubmit, onCancel, footerAction }: Pro
|
||||||
<View style={styles.pad}>
|
<View style={styles.pad}>
|
||||||
{KEYS.map((k, i) =>
|
{KEYS.map((k, i) =>
|
||||||
k === null ? (
|
k === null ? (
|
||||||
<View key={i} style={[styles.key, styles.keyEmpty]} />
|
// Zatwierdzenie tez w samej klawiaturze: na telewizorze w poziomie przyciski
|
||||||
|
// Cancel/Next lezaly pod dolna krawedzia ekranu i nie dalo sie przejsc dalej
|
||||||
|
// (zgloszenie 94305101, Fire TV AFTMM: "where is ok").
|
||||||
|
<Pressable
|
||||||
|
key={i}
|
||||||
|
disabled={pin.length < 4}
|
||||||
|
style={({ pressed }) => [
|
||||||
|
styles.key,
|
||||||
|
pin.length < 4 ? styles.keyEmpty : styles.keyOk,
|
||||||
|
pressed && styles.keyPressed,
|
||||||
|
]}
|
||||||
|
onPress={() => onSubmit(pin)}
|
||||||
|
>
|
||||||
|
{pin.length >= 4 ? <Text style={styles.keyOkText}>OK</Text> : null}
|
||||||
|
</Pressable>
|
||||||
) : (
|
) : (
|
||||||
<Pressable
|
<Pressable
|
||||||
key={i}
|
key={i}
|
||||||
|
|
@ -114,6 +128,8 @@ const styles = StyleSheet.create({
|
||||||
justifyContent: 'center',
|
justifyContent: 'center',
|
||||||
},
|
},
|
||||||
keyEmpty: { backgroundColor: 'transparent', borderColor: 'transparent' },
|
keyEmpty: { backgroundColor: 'transparent', borderColor: 'transparent' },
|
||||||
|
keyOk: { backgroundColor: theme.accent, borderColor: theme.accentText },
|
||||||
|
keyOkText: { color: theme.fg, fontSize: 20, fontWeight: '700' },
|
||||||
keyPressed: { backgroundColor: theme.bgElevated, borderColor: theme.borderFocus },
|
keyPressed: { backgroundColor: theme.bgElevated, borderColor: theme.borderFocus },
|
||||||
keyText: { color: theme.fg, fontSize: 26, fontWeight: '500' },
|
keyText: { color: theme.fg, fontSize: 26, fontWeight: '500' },
|
||||||
keyBack: { color: theme.muted, fontSize: 22 },
|
keyBack: { color: theme.muted, fontSize: 22 },
|
||||||
|
|
|
||||||
|
|
@ -187,6 +187,9 @@ const AD_HOSTS = [
|
||||||
'google-analytics.com', 'googletagmanager.com',
|
'google-analytics.com', 'googletagmanager.com',
|
||||||
'traffichaus.com', 'plugrush.com', 'clickadu.com',
|
'traffichaus.com', 'plugrush.com', 'clickadu.com',
|
||||||
'redirectvoluum.com', 'clickaine.com', 'hilltopads.net', 'popshq.com',
|
'redirectvoluum.com', 'clickaine.com', 'hilltopads.net', 'popshq.com',
|
||||||
|
// Wideo reklam tsyndicate (VAST -> sacdnssedge .mp4): na stronie skasowanego filmu fpoxxx
|
||||||
|
// bylo jedynym wideo i gralo jako scena (zgloszenie 1ee12943, 2026-10-12).
|
||||||
|
'sacdnssedge.com',
|
||||||
];
|
];
|
||||||
|
|
||||||
function detectMode(url: string): 'video' | 'webview' {
|
function detectMode(url: string): 'video' | 'webview' {
|
||||||
|
|
@ -522,6 +525,8 @@ function NativeVideoPlayer({ params }: { params: RouteParams }) {
|
||||||
links = await (await import('../lib/fpoxxxResolver')).resolveFpoxxxPage(resolvePageUrl);
|
links = await (await import('../lib/fpoxxxResolver')).resolveFpoxxxPage(resolvePageUrl);
|
||||||
} else if (resolvePageUrl && playOrigin === 'tube:porndoecom') {
|
} else if (resolvePageUrl && playOrigin === 'tube:porndoecom') {
|
||||||
links = await (await import('../lib/porndoeResolver')).resolvePorndoePage(resolvePageUrl);
|
links = await (await import('../lib/porndoeResolver')).resolvePorndoePage(resolvePageUrl);
|
||||||
|
} else if (resolvePageUrl && playOrigin === 'tube:rule34video') {
|
||||||
|
links = await (await import('../lib/getfileResolver')).resolveRule34Page(resolvePageUrl);
|
||||||
} else if (canBackendReresolve && sceneId && params.playbackId) {
|
} else if (canBackendReresolve && sceneId && params.playbackId) {
|
||||||
// Świeży resolve przez backend (VPS re-runuje extractor KVS → nowy nod + token).
|
// Świeży resolve przez backend (VPS re-runuje extractor KVS → nowy nod + token).
|
||||||
const res = await client.resolvePlayback(sceneId, params.playbackId);
|
const res = await client.resolvePlayback(sceneId, params.playbackId);
|
||||||
|
|
@ -1588,7 +1593,7 @@ const INJECTED_JS = `
|
||||||
'redirectvoluum.com','clickaine.com','hilltopads.net','popshq.com',
|
'redirectvoluum.com','clickaine.com','hilltopads.net','popshq.com',
|
||||||
// VIDEO-ad CDN/VAST preroll (mp4) — KVS hosty (yespornvip/pornditt) wstrzykują
|
// VIDEO-ad CDN/VAST preroll (mp4) — KVS hosty (yespornvip/pornditt) wstrzykują
|
||||||
// preroll który scrape inaczej łapie zamiast contentu. 2026-06-01.
|
// preroll który scrape inaczej łapie zamiast contentu. 2026-06-01.
|
||||||
'trafostatic.com','bkcdn.net','gripi.online','dipi.online','tsyndicate.com',
|
'trafostatic.com','bkcdn.net','gripi.online','dipi.online','tsyndicate.com','sacdnssedge.com',
|
||||||
'marzaent.com','yomeno.xyz','cdn-fc.com','tapioni.com','fh-wgt.com',
|
'marzaent.com','yomeno.xyz','cdn-fc.com','tapioni.com','fh-wgt.com',
|
||||||
'xlivrdr.com','dynspt.com','svtrck.site','urlhaus.com',
|
'xlivrdr.com','dynspt.com','svtrck.site','urlhaus.com',
|
||||||
// xtremestream (perverzija) trzyma preroll/midroll na własnym bunny-CDN i odtwarza je
|
// xtremestream (perverzija) trzyma preroll/midroll na własnym bunny-CDN i odtwarza je
|
||||||
|
|
@ -2138,7 +2143,7 @@ function EmbedWebViewPlayer({ params }: { params: RouteParams }) {
|
||||||
const isLikelyAd = React.useCallback((u: string): boolean => {
|
const isLikelyAd = React.useCallback((u: string): boolean => {
|
||||||
const low = u.toLowerCase();
|
const low = u.toLowerCase();
|
||||||
if (/\/(preroll|midroll|postroll|adroll|ad-?\d+|advert)/.test(low)) return true;
|
if (/\/(preroll|midroll|postroll|adroll|ad-?\d+|advert)/.test(low)) return true;
|
||||||
if (/(tsyndicate|exoclick|trafficstars|adskeeper|popmyads|magsrv|adcash)\./.test(low)) return true;
|
if (/(tsyndicate|sacdnssedge|exoclick|trafficstars|adskeeper|popmyads|magsrv|adcash)\./.test(low)) return true;
|
||||||
// Trailer / preview thumbnails, często `_preview.mp4` lub thumb `.jpg.mp4`
|
// Trailer / preview thumbnails, często `_preview.mp4` lub thumb `.jpg.mp4`
|
||||||
if (/_preview\.(mp4|m3u8)/.test(low)) return true;
|
if (/_preview\.(mp4|m3u8)/.test(low)) return true;
|
||||||
if (/\.jpg\.mp4/.test(low)) return true;
|
if (/\.jpg\.mp4/.test(low)) return true;
|
||||||
|
|
|
||||||
|
|
@ -26,6 +26,7 @@ import { resolveSxyprnPage } from '../lib/sxyprnResolver';
|
||||||
import { resolveEpornerPage } from '../lib/epornerResolver';
|
import { resolveEpornerPage } from '../lib/epornerResolver';
|
||||||
import { resolveFpoxxxPage } from '../lib/fpoxxxResolver';
|
import { resolveFpoxxxPage } from '../lib/fpoxxxResolver';
|
||||||
import { resolvePorndoePage } from '../lib/porndoeResolver';
|
import { resolvePorndoePage } from '../lib/porndoeResolver';
|
||||||
|
import { resolveRule34Page } from '../lib/getfileResolver';
|
||||||
import { prefersPageMode } from '../lib/pageMode';
|
import { prefersPageMode } from '../lib/pageMode';
|
||||||
import type { RootStackParamList } from '../navigation';
|
import type { RootStackParamList } from '../navigation';
|
||||||
import { fonts, theme } from '../theme';
|
import { fonts, theme } from '../theme';
|
||||||
|
|
@ -640,7 +641,9 @@ function PlaybackButton({
|
||||||
// hdporn.gg/fullmovies.xxx: backendowy get_file jest zbindowany do IP VPS (page-loader)
|
// hdporn.gg/fullmovies.xxx: backendowy get_file jest zbindowany do IP VPS (page-loader)
|
||||||
// → telefon dostaje 410 → fallback proxy (mignięcie). Re-fetchujemy STRONĘ na telefonie
|
// → telefon dostaje 410 → fallback proxy (mignięcie). Re-fetchujemy STRONĘ na telefonie
|
||||||
// (phone-bound get_file) → fpvcdn z IP telefonu → ExoPlayer gra direct (bez proxy/migotania).
|
// (phone-bound get_file) → fpvcdn z IP telefonu → ExoPlayer gra direct (bez proxy/migotania).
|
||||||
if (isDirect && isGetFileUrl(initialUrl)) {
|
// Link z resolvera na telefonie jest juz zwiazany z IP telefonu, drugie pobranie strony
|
||||||
|
// tylko opoznialo start (rule34video, 008e261d).
|
||||||
|
if (isDirect && via !== 'phone' && isGetFileUrl(initialUrl)) {
|
||||||
const ref = link.headers?.Referer || (refererHost ? `https://${refererHost}/` : undefined);
|
const ref = link.headers?.Referer || (refererHost ? `https://${refererHost}/` : undefined);
|
||||||
const resolved = await resolveGetFilePage(source.page_url, link.quality, ref);
|
const resolved = await resolveGetFilePage(source.page_url, link.quality, ref);
|
||||||
if (resolved) initialUrl = resolved;
|
if (resolved) initialUrl = resolved;
|
||||||
|
|
@ -648,7 +651,7 @@ function PlaybackButton({
|
||||||
|
|
||||||
// IP-bound tuby resolwowane phone-side: przekaż page_url, by Player mógł
|
// IP-bound tuby resolwowane phone-side: przekaż page_url, by Player mógł
|
||||||
// re-resolve'ować świeży token gdy native padnie na initial-load (zmiana IP / TTL).
|
// re-resolve'ować świeży token gdy native padnie na initial-load (zmiana IP / TTL).
|
||||||
const PHONE_RESOLVE_ORIGINS = ['tube:sxyprncom', 'tube:epornercom', 'tube:fpoxxx', 'tube:porndoecom'];
|
const PHONE_RESOLVE_ORIGINS = ['tube:sxyprncom', 'tube:epornercom', 'tube:fpoxxx', 'tube:porndoecom', 'tube:rule34video'];
|
||||||
nav.navigate('Player', {
|
nav.navigate('Player', {
|
||||||
url: initialUrl,
|
url: initialUrl,
|
||||||
sceneId,
|
sceneId,
|
||||||
|
|
@ -729,6 +732,8 @@ function PlaybackButton({
|
||||||
? resolvePornxpPage
|
? resolvePornxpPage
|
||||||
: source.origin === 'tube:porndoecom'
|
: source.origin === 'tube:porndoecom'
|
||||||
? resolvePorndoePage
|
? resolvePorndoePage
|
||||||
|
: source.origin === 'tube:rule34video'
|
||||||
|
? resolveRule34Page
|
||||||
: // hqfap zdjęty 2026-07-28: strona wdrożyła Cloudflare z JS-challenge na
|
: // hqfap zdjęty 2026-07-28: strona wdrożyła Cloudflare z JS-challenge na
|
||||||
// /watch/ (403 dla każdego zwykłego fetcha, także z pełnym kompletem
|
// /watch/ (403 dla każdego zwykłego fetcha, także z pełnym kompletem
|
||||||
// nagłówków przeglądarki), więc phone-resolve tylko padał i zrzucał usera
|
// nagłówków przeglądarki), więc phone-resolve tylko padał i zrzucał usera
|
||||||
|
|
|
||||||
88
scripts/backfill_dooplay_metadata.py
Normal file
88
scripts/backfill_dooplay_metadata.py
Normal file
|
|
@ -0,0 +1,88 @@
|
||||||
|
"""Uzupelnia obsade, studio i tagi filmow z serwisow dooplay po zmianie motywu (2026-10).
|
||||||
|
|
||||||
|
Mangoporn, streamporn.li i pandamovies przeszly na motyw "vid": obsada tylko w JSON-LD,
|
||||||
|
studio i rok w chipach bez rel="tag". Parser tego nie czytal, wiec filmy dodane od ok.
|
||||||
|
5.10 przyszly bez obsady i czesto bez studia (cichy fail parsera, ingest konczyl sie
|
||||||
|
"errors: 0"). Skrypt pobiera ich strony na nowo i puszcza przez resolve_movie, ktory przy
|
||||||
|
znanym refie tylko uzupelnia puste pola i dopina obsade/tagi. Strony przez proxy, bo
|
||||||
|
mangoporn z VPS bywa niedostepny (timeout, blad TLS).
|
||||||
|
|
||||||
|
python3 /srv/scripts/backfill_dooplay_metadata.py # dry-run: tylko liczy
|
||||||
|
python3 /srv/scripts/backfill_dooplay_metadata.py --apply [--since 2026-10-01]
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import functools
|
||||||
|
import logging
|
||||||
|
|
||||||
|
import app.connectors.dooplay as dp
|
||||||
|
from app.config import get_settings
|
||||||
|
|
||||||
|
dp.browser_get = functools.partial(dp.browser_get, proxy=get_settings().brightdata_proxy_url)
|
||||||
|
|
||||||
|
from sqlalchemy import text # noqa: E402
|
||||||
|
|
||||||
|
from app.db import session_scope # noqa: E402
|
||||||
|
from app.normalize.movies import normalize_movie # noqa: E402
|
||||||
|
from app.resolve.movie_resolver import resolve_movie # noqa: E402
|
||||||
|
|
||||||
|
_CONNECTORS = {
|
||||||
|
"mangoporn": dp.MangopornConnector,
|
||||||
|
"streampornvip": dp.StreampornVipConnector,
|
||||||
|
"pandamovies": dp.PandamoviesConnector,
|
||||||
|
}
|
||||||
|
|
||||||
|
_REFS_SQL = """
|
||||||
|
SELECT r.movie_id, r.external_id, r.url, so.id AS source_id, so.name
|
||||||
|
FROM movie_external_refs r
|
||||||
|
JOIN sources so ON so.id = r.source_id
|
||||||
|
JOIN movies m ON m.id = r.movie_id
|
||||||
|
WHERE so.name = ANY(:names) AND m.created_at >= :since AND r.url IS NOT NULL
|
||||||
|
AND (m.studio_id IS NULL
|
||||||
|
OR NOT EXISTS (SELECT 1 FROM movie_performers p WHERE p.movie_id = m.id))
|
||||||
|
ORDER BY m.created_at DESC
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
ap = argparse.ArgumentParser()
|
||||||
|
ap.add_argument("--apply", action="store_true")
|
||||||
|
ap.add_argument("--since", default="2026-10-01")
|
||||||
|
args = ap.parse_args()
|
||||||
|
logging.basicConfig(level=logging.WARNING)
|
||||||
|
|
||||||
|
with session_scope() as s:
|
||||||
|
refs = s.execute(text(_REFS_SQL), {"names": list(_CONNECTORS), "since": args.since}).all()
|
||||||
|
print(f"refow do uzupelnienia: {len(refs)}", flush=True)
|
||||||
|
if not args.apply:
|
||||||
|
return
|
||||||
|
|
||||||
|
conns = {name: cls() for name, cls in _CONNECTORS.items()}
|
||||||
|
done = failed = with_cast = 0
|
||||||
|
for n, ref in enumerate(refs, 1):
|
||||||
|
try:
|
||||||
|
raw = conns[ref.name]._fetch_detail(ref.url)
|
||||||
|
except Exception as e: # noqa: BLE001
|
||||||
|
failed += 1
|
||||||
|
print(f" blad {ref.name} {ref.url}: {str(e)[:100]}", flush=True)
|
||||||
|
continue
|
||||||
|
if raw is None:
|
||||||
|
failed += 1
|
||||||
|
continue
|
||||||
|
norm = normalize_movie(raw)
|
||||||
|
if norm.external_id != ref.external_id:
|
||||||
|
failed += 1
|
||||||
|
print(f" inny id {ref.external_id} -> {norm.external_id}, pomijam", flush=True)
|
||||||
|
continue
|
||||||
|
with session_scope() as s:
|
||||||
|
resolve_movie(s, norm=norm, source_id=ref.source_id)
|
||||||
|
done += 1
|
||||||
|
with_cast += bool(raw.performers)
|
||||||
|
if n % 25 == 0:
|
||||||
|
print(f"... {n}/{len(refs)}, uzupelnione {done}, z obsada {with_cast}, bledy {failed}", flush=True)
|
||||||
|
print(f"uzupelnione: {done}, z obsada: {with_cast}, bledy: {failed}", flush=True)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
98
scripts/fix_dooplay_junk_titles.py
Normal file
98
scripts/fix_dooplay_junk_titles.py
Normal file
|
|
@ -0,0 +1,98 @@
|
||||||
|
"""Czysci tytuly filmow z naglowka nowego motywu dooplay (10.2026) i scala duplikaty.
|
||||||
|
|
||||||
|
Motyw mangoporn/streamporn.li wstawia do h1 "Watch TYTUL 2026 by STUDIO Porn Movie Online
|
||||||
|
Free" albo "Now Playing: TYTUL". Parser od 2026-10-15 bierze tytul z okruszkow, ten skrypt
|
||||||
|
poprawia to, co weszlo wczesniej:
|
||||||
|
|
||||||
|
- "Now Playing: X" -> "X" (zawsze naglowek, nigdy prawdziwy tytul),
|
||||||
|
- "Watch X RRRR by S Porn Movie Online Free" -> "X" (tylko z tym ogonem; "Watch Me Eat My
|
||||||
|
Creampie 5" to prawdziwy tytul i zostaje),
|
||||||
|
- "Play X" zostaje, bo bywa tytulem ("Play Time!").
|
||||||
|
|
||||||
|
Jesli po poprawce istnieje juz inny film o tym samym title_normalized i zgodnym roku
|
||||||
|
(rowny albo jeden pusty), poprawiany film jest do niego dolaczany (merge_movies, starszy
|
||||||
|
zostaje). Wiecej niz jedno trafienie = tylko poprawka tytulu, bez scalania.
|
||||||
|
|
||||||
|
python scripts/fix_dooplay_junk_titles.py # podglad
|
||||||
|
python scripts/fix_dooplay_junk_titles.py --apply
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import logging
|
||||||
|
import re
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
from app.db import session_scope
|
||||||
|
from app.models.movie import Movie
|
||||||
|
from app.normalize.text import normalize
|
||||||
|
from app.resolve.movie_merge import merge_movies
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
_NOW_PLAYING_RE = re.compile(r"^Now\s+Playing:\s*(?P<title>.+?)\s*$", re.IGNORECASE | re.DOTALL)
|
||||||
|
_WATCH_TAIL_RE = re.compile(
|
||||||
|
r"^Watch\s+(?P<title>.+?)(?:\s+(?:19|20)\d{2}\s+by\s+.+?)?\s+Porn\s+Movie\s+Online\s+Free\s*$",
|
||||||
|
re.IGNORECASE | re.DOTALL,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def clean_title(title: str) -> str | None:
|
||||||
|
for rx in (_NOW_PLAYING_RE, _WATCH_TAIL_RE):
|
||||||
|
m = rx.match(title)
|
||||||
|
if m and m.group("title").strip():
|
||||||
|
return m.group("title").strip()
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def run(apply: bool) -> dict[str, int]:
|
||||||
|
counters = {"candidates": 0, "retitled": 0, "merged": 0, "ambiguous": 0}
|
||||||
|
with session_scope() as session:
|
||||||
|
rows = session.execute(
|
||||||
|
sa.select(Movie.id, Movie.title, Movie.release_year).where(
|
||||||
|
sa.or_(
|
||||||
|
Movie.title.op("~*")(r"^now playing:"),
|
||||||
|
Movie.title.op("~*")(r"^watch .+ porn movie online free\s*$"),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
).all()
|
||||||
|
for movie_id, title, year in rows:
|
||||||
|
new_title = clean_title(title)
|
||||||
|
if not new_title:
|
||||||
|
continue
|
||||||
|
counters["candidates"] += 1
|
||||||
|
norm = normalize(new_title)
|
||||||
|
twins = session.execute(
|
||||||
|
sa.select(Movie.id, Movie.release_year, Movie.created_at)
|
||||||
|
.where(Movie.title_normalized == norm, Movie.id != movie_id)
|
||||||
|
.order_by(Movie.created_at)
|
||||||
|
).all()
|
||||||
|
twins = [t for t in twins if t.release_year is None or year is None or t.release_year == year]
|
||||||
|
print(f"{title[:70]!r} -> {new_title[:60]!r} twins={len(twins)}")
|
||||||
|
if not apply:
|
||||||
|
continue
|
||||||
|
movie = session.get(Movie, movie_id)
|
||||||
|
movie.title = new_title[:512]
|
||||||
|
movie.title_normalized = norm[:512]
|
||||||
|
counters["retitled"] += 1
|
||||||
|
if len(twins) == 1:
|
||||||
|
merge_movies(session, keep_id=twins[0].id, drop_id=movie_id, resolved_by="fix_dooplay_junk_titles")
|
||||||
|
counters["merged"] += 1
|
||||||
|
elif len(twins) > 1:
|
||||||
|
counters["ambiguous"] += 1
|
||||||
|
session.commit()
|
||||||
|
return counters
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
logging.basicConfig(level=logging.INFO)
|
||||||
|
p = argparse.ArgumentParser()
|
||||||
|
p.add_argument("--apply", action="store_true")
|
||||||
|
args = p.parse_args()
|
||||||
|
print(run(args.apply))
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
110
scripts/fix_hentaimama_titles_thumbs.py
Normal file
110
scripts/fix_hentaimama_titles_thumbs.py
Normal file
|
|
@ -0,0 +1,110 @@
|
||||||
|
"""Hentaimama: ogon w tytulach i brakujace miniatury (2026-10-15).
|
||||||
|
|
||||||
|
1. Tytuly: od ~31.08.2026 og:title mial ogon "TYTUL - Watch Online Free in HD - Hentaimama"
|
||||||
|
i tak weszlo ok. 100 scen. Scraper juz go ucina, tu poprawiamy istniejace (title i
|
||||||
|
title_normalized).
|
||||||
|
2. Miniatury: ok. 2100 zrodel z pierwszego przebiegu (12.08) nie ma thumbnail_url, a te sceny
|
||||||
|
nie maja innego zrodla z obrazkiem. Strony maja og:image, pobieramy je po kolei.
|
||||||
|
|
||||||
|
python scripts/fix_hentaimama_titles_thumbs.py # podglad
|
||||||
|
python scripts/fix_hentaimama_titles_thumbs.py --apply [--limit N] [--sleep 0.5]
|
||||||
|
|
||||||
|
Dlugie, wiec na prodzie przez `docker compose run -d --rm worker python3 scripts/...`
|
||||||
|
(z pliku, nie przez stdin).
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import html as html_mod
|
||||||
|
import logging
|
||||||
|
import time
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
from app.connectors.direct_scrapers.hentaimama import _OG_IMAGE_RE, _TITLE_TAIL_RE
|
||||||
|
from app.db import session_scope
|
||||||
|
from app.extractors._fetch import fetch_tube_html
|
||||||
|
from app.normalize.text import normalize
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def fix_titles(apply: bool) -> int:
|
||||||
|
n = 0
|
||||||
|
with session_scope() as session:
|
||||||
|
rows = session.execute(
|
||||||
|
sa.text("SELECT id, title FROM scenes WHERE title ~* 'hentaimama\\s*$'")
|
||||||
|
).all()
|
||||||
|
for scene_id, title in rows:
|
||||||
|
new = _TITLE_TAIL_RE.sub("", title).strip()
|
||||||
|
if not new or new == title:
|
||||||
|
continue
|
||||||
|
n += 1
|
||||||
|
if n <= 5:
|
||||||
|
print(f"{title!r} -> {new!r}")
|
||||||
|
if apply:
|
||||||
|
session.execute(
|
||||||
|
sa.text("UPDATE scenes SET title=:t, title_normalized=:n WHERE id=:id"),
|
||||||
|
{"t": new[:512], "n": normalize(new)[:512], "id": scene_id},
|
||||||
|
)
|
||||||
|
if apply:
|
||||||
|
session.commit()
|
||||||
|
return n
|
||||||
|
|
||||||
|
|
||||||
|
def fix_thumbs(apply: bool, limit: int, sleep: float) -> dict[str, int]:
|
||||||
|
c = {"checked": 0, "filled": 0, "no_image": 0, "fetch_fail": 0}
|
||||||
|
with session_scope() as session:
|
||||||
|
rows = session.execute(
|
||||||
|
sa.text(
|
||||||
|
"SELECT id, page_url FROM playback_sources WHERE origin='tube:hentaimama' "
|
||||||
|
"AND thumbnail_url IS NULL AND dead_at IS NULL ORDER BY id LIMIT :lim"
|
||||||
|
),
|
||||||
|
{"lim": limit},
|
||||||
|
).all()
|
||||||
|
for pb_id, page_url in rows:
|
||||||
|
c["checked"] += 1
|
||||||
|
try:
|
||||||
|
page = fetch_tube_html(page_url, timeout=30.0)
|
||||||
|
except Exception as e:
|
||||||
|
c["fetch_fail"] += 1
|
||||||
|
log.info("fetch %s: %s", page_url, e)
|
||||||
|
time.sleep(sleep)
|
||||||
|
continue
|
||||||
|
m = _OG_IMAGE_RE.search(page)
|
||||||
|
if not m:
|
||||||
|
c["no_image"] += 1
|
||||||
|
else:
|
||||||
|
img = html_mod.unescape(m.group(1)).strip()
|
||||||
|
if apply:
|
||||||
|
with session_scope() as session:
|
||||||
|
session.execute(
|
||||||
|
sa.text("UPDATE playback_sources SET thumbnail_url=:u WHERE id=:id"),
|
||||||
|
{"u": img[:2048], "id": pb_id},
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
c["filled"] += 1
|
||||||
|
if c["filled"] <= 3:
|
||||||
|
print(page_url, "->", img)
|
||||||
|
if c["checked"] % 100 == 0:
|
||||||
|
log.info("thumbs %s", c)
|
||||||
|
time.sleep(sleep)
|
||||||
|
return c
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
logging.basicConfig(level=logging.INFO)
|
||||||
|
p = argparse.ArgumentParser()
|
||||||
|
p.add_argument("--apply", action="store_true")
|
||||||
|
p.add_argument("--limit", type=int, default=5)
|
||||||
|
p.add_argument("--sleep", type=float, default=0.5)
|
||||||
|
p.add_argument("--skip-titles", action="store_true")
|
||||||
|
args = p.parse_args()
|
||||||
|
if not args.skip_titles:
|
||||||
|
print("titles:", fix_titles(args.apply))
|
||||||
|
print("thumbs:", fix_thumbs(args.apply, args.limit, args.sleep))
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
94
scripts/fix_latestpornvideo_thumbs.py
Normal file
94
scripts/fix_latestpornvideo_thumbs.py
Normal file
|
|
@ -0,0 +1,94 @@
|
||||||
|
"""latestpornvideo: odzyskanie miniatur, ktore nie dzialaja (2026-10-15).
|
||||||
|
|
||||||
|
Scraper od dzis sam poprawia nowe wpisy (`better_thumb`), tu poprawiamy istniejace:
|
||||||
|
- brak miniatury,
|
||||||
|
- `pixhost.to/show/...` (strona podgladu zamiast obrazka),
|
||||||
|
- obrazek pixhost usuniety (przekierowanie na /image/removed.png),
|
||||||
|
- z `--imgds`: miniatury imgds.net (dawne ticdn.net), ktore daja 404.
|
||||||
|
|
||||||
|
W kazdym z tych przypadkow zapasem jest plakat odtwarzacza lulu z wpisu.
|
||||||
|
|
||||||
|
python scripts/fix_latestpornvideo_thumbs.py # podglad 20 wierszy
|
||||||
|
python scripts/fix_latestpornvideo_thumbs.py --apply --limit 20000 [--imgds]
|
||||||
|
|
||||||
|
Dlugie, na prodzie przez `docker compose run -d --rm worker python3 /srv/scripts/...`.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import logging
|
||||||
|
import time
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
from app.connectors.direct_scrapers.latestpornvideo import better_thumb
|
||||||
|
from app.db import session_scope
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def _imgds_missing(url: str) -> bool:
|
||||||
|
try:
|
||||||
|
r = httpx.head(url, headers={"User-Agent": "Mozilla/5.0"}, timeout=10.0, follow_redirects=True)
|
||||||
|
except Exception:
|
||||||
|
return False
|
||||||
|
return r.status_code == 404
|
||||||
|
|
||||||
|
|
||||||
|
def run(apply: bool, limit: int, imgds: bool, sleep: float) -> dict[str, int]:
|
||||||
|
c = {"checked": 0, "changed": 0, "unchanged": 0, "lost": 0}
|
||||||
|
cond = (
|
||||||
|
"thumbnail_url IS NULL OR thumbnail_url LIKE '%pixhost%'"
|
||||||
|
+ (" OR thumbnail_url LIKE 'https://imgds.net/snaps/%'" if imgds else "")
|
||||||
|
)
|
||||||
|
with session_scope() as s:
|
||||||
|
rows = s.execute(
|
||||||
|
sa.text(
|
||||||
|
"SELECT id, page_url, thumbnail_url FROM playback_sources "
|
||||||
|
"WHERE origin='tube:latestpornvideocom' AND dead_at IS NULL "
|
||||||
|
f"AND ({cond}) ORDER BY id LIMIT :lim"
|
||||||
|
),
|
||||||
|
{"lim": limit},
|
||||||
|
).all()
|
||||||
|
for pb_id, page_url, thumb in rows:
|
||||||
|
c["checked"] += 1
|
||||||
|
candidate = thumb
|
||||||
|
if thumb and thumb.startswith("https://imgds.net/snaps/") and _imgds_missing(thumb):
|
||||||
|
candidate = None
|
||||||
|
new = better_thumb(candidate, page_url, timeout=30.0)
|
||||||
|
if new == thumb:
|
||||||
|
c["unchanged"] += 1
|
||||||
|
elif not new:
|
||||||
|
c["lost"] += 1
|
||||||
|
else:
|
||||||
|
c["changed"] += 1
|
||||||
|
if c["changed"] <= 5:
|
||||||
|
print(f"{page_url}: {thumb} -> {new}")
|
||||||
|
if apply:
|
||||||
|
with session_scope() as s:
|
||||||
|
s.execute(
|
||||||
|
sa.text("UPDATE playback_sources SET thumbnail_url=:u WHERE id=:id"),
|
||||||
|
{"u": new[:2048], "id": pb_id},
|
||||||
|
)
|
||||||
|
s.commit()
|
||||||
|
if c["checked"] % 200 == 0:
|
||||||
|
log.info("lpv thumbs %s", c)
|
||||||
|
time.sleep(sleep)
|
||||||
|
return c
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
logging.basicConfig(level=logging.INFO)
|
||||||
|
p = argparse.ArgumentParser()
|
||||||
|
p.add_argument("--apply", action="store_true")
|
||||||
|
p.add_argument("--limit", type=int, default=20)
|
||||||
|
p.add_argument("--imgds", action="store_true")
|
||||||
|
p.add_argument("--sleep", type=float, default=0.2)
|
||||||
|
args = p.parse_args()
|
||||||
|
print(run(args.apply, args.limit, args.imgds, args.sleep))
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
20
scripts/ingest_movies_backfill.py
Normal file
20
scripts/ingest_movies_backfill.py
Normal file
|
|
@ -0,0 +1,20 @@
|
||||||
|
"""Pelne zaciagniecie filmow z jednego zrodla (bez delty), np. po przerwie w ingescie.
|
||||||
|
|
||||||
|
Codzienny _job_movie_ingest idzie w trybie przyrostowym (dooplay: 3 strony listy), wiec
|
||||||
|
po kilku tygodniach przestoju zrodla zaleglosci same sie nie nadrobia. Pelny tryb dooplay
|
||||||
|
czyta do 50 stron. Uzycie (z pliku, w osobnym kontenerze):
|
||||||
|
docker compose run -d --name X worker python3 /srv/scripts/ingest_movies_backfill.py pandamovies [limit]
|
||||||
|
"""
|
||||||
|
import logging
|
||||||
|
import sys
|
||||||
|
|
||||||
|
from app.connectors import get_movie_connectors
|
||||||
|
from app.ingest import ingest_movies_from_connector
|
||||||
|
|
||||||
|
logging.basicConfig(level=logging.INFO)
|
||||||
|
name = sys.argv[1]
|
||||||
|
limit = int(sys.argv[2]) if len(sys.argv) > 2 else None
|
||||||
|
connectors = dict(get_movie_connectors())
|
||||||
|
if name not in connectors:
|
||||||
|
sys.exit(f"nie ma konektora filmow {name!r}, sa: {sorted(connectors)}")
|
||||||
|
print(ingest_movies_from_connector(connectors[name](), limit=limit, use_delta=False))
|
||||||
14
scripts/ingest_xtapes_backfill.py
Normal file
14
scripts/ingest_xtapes_backfill.py
Normal file
|
|
@ -0,0 +1,14 @@
|
||||||
|
"""Pierwsze zaciagniecie filmow z xtapes.la (2026-10-15): pelny tryb, do 20 stron REST.
|
||||||
|
|
||||||
|
Dalej filmy dochodza codziennie przez _job_movie_ingest (tryb przyrostowy, after=).
|
||||||
|
docker compose run -d --rm worker python3 /srv/scripts/ingest_xtapes_backfill.py [limit]
|
||||||
|
"""
|
||||||
|
import logging
|
||||||
|
import sys
|
||||||
|
|
||||||
|
from app.connectors.xtapes import XtapesConnector
|
||||||
|
from app.ingest import ingest_movies_from_connector
|
||||||
|
|
||||||
|
logging.basicConfig(level=logging.INFO)
|
||||||
|
limit = int(sys.argv[1]) if len(sys.argv) > 1 else None
|
||||||
|
print(ingest_movies_from_connector(XtapesConnector(), limit=limit, use_delta=False))
|
||||||
79
scripts/merge_hentai_episode_dupes.py
Normal file
79
scripts/merge_hentai_episode_dupes.py
Normal file
|
|
@ -0,0 +1,79 @@
|
||||||
|
"""Scala te same odcinki hentai z hentaimama i xanimeporn (2026-10-15).
|
||||||
|
|
||||||
|
Tytuly hentaimama mialy do 15.10 ogon "- Watch Online Free in HD - Hentaimama", a xanime
|
||||||
|
dokleja "[Sub-ENG]", wiec ten sam odcinek byl dwiema scenami. Para = scena tylko z
|
||||||
|
hentaimama i scena tylko z xanimeporn o tym samym title_normalized po zdjeciu
|
||||||
|
"sub eng"/"raw"/"uncensored" z konca. Scalamy wylacznie pary jeden do jednego
|
||||||
|
(scena hentaimama zostaje, ma dlugosc i miniature).
|
||||||
|
|
||||||
|
python scripts/merge_hentai_episode_dupes.py # podglad
|
||||||
|
python scripts/merge_hentai_episode_dupes.py --apply
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import logging
|
||||||
|
from collections import defaultdict
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
from app.db import session_scope
|
||||||
|
from app.resolve.scene_merge import merge_scenes
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
_PAIRS_SQL = """
|
||||||
|
with x as (
|
||||||
|
select s.id, regexp_replace(s.title_normalized, '\\s+(sub eng|eng sub|english sub|raw|uncensored)$', '') k
|
||||||
|
from scenes s
|
||||||
|
where exists (select 1 from playback_sources p where p.scene_id = s.id and p.origin = 'tube:xanimeporn')
|
||||||
|
and not exists (select 1 from playback_sources p where p.scene_id = s.id and p.origin = 'tube:hentaimama')
|
||||||
|
), h as (
|
||||||
|
select s.id, s.title_normalized k
|
||||||
|
from scenes s
|
||||||
|
where exists (select 1 from playback_sources p where p.scene_id = s.id and p.origin = 'tube:hentaimama')
|
||||||
|
and not exists (select 1 from playback_sources p where p.scene_id = s.id and p.origin = 'tube:xanimeporn')
|
||||||
|
)
|
||||||
|
select h.id as keep_id, x.id as drop_id, h.k from x join h on h.k = x.k
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def run(apply: bool) -> dict[str, int]:
|
||||||
|
with session_scope() as s:
|
||||||
|
rows = s.execute(sa.text(_PAIRS_SQL)).all()
|
||||||
|
by_keep: dict = defaultdict(list)
|
||||||
|
by_drop: dict = defaultdict(list)
|
||||||
|
for keep_id, drop_id, _k in rows:
|
||||||
|
by_keep[keep_id].append(drop_id)
|
||||||
|
by_drop[drop_id].append(keep_id)
|
||||||
|
pairs = [(k, d, t) for k, d, t in rows if len(by_keep[k]) == 1 and len(by_drop[d]) == 1]
|
||||||
|
c = {"candidates": len(rows), "one_to_one": len(pairs), "merged": 0, "errors": 0}
|
||||||
|
for keep_id, drop_id, title in pairs[:5]:
|
||||||
|
print(f"{title!r}: keep {keep_id} <- {drop_id}")
|
||||||
|
if not apply:
|
||||||
|
return c
|
||||||
|
for keep_id, drop_id, _t in pairs:
|
||||||
|
try:
|
||||||
|
with session_scope() as s:
|
||||||
|
merge_scenes(s, keep_id=keep_id, drop_id=drop_id, resolved_by="merge_hentai_episode_dupes")
|
||||||
|
s.commit()
|
||||||
|
c["merged"] += 1
|
||||||
|
except Exception as e:
|
||||||
|
c["errors"] += 1
|
||||||
|
log.warning("merge %s <- %s padl: %s", keep_id, drop_id, e)
|
||||||
|
if c["merged"] % 200 == 0 and c["merged"]:
|
||||||
|
log.info("hentai dupes %s", c)
|
||||||
|
return c
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
logging.basicConfig(level=logging.INFO)
|
||||||
|
p = argparse.ArgumentParser()
|
||||||
|
p.add_argument("--apply", action="store_true")
|
||||||
|
args = p.parse_args()
|
||||||
|
print(run(args.apply))
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
138
scripts/merge_twin_studios.py
Normal file
138
scripts/merge_twin_studios.py
Normal file
|
|
@ -0,0 +1,138 @@
|
||||||
|
"""Scala blizniacze studia: te same nazwy po usunieciu spacji ("EvilAngel" = "Evil Angel",
|
||||||
|
"Let's Post It" = "Lets PostIt"). 2026-10-16: ok. 2 tys. grup z tubow, TPDB i stashdb;
|
||||||
|
rozdzielaly katalog studia na kilka stron i psuly dopasowanie filmow (zgloszenie f8c9486e).
|
||||||
|
|
||||||
|
Zostaje studio z najwiekszym katalogiem (scene_count + filmy), potem z ref TPDB/stashdb,
|
||||||
|
potem zapis ze spacja, potem starsze. Do niego przechodza sceny, filmy, refy zewnetrzne,
|
||||||
|
aliasy, studia podrzedne, ulubione i ukrycia uzytkownikow; nazwa scalanego zostaje
|
||||||
|
aliasem. Ulubione na obu: zostaje jeden wpis z wczesniejszym created_at i pozniejszym
|
||||||
|
last_seen_at (bez zalewu "+N"). Kazda grupa w osobnej transakcji.
|
||||||
|
|
||||||
|
python3 /srv/scripts/merge_twin_studios.py # dry-run, plan i liczby
|
||||||
|
python3 /srv/scripts/merge_twin_studios.py --apply
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
|
||||||
|
from sqlalchemy import text
|
||||||
|
|
||||||
|
from app.db import session_scope
|
||||||
|
|
||||||
|
_GROUPS_SQL = """
|
||||||
|
WITH s AS (
|
||||||
|
SELECT id, replace(name_normalized, ' ', '') AS k FROM studios
|
||||||
|
WHERE length(replace(name_normalized, ' ', '')) >= 4
|
||||||
|
)
|
||||||
|
SELECT k, array_agg(id) FROM s
|
||||||
|
WHERE k IN (SELECT k FROM s GROUP BY k HAVING count(*) > 1)
|
||||||
|
GROUP BY k ORDER BY k
|
||||||
|
"""
|
||||||
|
|
||||||
|
_MEMBERS_SQL = """
|
||||||
|
SELECT st.id, st.name, st.name_normalized, st.parent_studio_id, st.scene_count,
|
||||||
|
(SELECT count(*) FROM movies m WHERE m.studio_id = st.id) AS movies,
|
||||||
|
EXISTS (SELECT 1 FROM studio_external_refs r JOIN sources so ON so.id = r.source_id
|
||||||
|
WHERE r.studio_id = st.id AND so.name IN ('tpdb', 'stashdb')) AS canonical,
|
||||||
|
st.created_at
|
||||||
|
FROM studios st WHERE st.id = ANY(:ids)
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def _rank(m) -> tuple:
|
||||||
|
return (m.scene_count + m.movies, m.canonical, " " in m.name_normalized, -m.created_at.timestamp())
|
||||||
|
|
||||||
|
|
||||||
|
def _merge(s, keep, drop) -> int:
|
||||||
|
p = {"keep": keep.id, "drop": drop.id}
|
||||||
|
s.execute(text(
|
||||||
|
"UPDATE studios SET parent_studio_id = :keep WHERE parent_studio_id = :drop AND id <> :keep"
|
||||||
|
), p)
|
||||||
|
# Rodzic zatrzymanego: nie moze wskazywac na siebie; brakujacy dziedziczy po scalanym.
|
||||||
|
keep_parent = keep.parent_studio_id
|
||||||
|
if keep_parent == drop.id or keep_parent is None:
|
||||||
|
keep_parent = drop.parent_studio_id if drop.parent_studio_id not in (keep.id, drop.id) else None
|
||||||
|
s.execute(text("""
|
||||||
|
UPDATE studios k SET parent_studio_id = :parent,
|
||||||
|
network = coalesce(k.network, d.network),
|
||||||
|
homepage_url = coalesce(k.homepage_url, d.homepage_url),
|
||||||
|
scene_count = k.scene_count + d.scene_count,
|
||||||
|
updated_at = now()
|
||||||
|
FROM studios d WHERE k.id = :keep AND d.id = :drop
|
||||||
|
"""), {**p, "parent": keep_parent})
|
||||||
|
|
||||||
|
moved = s.execute(text("UPDATE scenes SET studio_id = :keep WHERE studio_id = :drop"), p).rowcount
|
||||||
|
s.execute(text("UPDATE movies SET studio_id = :keep WHERE studio_id = :drop"), p)
|
||||||
|
s.execute(text("UPDATE studio_external_refs SET studio_id = :keep WHERE studio_id = :drop"), p)
|
||||||
|
|
||||||
|
s.execute(text("""
|
||||||
|
DELETE FROM studio_aliases a WHERE a.studio_id = :drop AND EXISTS (
|
||||||
|
SELECT 1 FROM studio_aliases b WHERE b.studio_id = :keep AND b.alias_normalized = a.alias_normalized)
|
||||||
|
"""), p)
|
||||||
|
s.execute(text("UPDATE studio_aliases SET studio_id = :keep WHERE studio_id = :drop"), p)
|
||||||
|
s.execute(text("""
|
||||||
|
INSERT INTO studio_aliases (studio_id, alias, alias_normalized)
|
||||||
|
SELECT :keep, d.name, d.name_normalized FROM studios d, studios k
|
||||||
|
WHERE d.id = :drop AND k.id = :keep AND d.name_normalized <> k.name_normalized
|
||||||
|
AND NOT EXISTS (SELECT 1 FROM studio_aliases b
|
||||||
|
WHERE b.studio_id = :keep AND b.alias_normalized = d.name_normalized)
|
||||||
|
"""), p)
|
||||||
|
|
||||||
|
s.execute(text("""
|
||||||
|
UPDATE favorite_studios k SET created_at = least(k.created_at, d.created_at),
|
||||||
|
last_seen_at = greatest(k.last_seen_at, d.last_seen_at)
|
||||||
|
FROM favorite_studios d
|
||||||
|
WHERE k.studio_id = :keep AND d.studio_id = :drop AND d.device_id = k.device_id
|
||||||
|
"""), p)
|
||||||
|
s.execute(text("""
|
||||||
|
DELETE FROM favorite_studios d WHERE d.studio_id = :drop AND EXISTS (
|
||||||
|
SELECT 1 FROM favorite_studios k WHERE k.studio_id = :keep AND k.device_id = d.device_id)
|
||||||
|
"""), p)
|
||||||
|
s.execute(text("UPDATE favorite_studios SET studio_id = :keep WHERE studio_id = :drop"), p)
|
||||||
|
s.execute(text("""
|
||||||
|
DELETE FROM blacklisted_studios d WHERE d.studio_id = :drop AND EXISTS (
|
||||||
|
SELECT 1 FROM blacklisted_studios k WHERE k.studio_id = :keep AND k.device_id = d.device_id)
|
||||||
|
"""), p)
|
||||||
|
s.execute(text("UPDATE blacklisted_studios SET studio_id = :keep WHERE studio_id = :drop"), p)
|
||||||
|
|
||||||
|
s.execute(text("DELETE FROM studios WHERE id = :drop"), p)
|
||||||
|
return moved
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
ap = argparse.ArgumentParser()
|
||||||
|
ap.add_argument("--apply", action="store_true")
|
||||||
|
ap.add_argument("--limit", type=int, default=0)
|
||||||
|
args = ap.parse_args()
|
||||||
|
|
||||||
|
with session_scope() as s:
|
||||||
|
groups = s.execute(text(_GROUPS_SQL)).all()
|
||||||
|
if args.limit:
|
||||||
|
groups = groups[: args.limit]
|
||||||
|
print(f"grup: {len(groups)}", flush=True)
|
||||||
|
|
||||||
|
dropped = scenes = 0
|
||||||
|
for n, (key, ids) in enumerate(groups, 1):
|
||||||
|
with session_scope() as s:
|
||||||
|
members = sorted(s.execute(text(_MEMBERS_SQL), {"ids": list(ids)}).all(), key=_rank, reverse=True)
|
||||||
|
if len(members) < 2:
|
||||||
|
continue
|
||||||
|
keep, rest = members[0], members[1:]
|
||||||
|
print(
|
||||||
|
f"ZOSTAJE {keep.name!r} ({keep.scene_count}/{keep.movies}) <= "
|
||||||
|
+ ", ".join(f"{m.name!r} ({m.scene_count}/{m.movies})" for m in rest),
|
||||||
|
flush=True,
|
||||||
|
)
|
||||||
|
if args.apply:
|
||||||
|
for d in rest:
|
||||||
|
# swiezo z bazy: poprzednie scalenie w tej grupie moglo zmienic rodzica
|
||||||
|
fresh = {r.id: r for r in s.execute(text(_MEMBERS_SQL), {"ids": [keep.id, d.id]}).all()}
|
||||||
|
scenes += _merge(s, fresh[keep.id], fresh[d.id])
|
||||||
|
dropped += len(rest)
|
||||||
|
if n % 200 == 0:
|
||||||
|
print(f"... {n}/{len(groups)}, studiow scalonych: {dropped}, scen przepietych: {scenes}", flush=True)
|
||||||
|
print(f"{'scalono' if args.apply else 'do scalenia'}: {dropped} studiow, scen przepietych: {scenes}", flush=True)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
176
scripts/porntrex_duration_walk.py
Normal file
176
scripts/porntrex_duration_walk.py
Normal file
|
|
@ -0,0 +1,176 @@
|
||||||
|
"""porntrex: dolicza długość scenom bez niej z listingu `/latest-updates/N/`.
|
||||||
|
|
||||||
|
Strona szczegółów porntrex nie podaje własnej długości (tylko długości filmów z paska
|
||||||
|
obok), więc backfill ze strony (`duration_backfill.py`) tu nie działa. Kafelek listingu
|
||||||
|
ją pokazuje (`class="durations"`), 85 kafelków na stronę, katalog ma ok. 8300 stron
|
||||||
|
(strona 15000 daje 404). Przebieg idzie po listingu od najnowszych i dla każdego kafelka
|
||||||
|
ustawia `scenes.duration_sec`, jeśli jest NULL. Nic nie nadpisuje.
|
||||||
|
|
||||||
|
Adresy: ścieżka search zapisuje `https://www.porntrex.com/video/<id>/<slug>`, browse
|
||||||
|
`https://porntrex.com/video/<id>/<slug>` (bez www), dlatego dopasowanie po obu wariantach.
|
||||||
|
|
||||||
|
Uruchomienie (długi przebieg, nie przez `exec`, patrz pamięć o restartach workera):
|
||||||
|
docker compose run -d --rm worker python3 scripts/porntrex_duration_walk.py --start 1
|
||||||
|
docker compose run --rm worker python3 scripts/porntrex_duration_walk.py --start 1 --end 20 --dry-run
|
||||||
|
Postęp w `--state-file` (domyślnie /tmp/porntrex_duration_walk.json), wznowienie od
|
||||||
|
zapisanej strony przy braku `--start`.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
|
||||||
|
from sqlalchemy import text
|
||||||
|
|
||||||
|
from app.connectors.direct_scrapers.porntrex_browse import _A_RE, _DUR_RE, _parse_duration
|
||||||
|
from app.db import session_scope
|
||||||
|
from app.extractors import browser_get
|
||||||
|
|
||||||
|
log = logging.getLogger("porntrex_duration_walk")
|
||||||
|
_LISTING = "https://www.porntrex.com/latest-updates/{page}/"
|
||||||
|
_ORIGIN = "tube:porntrexcom"
|
||||||
|
|
||||||
|
|
||||||
|
def parse_cards(html: str) -> dict[str, int]:
|
||||||
|
"""Kafelki listingu → {url bez www i bez końcowego '/': sekundy}."""
|
||||||
|
out: dict[str, int] = {}
|
||||||
|
anchors = list(_A_RE.finditer(html))
|
||||||
|
for idx, m in enumerate(anchors):
|
||||||
|
url = m.group("url").replace("://www.", "://").rstrip("/")
|
||||||
|
if url in out:
|
||||||
|
continue
|
||||||
|
end = anchors[idx + 1].start() if idx + 1 < len(anchors) else m.end() + 700
|
||||||
|
dm = _DUR_RE.search(html[m.start():end])
|
||||||
|
sec = _parse_duration(dm.group(1) if dm else None)
|
||||||
|
if sec:
|
||||||
|
out[url] = sec
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def variants(url: str) -> list[str]:
|
||||||
|
bare = url.replace("://www.", "://").rstrip("/")
|
||||||
|
www = bare.replace("https://", "https://www.", 1)
|
||||||
|
return [bare, bare + "/", www, www + "/"]
|
||||||
|
|
||||||
|
|
||||||
|
def apply(cards: dict[str, int], *, dry_run: bool) -> dict[str, int]:
|
||||||
|
"""Zwraca liczniki: matched (źródła w bazie), filled (ustawione), had (już miały)."""
|
||||||
|
counters = {"matched": 0, "filled": 0, "had": 0}
|
||||||
|
if not cards:
|
||||||
|
return counters
|
||||||
|
lookup: dict[str, int] = {}
|
||||||
|
for url, sec in cards.items():
|
||||||
|
for v in variants(url):
|
||||||
|
lookup[v] = sec
|
||||||
|
with session_scope() as s:
|
||||||
|
rows = s.execute(
|
||||||
|
text(
|
||||||
|
"""
|
||||||
|
SELECT ps.page_url, ps.id AS ps_id, sc.id AS scene_id, sc.duration_sec
|
||||||
|
FROM playback_sources ps
|
||||||
|
JOIN scenes sc ON sc.id = ps.scene_id
|
||||||
|
WHERE ps.origin = :origin AND ps.page_url = ANY(:urls)
|
||||||
|
"""
|
||||||
|
),
|
||||||
|
{"origin": _ORIGIN, "urls": list(lookup)},
|
||||||
|
).fetchall()
|
||||||
|
for page_url, ps_id, scene_id, have in rows:
|
||||||
|
counters["matched"] += 1
|
||||||
|
sec = lookup.get(page_url)
|
||||||
|
if not sec:
|
||||||
|
continue
|
||||||
|
if have:
|
||||||
|
counters["had"] += 1
|
||||||
|
continue
|
||||||
|
counters["filled"] += 1
|
||||||
|
if dry_run:
|
||||||
|
continue
|
||||||
|
s.execute(
|
||||||
|
text("UPDATE scenes SET duration_sec = :d WHERE id = :id AND duration_sec IS NULL"),
|
||||||
|
{"d": sec, "id": scene_id},
|
||||||
|
)
|
||||||
|
s.execute(
|
||||||
|
text(
|
||||||
|
"UPDATE playback_sources SET duration_sec = :d "
|
||||||
|
" WHERE id = :id AND duration_sec IS NULL"
|
||||||
|
),
|
||||||
|
{"d": sec, "id": ps_id},
|
||||||
|
)
|
||||||
|
return counters
|
||||||
|
|
||||||
|
|
||||||
|
def _fetch(page: int) -> tuple[int, str]:
|
||||||
|
res = browser_get(_LISTING.format(page=page), timeout=45)
|
||||||
|
status = int(getattr(res, "status_code", 200) or 200)
|
||||||
|
html = res.text if hasattr(res, "text") else str(res)
|
||||||
|
return status, html
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
ap = argparse.ArgumentParser()
|
||||||
|
ap.add_argument("--start", type=int, default=None, help="pierwsza strona (domyślnie ze state-file albo 1)")
|
||||||
|
ap.add_argument("--end", type=int, default=20000, help="ostatnia strona (przebieg i tak staje na 404)")
|
||||||
|
ap.add_argument("--pace", type=float, default=1.0, help="odstęp między stronami w sekundach")
|
||||||
|
ap.add_argument("--dry-run", action="store_true")
|
||||||
|
ap.add_argument("--state-file", default="/tmp/porntrex_duration_walk.json")
|
||||||
|
args = ap.parse_args()
|
||||||
|
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||||||
|
|
||||||
|
start = args.start
|
||||||
|
if start is None:
|
||||||
|
try:
|
||||||
|
with open(args.state_file, encoding="utf-8") as f:
|
||||||
|
start = int(json.load(f).get("next_page", 1))
|
||||||
|
except (OSError, ValueError):
|
||||||
|
start = 1
|
||||||
|
|
||||||
|
total = {"pages": 0, "cards": 0, "matched": 0, "filled": 0, "had": 0}
|
||||||
|
empty_streak = 0
|
||||||
|
t0 = time.time()
|
||||||
|
for page in range(start, args.end + 1):
|
||||||
|
try:
|
||||||
|
status, html = _fetch(page)
|
||||||
|
except Exception as e:
|
||||||
|
log.warning("strona %d: błąd pobrania %s, czekam 30 s", page, e)
|
||||||
|
time.sleep(30)
|
||||||
|
try:
|
||||||
|
status, html = _fetch(page)
|
||||||
|
except Exception as e2:
|
||||||
|
log.error("strona %d: drugi błąd %s, przerywam", page, e2)
|
||||||
|
break
|
||||||
|
cards = parse_cards(html) if status < 400 else {}
|
||||||
|
if not cards:
|
||||||
|
empty_streak += 1
|
||||||
|
log.info("strona %d: http %s, brak kafelków (%d z rzędu)", page, status, empty_streak)
|
||||||
|
if empty_streak >= 3:
|
||||||
|
log.info("koniec listingu na stronie %d", page)
|
||||||
|
break
|
||||||
|
time.sleep(args.pace)
|
||||||
|
continue
|
||||||
|
empty_streak = 0
|
||||||
|
c = apply(cards, dry_run=args.dry_run)
|
||||||
|
total["pages"] += 1
|
||||||
|
total["cards"] += len(cards)
|
||||||
|
for k in ("matched", "filled", "had"):
|
||||||
|
total[k] += c[k]
|
||||||
|
log.info(
|
||||||
|
"strona %d: kafelków %d, w bazie %d, doliczone %d, miały %d%s",
|
||||||
|
page, len(cards), c["matched"], c["filled"], c["had"], " (dry-run)" if args.dry_run else "",
|
||||||
|
)
|
||||||
|
if not args.dry_run:
|
||||||
|
try:
|
||||||
|
with open(args.state_file, "w", encoding="utf-8") as f:
|
||||||
|
json.dump({"next_page": page + 1, "total": total, "ts": time.time()}, f)
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
time.sleep(args.pace)
|
||||||
|
log.info("razem: %s w %.0f s", total, time.time() - t0)
|
||||||
|
print(json.dumps(total))
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
98
scripts/reap_kvs_404.py
Normal file
98
scripts/reap_kvs_404.py
Normal file
|
|
@ -0,0 +1,98 @@
|
||||||
|
"""Oznacza martwe zrodla KVS (yespornvip i inne) ze strona sceny 404/410 (2026-10-15).
|
||||||
|
|
||||||
|
resolve_kvs do 15.10 zamienial 404 na "chwilowy blad", wiec usuniete filmy wisialy jako
|
||||||
|
zywe (yespornvip: ok. polowa z 39 tys. zrodel, zgloszenie 5e81eb76). Od 15.10 resolve sam
|
||||||
|
oznacza 404 przy kliknieciu, ten skrypt czysci zaleglosci bez czekania na klikniecia.
|
||||||
|
|
||||||
|
Dwie kontrole (patrz notatka o przemiale zrodel): strona daje 404/410 DWA razy, w odstepie
|
||||||
|
co najmniej --recheck-after sekund, i w tresci nie ma odtwarzacza (license_code). Przekierowanie
|
||||||
|
na nowy adres z odtwarzaczem to zywe zrodlo i zostaje.
|
||||||
|
|
||||||
|
python scripts/reap_kvs_404.py --origin tube:yespornvip --limit 100 # podglad
|
||||||
|
python scripts/reap_kvs_404.py --origin tube:yespornvip --limit 50000 --apply
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import logging
|
||||||
|
import time
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
from app.db import session_scope
|
||||||
|
from app.extractors._fetch import browser_get
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def _status(url: str) -> tuple[int, bool]:
|
||||||
|
try:
|
||||||
|
r = browser_get(url, timeout=20.0, follow_redirects=True)
|
||||||
|
except Exception:
|
||||||
|
return 0, False
|
||||||
|
return r.status_code, "license_code" in (r.text or "")
|
||||||
|
|
||||||
|
|
||||||
|
def run(origin: str, limit: int, apply: bool, rps: float, recheck_after: float) -> dict[str, int]:
|
||||||
|
c = {"checked": 0, "alive": 0, "first_404": 0, "dead": 0, "other": 0}
|
||||||
|
with session_scope() as s:
|
||||||
|
rows = s.execute(
|
||||||
|
sa.text(
|
||||||
|
"SELECT id, page_url FROM playback_sources WHERE origin = :o AND dead_at IS NULL "
|
||||||
|
"ORDER BY id LIMIT :lim"
|
||||||
|
),
|
||||||
|
{"o": origin, "lim": limit},
|
||||||
|
).all()
|
||||||
|
suspects: list[tuple] = []
|
||||||
|
for pb_id, url in rows:
|
||||||
|
c["checked"] += 1
|
||||||
|
st, player = _status(url)
|
||||||
|
if st in (404, 410) and not player:
|
||||||
|
c["first_404"] += 1
|
||||||
|
suspects.append((pb_id, url, time.time()))
|
||||||
|
elif st and st < 400:
|
||||||
|
c["alive"] += 1
|
||||||
|
else:
|
||||||
|
c["other"] += 1
|
||||||
|
if c["checked"] % 500 == 0:
|
||||||
|
log.info("kvs reap pass1 %s", c)
|
||||||
|
time.sleep(1.0 / rps)
|
||||||
|
dead: list = []
|
||||||
|
for pb_id, url, t0 in suspects:
|
||||||
|
wait = recheck_after - (time.time() - t0)
|
||||||
|
if wait > 0:
|
||||||
|
time.sleep(wait)
|
||||||
|
st, player = _status(url)
|
||||||
|
if st in (404, 410) and not player:
|
||||||
|
dead.append(pb_id)
|
||||||
|
time.sleep(1.0 / rps)
|
||||||
|
c["dead"] = len(dead)
|
||||||
|
print(c)
|
||||||
|
if apply and dead:
|
||||||
|
reason = f"tube page 404 (reap_kvs_404 {datetime.now(UTC):%Y-%m-%d})"
|
||||||
|
for i in range(0, len(dead), 500):
|
||||||
|
with session_scope() as s:
|
||||||
|
s.execute(
|
||||||
|
sa.text("UPDATE playback_sources SET dead_at = now(), dead_reason = :r WHERE id = ANY(:ids)"),
|
||||||
|
{"r": reason, "ids": dead[i:i + 500]},
|
||||||
|
)
|
||||||
|
s.commit()
|
||||||
|
return c
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
logging.basicConfig(level=logging.INFO)
|
||||||
|
p = argparse.ArgumentParser()
|
||||||
|
p.add_argument("--origin", required=True)
|
||||||
|
p.add_argument("--limit", type=int, default=100)
|
||||||
|
p.add_argument("--apply", action="store_true")
|
||||||
|
p.add_argument("--rps", type=float, default=3.0)
|
||||||
|
p.add_argument("--recheck-after", type=float, default=60.0)
|
||||||
|
args = p.parse_args()
|
||||||
|
run(args.origin, args.limit, args.apply, args.rps, args.recheck_after)
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
200
scripts/remerge_movies_roman_studio.py
Normal file
200
scripts/remerge_movies_roman_studio.py
Normal file
|
|
@ -0,0 +1,200 @@
|
||||||
|
"""Jednorazowy dedup filmow po poprawkach oceny z 2026-10-16 (zgloszenie f8c9486e).
|
||||||
|
|
||||||
|
Ocena (movie_score) liczy teraz cyfry rzymskie jak arabskie ("Hardcore Gangbangs VIII"
|
||||||
|
= "... 8") i uznaje za zgodne studia o tej samej nazwie bez spacji ("EvilAngel" z xtapes
|
||||||
|
i "Evil Angel"). Istniejace pary, ktore ingest wtedy rozdzielil, zostaja rozdzielone,
|
||||||
|
wiec ten skrypt przechodzi tylko po parach, na ktore poprawka ma wplyw, i scala te z
|
||||||
|
ocena >= AUTO_THRESHOLD. Pary odrzucone przez czlowieka (merge_candidates.rejected)
|
||||||
|
pomija. Zostaje starszy film (jego tytul i id), mlodszy jest wlewany.
|
||||||
|
|
||||||
|
python3 /srv/scripts/remerge_movies_roman_studio.py # dry-run
|
||||||
|
python3 /srv/scripts/remerge_movies_roman_studio.py --apply
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import uuid
|
||||||
|
|
||||||
|
from sqlalchemy import select, text
|
||||||
|
|
||||||
|
from app.db import session_scope
|
||||||
|
from app.models.merge_candidate import MergeCandidate, MergeKind, MergeStatus
|
||||||
|
from app.models.movie import Movie, MoviePerformer
|
||||||
|
from app.resolve.movie_match import find_movie_candidates
|
||||||
|
from app.resolve.movie_merge import MovieMergeError, merge_movies
|
||||||
|
from app.resolve.movie_score import (
|
||||||
|
AUTO_THRESHOLD,
|
||||||
|
_roman_to_arabic,
|
||||||
|
_same_studio_name,
|
||||||
|
score_movie_candidate,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _affected_ids(session) -> list[uuid.UUID]:
|
||||||
|
"""Filmy z cyfra rzymska w tytule albo ze studiem, ktore ma blizniaka bez spacji."""
|
||||||
|
rows = session.execute(text("""
|
||||||
|
WITH twins AS (
|
||||||
|
SELECT a.id FROM studios a JOIN studios b
|
||||||
|
ON a.id <> b.id
|
||||||
|
AND replace(a.name_normalized, ' ', '') = replace(b.name_normalized, ' ', '')
|
||||||
|
AND length(replace(a.name_normalized, ' ', '')) >= 4
|
||||||
|
)
|
||||||
|
SELECT m.id FROM movies m
|
||||||
|
WHERE m.title_normalized ~ '(^| )(x{0,3})(ix|iv|v?i{0,3})( |$)'
|
||||||
|
OR m.studio_id IN (SELECT id FROM twins)
|
||||||
|
ORDER BY m.created_at DESC
|
||||||
|
""")).scalars().all()
|
||||||
|
return list(rows)
|
||||||
|
|
||||||
|
|
||||||
|
def _rejected_pairs(session) -> set[frozenset]:
|
||||||
|
rows = session.execute(
|
||||||
|
select(MergeCandidate.left_id, MergeCandidate.right_id).where(
|
||||||
|
MergeCandidate.kind == MergeKind.movie,
|
||||||
|
MergeCandidate.status == MergeStatus.rejected,
|
||||||
|
)
|
||||||
|
).all()
|
||||||
|
return {frozenset((a, b)) for a, b in rows}
|
||||||
|
|
||||||
|
|
||||||
|
def _keep_rank(session, movie: Movie) -> tuple:
|
||||||
|
"""Zostaje film z rokiem, potem z liczniejsza obsada, potem starszy. Sam wiek dawal
|
||||||
|
np. "Hard Core Gangbangs 8" bez roku zamiast "Hardcore Gangbangs VIII" z TPDB."""
|
||||||
|
cast = session.execute(
|
||||||
|
text("SELECT count(*) FROM movie_performers WHERE movie_id = :m"), {"m": movie.id}
|
||||||
|
).scalar_one()
|
||||||
|
return (movie.release_year is not None, cast, -movie.created_at.timestamp())
|
||||||
|
|
||||||
|
|
||||||
|
def _same_title_studio(apply: bool) -> None:
|
||||||
|
"""Grupy filmow o identycznym tytule i studiu. find_movie_candidates bral dowolne 50
|
||||||
|
kandydatow bez kolejnosci (do 2026-10-16), wiec przy duzych studiach wierny duplikat
|
||||||
|
wypadal z puli. Kazdy czlonek grupy oceniany wzgledem zostajacego; ten sam prog co
|
||||||
|
ingest, wiec wznowienia z innym rokiem (ocena max 0,8) zostaja osobno."""
|
||||||
|
with session_scope() as s:
|
||||||
|
groups = s.execute(text("""
|
||||||
|
SELECT array_agg(id) FROM movies
|
||||||
|
WHERE studio_id IS NOT NULL AND length(title_normalized) >= 4
|
||||||
|
GROUP BY title_normalized, studio_id HAVING count(*) > 1
|
||||||
|
""")).scalars().all()
|
||||||
|
rejected = _rejected_pairs(s)
|
||||||
|
print(f"grup: {len(groups)}", flush=True)
|
||||||
|
merged = 0
|
||||||
|
for ids in groups:
|
||||||
|
with session_scope() as s:
|
||||||
|
members = sorted((s.get(Movie, i) for i in ids), key=lambda x: _keep_rank(s, x), reverse=True)
|
||||||
|
keep = members[0]
|
||||||
|
for m in members[1:]:
|
||||||
|
if frozenset((keep.id, m.id)) in rejected:
|
||||||
|
continue
|
||||||
|
perfs = [
|
||||||
|
r.performer_id
|
||||||
|
for r in s.execute(
|
||||||
|
MoviePerformer.__table__.select().where(MoviePerformer.movie_id == m.id)
|
||||||
|
).fetchall()
|
||||||
|
]
|
||||||
|
b = score_movie_candidate(
|
||||||
|
s,
|
||||||
|
candidate=keep,
|
||||||
|
norm_title_normalized=m.title_normalized,
|
||||||
|
norm_release_year=m.release_year,
|
||||||
|
norm_studio_id=m.studio_id,
|
||||||
|
norm_performer_ids=perfs,
|
||||||
|
)
|
||||||
|
if b.composite < AUTO_THRESHOLD:
|
||||||
|
continue
|
||||||
|
print(
|
||||||
|
f"{b.composite:.2f} ZOSTAJE {keep.title!r} ({keep.release_year})"
|
||||||
|
f" <= {m.title!r} ({m.release_year})",
|
||||||
|
flush=True,
|
||||||
|
)
|
||||||
|
if apply:
|
||||||
|
try:
|
||||||
|
merge_movies(s, keep_id=keep.id, drop_id=m.id, resolved_by="remerge_same_title_studio_20261016")
|
||||||
|
except MovieMergeError as e:
|
||||||
|
print(" blad:", e, flush=True)
|
||||||
|
continue
|
||||||
|
merged += 1
|
||||||
|
print(f"{'scalono' if apply else 'do scalenia'}: {merged}", flush=True)
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
ap = argparse.ArgumentParser()
|
||||||
|
ap.add_argument("--apply", action="store_true")
|
||||||
|
ap.add_argument("--limit", type=int, default=0)
|
||||||
|
ap.add_argument("--same-title-studio", action="store_true")
|
||||||
|
args = ap.parse_args()
|
||||||
|
if args.same_title_studio:
|
||||||
|
_same_title_studio(args.apply)
|
||||||
|
return
|
||||||
|
|
||||||
|
with session_scope() as s:
|
||||||
|
ids = _affected_ids(s)
|
||||||
|
rejected = _rejected_pairs(s)
|
||||||
|
if args.limit:
|
||||||
|
ids = ids[: args.limit]
|
||||||
|
print(f"filmy do sprawdzenia: {len(ids)}", flush=True)
|
||||||
|
|
||||||
|
gone: set[uuid.UUID] = set()
|
||||||
|
merged = 0
|
||||||
|
for n, mid in enumerate(ids, 1):
|
||||||
|
if n % 1000 == 0:
|
||||||
|
print(f"... {n}/{len(ids)}, par: {merged}", flush=True)
|
||||||
|
if mid in gone:
|
||||||
|
continue
|
||||||
|
with session_scope() as s:
|
||||||
|
m = s.get(Movie, mid)
|
||||||
|
if m is None or not m.title_normalized:
|
||||||
|
continue
|
||||||
|
perfs = [
|
||||||
|
r.performer_id
|
||||||
|
for r in s.execute(
|
||||||
|
MoviePerformer.__table__.select().where(MoviePerformer.movie_id == m.id)
|
||||||
|
).fetchall()
|
||||||
|
]
|
||||||
|
best, best_b = None, None
|
||||||
|
for c in find_movie_candidates(
|
||||||
|
s, title_normalized=m.title_normalized, studio_id=m.studio_id, release_year=m.release_year
|
||||||
|
):
|
||||||
|
if c.id == m.id or c.id in gone or frozenset((c.id, m.id)) in rejected:
|
||||||
|
continue
|
||||||
|
roman = _roman_to_arabic(c.title_normalized or "") != (c.title_normalized or "") or (
|
||||||
|
_roman_to_arabic(m.title_normalized) != m.title_normalized
|
||||||
|
)
|
||||||
|
twin_studio = bool(
|
||||||
|
c.studio_id and m.studio_id and c.studio_id != m.studio_id
|
||||||
|
and _same_studio_name(s, c.studio_id, m.studio_id)
|
||||||
|
)
|
||||||
|
if not (roman or twin_studio):
|
||||||
|
continue # ta para wyglada dla oceny tak samo jak przy ingescie
|
||||||
|
b = score_movie_candidate(
|
||||||
|
s,
|
||||||
|
candidate=c,
|
||||||
|
norm_title_normalized=m.title_normalized,
|
||||||
|
norm_release_year=m.release_year,
|
||||||
|
norm_studio_id=m.studio_id,
|
||||||
|
norm_performer_ids=perfs,
|
||||||
|
)
|
||||||
|
if b.composite >= AUTO_THRESHOLD and (best_b is None or b.composite > best_b.composite):
|
||||||
|
best, best_b = c, b
|
||||||
|
if best is None:
|
||||||
|
continue
|
||||||
|
keep, drop = sorted((best, m), key=lambda x: _keep_rank(s, x), reverse=True)
|
||||||
|
print(
|
||||||
|
f"{best_b.composite:.2f} ZOSTAJE {keep.title!r} ({keep.release_year})"
|
||||||
|
f" <= {drop.title!r} ({drop.release_year})",
|
||||||
|
flush=True,
|
||||||
|
)
|
||||||
|
if args.apply:
|
||||||
|
try:
|
||||||
|
merge_movies(s, keep_id=keep.id, drop_id=drop.id, resolved_by="remerge_roman_studio_20261016")
|
||||||
|
except MovieMergeError as e:
|
||||||
|
print(" blad:", e, flush=True)
|
||||||
|
continue
|
||||||
|
gone.add(drop.id)
|
||||||
|
merged += 1
|
||||||
|
print(f"{'scalono' if args.apply else 'do scalenia'}: {merged}", flush=True)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
|
|
@ -25,6 +25,7 @@ from app.connectors.dooplay import (
|
||||||
MangopornConnector,
|
MangopornConnector,
|
||||||
PandamoviesConnector,
|
PandamoviesConnector,
|
||||||
StreampornConnector,
|
StreampornConnector,
|
||||||
|
StreampornVipConnector,
|
||||||
)
|
)
|
||||||
from app.db import session_scope
|
from app.db import session_scope
|
||||||
from app.models.movie import Movie
|
from app.models.movie import Movie
|
||||||
|
|
@ -38,6 +39,7 @@ CONNECTORS = {
|
||||||
"pandamovies": PandamoviesConnector,
|
"pandamovies": PandamoviesConnector,
|
||||||
"mangoporn": MangopornConnector,
|
"mangoporn": MangopornConnector,
|
||||||
"streamporn": StreampornConnector,
|
"streamporn": StreampornConnector,
|
||||||
|
"streampornvip": StreampornVipConnector,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
89
scripts/revive_user_marked_last_sources.py
Normal file
89
scripts/revive_user_marked_last_sources.py
Normal file
|
|
@ -0,0 +1,89 @@
|
||||||
|
"""Przywraca ostatnie zrodla scen zabite przytrzymaniem w apce, jesli strona zyje (2026-10-15).
|
||||||
|
|
||||||
|
Do 15.10 przytrzymanie "mark as broken" zabijalo zrodlo bez sprawdzenia. W 30 dni 490 scen
|
||||||
|
zostalo przez to bez zadnego zrodla i zniknelo z list (porndoe: 403 z sieci uzytkownika,
|
||||||
|
galaxyporn, perverzija, javflix...). Od 15.10 serwer sprawdza ostatnie zrodlo przed
|
||||||
|
oznaczeniem; ten skrypt cofa wczesniejsze przypadki, ale tylko z POZYTYWNYM dowodem:
|
||||||
|
- ekstraktor zwraca zrodla (strona i film istnieja),
|
||||||
|
- dla serwisow grajacych tylko w WebView (ekstraktor nic nie pobiera): strona odpowiada 200.
|
||||||
|
|
||||||
|
python scripts/revive_user_marked_last_sources.py # podglad
|
||||||
|
python scripts/revive_user_marked_last_sources.py --apply [--days 30]
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import logging
|
||||||
|
from collections import Counter
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
from app.db import session_scope
|
||||||
|
from app.extractors import is_vps_blocked_fallback, try_extract
|
||||||
|
from app.extractors._fetch import browser_get
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def _alive(origin: str, page_url: str) -> bool:
|
||||||
|
if not origin.startswith(("tube:", "pornapp:")):
|
||||||
|
return False
|
||||||
|
sitetag = origin.split(":", 1)[1]
|
||||||
|
if is_vps_blocked_fallback(sitetag):
|
||||||
|
try:
|
||||||
|
return browser_get(page_url, timeout=20.0).status_code < 400
|
||||||
|
except Exception:
|
||||||
|
return False
|
||||||
|
try:
|
||||||
|
return bool(try_extract(sitetag, page_url))
|
||||||
|
except Exception:
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def run(apply: bool, days: int) -> dict:
|
||||||
|
with session_scope() as s:
|
||||||
|
rows = s.execute(
|
||||||
|
sa.text(
|
||||||
|
"""
|
||||||
|
SELECT p.id, p.origin, p.page_url
|
||||||
|
FROM playback_sources p
|
||||||
|
WHERE p.dead_reason LIKE 'user-marked dead%'
|
||||||
|
AND p.dead_at > now() - make_interval(days => :d)
|
||||||
|
AND NOT EXISTS (
|
||||||
|
SELECT 1 FROM playback_sources o
|
||||||
|
WHERE o.scene_id = p.scene_id AND o.dead_at IS NULL
|
||||||
|
)
|
||||||
|
"""
|
||||||
|
),
|
||||||
|
{"d": days},
|
||||||
|
).all()
|
||||||
|
revive: list = []
|
||||||
|
by_origin: Counter = Counter()
|
||||||
|
for pb_id, origin, url in rows:
|
||||||
|
if _alive(origin, url):
|
||||||
|
revive.append(pb_id)
|
||||||
|
by_origin[origin] += 1
|
||||||
|
print({"candidates": len(rows), "alive": len(revive)}, by_origin.most_common(8))
|
||||||
|
if apply and revive:
|
||||||
|
with session_scope() as s:
|
||||||
|
s.execute(
|
||||||
|
sa.text("UPDATE playback_sources SET dead_at = NULL, dead_reason = NULL WHERE id = ANY(:ids)"),
|
||||||
|
{"ids": revive},
|
||||||
|
)
|
||||||
|
s.commit()
|
||||||
|
print("revived", len(revive))
|
||||||
|
return {"candidates": len(rows), "revived": len(revive) if apply else 0}
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
logging.basicConfig(level=logging.WARNING)
|
||||||
|
p = argparse.ArgumentParser()
|
||||||
|
p.add_argument("--apply", action="store_true")
|
||||||
|
p.add_argument("--days", type=int, default=30)
|
||||||
|
args = p.parse_args()
|
||||||
|
run(args.apply, args.days)
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
|
|
@ -95,7 +95,7 @@
|
||||||
</p>
|
</p>
|
||||||
|
|
||||||
<div class="flex flex-wrap gap-3">
|
<div class="flex flex-wrap gap-3">
|
||||||
<a href="/goon-v0.4.2.apk"
|
<a href="/goon-v0.4.4.apk"
|
||||||
class="px-6 py-4 rounded-xl bg-accent text-white font-bold hover:bg-accentDeep transition glow">
|
class="px-6 py-4 rounded-xl bg-accent text-white font-bold hover:bg-accentDeep transition glow">
|
||||||
Download APK
|
Download APK
|
||||||
</a>
|
</a>
|
||||||
|
|
@ -240,6 +240,25 @@
|
||||||
</p>
|
</p>
|
||||||
|
|
||||||
<div class="space-y-6">
|
<div class="space-y-6">
|
||||||
|
<div class="bg-card border border-border rounded-2xl p-6">
|
||||||
|
<div class="font-mono text-xs text-accent mb-2">October 2026 · 0.4.4</div>
|
||||||
|
<h3 class="text-lg font-bold mb-2">TV remote you can see</h3>
|
||||||
|
<p class="text-sm text-gray-400 leading-relaxed">
|
||||||
|
On Fire TV and Android TV the item picked with the remote now gets a clear outline,
|
||||||
|
so you always know where you are. Tablets start in landscape instead of a narrow
|
||||||
|
portrait strip. Installs over 0.4.3 (same signing key).
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
<div class="bg-card border border-border rounded-2xl p-6">
|
||||||
|
<div class="font-mono text-xs text-accent mb-2">October 2026 · 0.4.3</div>
|
||||||
|
<h3 class="text-lg font-bold mb-2">Android TV stays up</h3>
|
||||||
|
<p class="text-sm text-gray-400 leading-relaxed">
|
||||||
|
On TV boxes the app could crash when it was opened from two places at once,
|
||||||
|
for example the TV home screen and a shortcut. Fixed by running a single
|
||||||
|
activity. Native crash reports now carry debug symbols, so the remaining
|
||||||
|
rare crashes can be traced. Installs over 0.4.2 (same signing key).
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
<div class="bg-card border border-border rounded-2xl p-6">
|
<div class="bg-card border border-border rounded-2xl p-6">
|
||||||
<div class="font-mono text-xs text-accent mb-2">September 2026 · 0.4.0</div>
|
<div class="font-mono text-xs text-accent mb-2">September 2026 · 0.4.0</div>
|
||||||
<h3 class="text-lg font-bold mb-2">Edge</h3>
|
<h3 class="text-lg font-bold mb-2">Edge</h3>
|
||||||
|
|
@ -327,7 +346,7 @@
|
||||||
<li class="bg-card border border-border rounded-2xl p-5 flex gap-4">
|
<li class="bg-card border border-border rounded-2xl p-5 flex gap-4">
|
||||||
<span class="text-accent font-extrabold text-xl">1</span>
|
<span class="text-accent font-extrabold text-xl">1</span>
|
||||||
<span class="text-sm text-gray-300 leading-relaxed">
|
<span class="text-sm text-gray-300 leading-relaxed">
|
||||||
<a href="/goon-v0.4.2.apk" class="text-accent font-bold hover:underline">Download the APK</a>
|
<a href="/goon-v0.4.4.apk" class="text-accent font-bold hover:underline">Download the APK</a>
|
||||||
and open it. Allow "install from unknown sources" for your browser if Android asks.
|
and open it. Allow "install from unknown sources" for your browser if Android asks.
|
||||||
</span>
|
</span>
|
||||||
</li>
|
</li>
|
||||||
|
|
@ -404,7 +423,7 @@
|
||||||
</p>
|
</p>
|
||||||
</div>
|
</div>
|
||||||
<div class="flex flex-col gap-1 text-right">
|
<div class="flex flex-col gap-1 text-right">
|
||||||
<a href="/goon-v0.4.2.apk" class="hover:text-accent transition">Download APK</a>
|
<a href="/goon-v0.4.4.apk" class="hover:text-accent transition">Download APK</a>
|
||||||
<a href="https://git.goon-foss.org/goon-foss/goon" class="hover:text-accent transition">Source code</a>
|
<a href="https://git.goon-foss.org/goon-foss/goon" class="hover:text-accent transition">Source code</a>
|
||||||
<a href="#donate" class="hover:text-accent transition">Donate</a>
|
<a href="#donate" class="hover:text-accent transition">Donate</a>
|
||||||
</div>
|
</div>
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue