feat(jav): javflix.cc scraper + extractor (first JAV source, gated off main feed)

First site of the JAV vertical (user request, separate section). javflix.cc is a
WordPress JAV aggregator, VPS-reachable and server-rendered. Scraper parses itemprop
metadata (title, thumbnail, uploadDate; no duration) and the JAV code (BKD-368 style)
as a tag. Playback: server buttons are <a class="myLink" href="<embed>"> (streamtape,
voe, doodstream, emturbovid) whose href sits in the raw HTML; the generic _embed_iframe
extractor already resolves those via its anchor-hoster pattern, so javflix registers
under sitetag "javflix" with a thin wrapper that drops the players.mp4 placeholder.
Added emturbovid to the anchor-hoster host list.

Verified end-to-end on prod: scrape a listing -> RawScene with metadata + code tag,
resolve -> emturbovid/voe/doodstream hoster sources (all Goon-playable).

NOT registered in ALL_BROWSE_SCRAPERS yet. JAV is a separate vertical (user decision)
that must be gated out of the main scenes feed before ingest so it does not flood the
western catalog. Next: feed gating (exclude JAV origins by default) + a mobile JAV tab
+ on-device playback test, then enable scheduled ingest. jav.guru / vjav / supjav
follow the same pattern.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
goon-foss 2026-07-10 10:58:34 +02:00
parent 2d3831bd53
commit d8d00295e1
4 changed files with 120 additions and 1 deletions

View file

@ -0,0 +1,94 @@
"""javflix.cc — JAV browse scraper (WordPress, English-subbed JAV).
Osobna pula JAV (kody typu BKD-368, tytuły azjatyckie), origin `tube:javflix`.
NIE deduplikuje się z zachodnim katalogiem świadomie orphan vertical (sekcja JAV).
Struktura (RE 2026-07-10):
- listing: `/page/N/` (WordPress archive), posty pod `/<slug>/` (np. /bkd-368-english-subtitle/)
- detail: metadane w `itemprop` (name/thumbnailUrl/uploadDate; BRAK duration),
playback = przyciski serwerów `<a class="myLink" name="<hoster>" href="<embed>">`
(streamtape/voe/doodstream/emturbovid). Href jest w SUROWYM HTML (theme strippuje go
po renderze JS, ale scraper widzi surowy). Embed-hostery obsługuje generyczny
ekstraktor `_embed_iframe` (anchor-hoster pattern) rejestrujemy go pod `javflix`.
"""
from __future__ import annotations
import re
from app.connectors.base import RawPlaybackSource, RawScene, RawTag
from app.connectors.direct_scrapers._browse_base import BaseBrowseScraper
from app.connectors.direct_scrapers._playtube import _parse_iso_date
_BASE = "https://javflix.cc"
# Posty to `javflix.cc/<slug>/`. Odsiewamy strony nie-postowe (taksonomie, statyczne).
_POST_RE = re.compile(r'href="(https://javflix\.cc/[a-z0-9][a-z0-9\-]{4,}/)"', re.IGNORECASE)
_NON_POST = (
"/page/", "/category/", "/categories/", "/genre/", "/maker/", "/actress/",
"/actors/", "/tag/", "/tags/", "/studio/", "/label/", "/series/", "/wp-",
"/18-usc", "/dmca", "/contact", "/privacy", "/about", "/2257",
)
_CODE_RE = re.compile(r"^([a-z]+-?\d+[a-z]?)", re.IGNORECASE)
def _itemprop(html: str, name: str) -> str | None:
m = re.search(
rf'itemprop="{name}"\s+content="([^"]+)"', html, re.IGNORECASE
)
return m.group(1).strip() if m else None
class JavflixScraper(BaseBrowseScraper):
sitetag = "javflix"
def _listing_url(self, page: int) -> str:
return f"{_BASE}/" if page <= 1 else f"{_BASE}/page/{page}/"
def _extract_scene_urls(self, listing_html: str) -> list[str]:
seen: set[str] = set()
out: list[str] = []
for m in _POST_RE.finditer(listing_html):
url = m.group(1)
if any(x in url for x in _NON_POST):
continue
if url.rstrip("/") == _BASE:
continue
if url not in seen:
seen.add(url)
out.append(url)
return out
def _parse_detail(self, scene_url: str, detail_html: str) -> RawScene | None:
title = _itemprop(detail_html, "name")
if not title:
tm = re.search(r"<title>([^<]+)</title>", detail_html)
title = tm.group(1).split(" ")[0].strip() if tm else None
if not title:
return None
thumb = _itemprop(detail_html, "thumbnailUrl")
up = _itemprop(detail_html, "uploadDate")
release_date = _parse_iso_date(up) if up else None
# JAV code (BKD-368) ze sluga — kanoniczny identyfikator, dodajemy jako tag
# (searchable) bo javflix nie ma osobnego pola kodu.
slug = scene_url.rstrip("/").rsplit("/", 1)[-1]
cm = _CODE_RE.match(slug)
tags: list[RawTag] = []
if cm:
code = cm.group(1).upper()
tags.append(RawTag(external_id=f"javcode:{code}", name=code, slug=cm.group(1).lower()))
return RawScene(
external_id=f"{self.sitetag}:{scene_url}",
title=title,
release_date=release_date,
url=scene_url,
tags=tags,
playback_sources=[
RawPlaybackSource(
origin=f"tube:{self.sitetag}",
page_url=scene_url,
thumbnail_url=thumb,
)
],
)

View file

@ -33,6 +33,7 @@ from app.extractors.tubes import (
hdporngg,
hqfap,
hqporner,
javflix,
neporn,
latestpornvideo,
paradisehill,
@ -170,6 +171,11 @@ _REGISTRY: dict[str, Callable[[str], list[StreamSource] | None]] = {
# strona wróciła na CDN vstor.top z realnymi plikami (portable cross-IP, zweryfikowane),
# user request. 4k69 zostaje usunięty (nie sprawdzany ponownie).
"hqfapcom": hqfap.extract,
# javflix (JAV, WordPress) — przyciski serwerów to `<a class="myLink" href="<embed>">`
# (streamtape/voe/doodstream/emturbovid). Generyczny _embed_iframe łapie je anchor-hoster
# patternem → type='hoster', telefon resolwuje (voe backend, dood/filemoon phone-side).
# Wrapper javflix.extract odsiewa placeholder players.mp4.
"javflix": javflix.extract,
# neporn — KVS function/0 + license (jak freshporno). Server-side _kvs resolve →
# data001.neporn.com/remote_control.php portable (cross-IP 206, 2026-06-10).
"neporncom": neporn.extract,

View file

@ -124,7 +124,7 @@ _JS_SERVER_URL_RE = re.compile(
_ANCHOR_HOSTER_RE = re.compile(
r'<a\s+[^>]*href=["\'](?P<url>https?://(?:'
r'playmogo|luluvid|doodporn|doodstream|dood\.[a-z]+|streamtape|streamta\.pe|'
r'filemoon|streamwish|sdefx|veev|turbovidhls|gounlimited|iceyfile|hlswish|'
r'filemoon|emturbovid|streamwish|sdefx|veev|turbovidhls|gounlimited|iceyfile|hlswish|'
r'mixdrop|voe|vidoza|mediafire|asnwish|obeywish|streamruby|hqq\.[a-z]+|'
r'feurl|streamhide|krakenfiles|earnvids|jollytuna|peekvids|playerwish'
r')\.[a-z]{2,8}/[^"\']+)["\']',

View file

@ -0,0 +1,19 @@
"""javflix.cc extractor — cienki wrapper na generyczny _embed_iframe.
javflix trzyma hostery w `<a class="myLink" href="<embed>">` (streamtape/voe/doodstream/
emturbovid), które _embed_iframe łapie anchor-hoster patternem. Wrapper odsiewa tylko
placeholder `players.mp4` (pusty iframe zanim JS podmieni src) bez tego trafiał jako
martwe pierwsze źródło type='mp4'.
"""
from __future__ import annotations
from app.extractors._models import StreamSource
from app.extractors.tubes import _embed_iframe
def extract(page_url: str, *, timeout: float = 60.0) -> list[StreamSource] | None:
srcs = _embed_iframe.extract(page_url, timeout=timeout)
if not srcs:
return None
srcs = [s for s in srcs if "players.mp4" not in s.link]
return srcs or None