feat(jav): javflix.cc scraper + extractor (first JAV source, gated off main feed)
First site of the JAV vertical (user request, separate section). javflix.cc is a WordPress JAV aggregator, VPS-reachable and server-rendered. Scraper parses itemprop metadata (title, thumbnail, uploadDate; no duration) and the JAV code (BKD-368 style) as a tag. Playback: server buttons are <a class="myLink" href="<embed>"> (streamtape, voe, doodstream, emturbovid) whose href sits in the raw HTML; the generic _embed_iframe extractor already resolves those via its anchor-hoster pattern, so javflix registers under sitetag "javflix" with a thin wrapper that drops the players.mp4 placeholder. Added emturbovid to the anchor-hoster host list. Verified end-to-end on prod: scrape a listing -> RawScene with metadata + code tag, resolve -> emturbovid/voe/doodstream hoster sources (all Goon-playable). NOT registered in ALL_BROWSE_SCRAPERS yet. JAV is a separate vertical (user decision) that must be gated out of the main scenes feed before ingest so it does not flood the western catalog. Next: feed gating (exclude JAV origins by default) + a mobile JAV tab + on-device playback test, then enable scheduled ingest. jav.guru / vjav / supjav follow the same pattern. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
parent
2d3831bd53
commit
d8d00295e1
4 changed files with 120 additions and 1 deletions
94
app/connectors/direct_scrapers/javflix.py
Normal file
94
app/connectors/direct_scrapers/javflix.py
Normal file
|
|
@ -0,0 +1,94 @@
|
|||
"""javflix.cc — JAV browse scraper (WordPress, English-subbed JAV).
|
||||
|
||||
Osobna pula JAV (kody typu BKD-368, tytuły azjatyckie), origin `tube:javflix`.
|
||||
NIE deduplikuje się z zachodnim katalogiem — świadomie orphan vertical (sekcja JAV).
|
||||
|
||||
Struktura (RE 2026-07-10):
|
||||
- listing: `/page/N/` (WordPress archive), posty pod `/<slug>/` (np. /bkd-368-english-subtitle/)
|
||||
- detail: metadane w `itemprop` (name/thumbnailUrl/uploadDate; BRAK duration),
|
||||
playback = przyciski serwerów `<a class="myLink" name="<hoster>" href="<embed>">`
|
||||
(streamtape/voe/doodstream/emturbovid). Href jest w SUROWYM HTML (theme strippuje go
|
||||
po renderze JS, ale scraper widzi surowy). Embed-hostery obsługuje generyczny
|
||||
ekstraktor `_embed_iframe` (anchor-hoster pattern) → rejestrujemy go pod `javflix`.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from app.connectors.base import RawPlaybackSource, RawScene, RawTag
|
||||
from app.connectors.direct_scrapers._browse_base import BaseBrowseScraper
|
||||
from app.connectors.direct_scrapers._playtube import _parse_iso_date
|
||||
|
||||
_BASE = "https://javflix.cc"
|
||||
# Posty to `javflix.cc/<slug>/`. Odsiewamy strony nie-postowe (taksonomie, statyczne).
|
||||
_POST_RE = re.compile(r'href="(https://javflix\.cc/[a-z0-9][a-z0-9\-]{4,}/)"', re.IGNORECASE)
|
||||
_NON_POST = (
|
||||
"/page/", "/category/", "/categories/", "/genre/", "/maker/", "/actress/",
|
||||
"/actors/", "/tag/", "/tags/", "/studio/", "/label/", "/series/", "/wp-",
|
||||
"/18-usc", "/dmca", "/contact", "/privacy", "/about", "/2257",
|
||||
)
|
||||
_CODE_RE = re.compile(r"^([a-z]+-?\d+[a-z]?)", re.IGNORECASE)
|
||||
|
||||
|
||||
def _itemprop(html: str, name: str) -> str | None:
|
||||
m = re.search(
|
||||
rf'itemprop="{name}"\s+content="([^"]+)"', html, re.IGNORECASE
|
||||
)
|
||||
return m.group(1).strip() if m else None
|
||||
|
||||
|
||||
class JavflixScraper(BaseBrowseScraper):
|
||||
sitetag = "javflix"
|
||||
|
||||
def _listing_url(self, page: int) -> str:
|
||||
return f"{_BASE}/" if page <= 1 else f"{_BASE}/page/{page}/"
|
||||
|
||||
def _extract_scene_urls(self, listing_html: str) -> list[str]:
|
||||
seen: set[str] = set()
|
||||
out: list[str] = []
|
||||
for m in _POST_RE.finditer(listing_html):
|
||||
url = m.group(1)
|
||||
if any(x in url for x in _NON_POST):
|
||||
continue
|
||||
if url.rstrip("/") == _BASE:
|
||||
continue
|
||||
if url not in seen:
|
||||
seen.add(url)
|
||||
out.append(url)
|
||||
return out
|
||||
|
||||
def _parse_detail(self, scene_url: str, detail_html: str) -> RawScene | None:
|
||||
title = _itemprop(detail_html, "name")
|
||||
if not title:
|
||||
tm = re.search(r"<title>([^<]+)</title>", detail_html)
|
||||
title = tm.group(1).split(" – ")[0].strip() if tm else None
|
||||
if not title:
|
||||
return None
|
||||
|
||||
thumb = _itemprop(detail_html, "thumbnailUrl")
|
||||
up = _itemprop(detail_html, "uploadDate")
|
||||
release_date = _parse_iso_date(up) if up else None
|
||||
|
||||
# JAV code (BKD-368) ze sluga — kanoniczny identyfikator, dodajemy jako tag
|
||||
# (searchable) bo javflix nie ma osobnego pola kodu.
|
||||
slug = scene_url.rstrip("/").rsplit("/", 1)[-1]
|
||||
cm = _CODE_RE.match(slug)
|
||||
tags: list[RawTag] = []
|
||||
if cm:
|
||||
code = cm.group(1).upper()
|
||||
tags.append(RawTag(external_id=f"javcode:{code}", name=code, slug=cm.group(1).lower()))
|
||||
|
||||
return RawScene(
|
||||
external_id=f"{self.sitetag}:{scene_url}",
|
||||
title=title,
|
||||
release_date=release_date,
|
||||
url=scene_url,
|
||||
tags=tags,
|
||||
playback_sources=[
|
||||
RawPlaybackSource(
|
||||
origin=f"tube:{self.sitetag}",
|
||||
page_url=scene_url,
|
||||
thumbnail_url=thumb,
|
||||
)
|
||||
],
|
||||
)
|
||||
|
|
@ -33,6 +33,7 @@ from app.extractors.tubes import (
|
|||
hdporngg,
|
||||
hqfap,
|
||||
hqporner,
|
||||
javflix,
|
||||
neporn,
|
||||
latestpornvideo,
|
||||
paradisehill,
|
||||
|
|
@ -170,6 +171,11 @@ _REGISTRY: dict[str, Callable[[str], list[StreamSource] | None]] = {
|
|||
# strona wróciła na CDN vstor.top z realnymi plikami (portable cross-IP, zweryfikowane),
|
||||
# user request. 4k69 zostaje usunięty (nie sprawdzany ponownie).
|
||||
"hqfapcom": hqfap.extract,
|
||||
# javflix (JAV, WordPress) — przyciski serwerów to `<a class="myLink" href="<embed>">`
|
||||
# (streamtape/voe/doodstream/emturbovid). Generyczny _embed_iframe łapie je anchor-hoster
|
||||
# patternem → type='hoster', telefon resolwuje (voe backend, dood/filemoon phone-side).
|
||||
# Wrapper javflix.extract odsiewa placeholder players.mp4.
|
||||
"javflix": javflix.extract,
|
||||
# neporn — KVS function/0 + license (jak freshporno). Server-side _kvs resolve →
|
||||
# data001.neporn.com/remote_control.php portable (cross-IP 206, 2026-06-10).
|
||||
"neporncom": neporn.extract,
|
||||
|
|
|
|||
|
|
@ -124,7 +124,7 @@ _JS_SERVER_URL_RE = re.compile(
|
|||
_ANCHOR_HOSTER_RE = re.compile(
|
||||
r'<a\s+[^>]*href=["\'](?P<url>https?://(?:'
|
||||
r'playmogo|luluvid|doodporn|doodstream|dood\.[a-z]+|streamtape|streamta\.pe|'
|
||||
r'filemoon|streamwish|sdefx|veev|turbovidhls|gounlimited|iceyfile|hlswish|'
|
||||
r'filemoon|emturbovid|streamwish|sdefx|veev|turbovidhls|gounlimited|iceyfile|hlswish|'
|
||||
r'mixdrop|voe|vidoza|mediafire|asnwish|obeywish|streamruby|hqq\.[a-z]+|'
|
||||
r'feurl|streamhide|krakenfiles|earnvids|jollytuna|peekvids|playerwish'
|
||||
r')\.[a-z]{2,8}/[^"\']+)["\']',
|
||||
|
|
|
|||
19
app/extractors/tubes/javflix.py
Normal file
19
app/extractors/tubes/javflix.py
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
"""javflix.cc extractor — cienki wrapper na generyczny _embed_iframe.
|
||||
|
||||
javflix trzyma hostery w `<a class="myLink" href="<embed>">` (streamtape/voe/doodstream/
|
||||
emturbovid), które _embed_iframe łapie anchor-hoster patternem. Wrapper odsiewa tylko
|
||||
placeholder `players.mp4` (pusty iframe zanim JS podmieni src) — bez tego trafiał jako
|
||||
martwe pierwsze źródło type='mp4'.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from app.extractors._models import StreamSource
|
||||
from app.extractors.tubes import _embed_iframe
|
||||
|
||||
|
||||
def extract(page_url: str, *, timeout: float = 60.0) -> list[StreamSource] | None:
|
||||
srcs = _embed_iframe.extract(page_url, timeout=timeout)
|
||||
if not srcs:
|
||||
return None
|
||||
srcs = [s for s in srcs if "players.mp4" not in s.link]
|
||||
return srcs or None
|
||||
Loading…
Add table
Reference in a new issue