feat(jav): javflix.cc scraper + extractor (first JAV source, gated off main feed)
First site of the JAV vertical (user request, separate section). javflix.cc is a WordPress JAV aggregator, VPS-reachable and server-rendered. Scraper parses itemprop metadata (title, thumbnail, uploadDate; no duration) and the JAV code (BKD-368 style) as a tag. Playback: server buttons are <a class="myLink" href="<embed>"> (streamtape, voe, doodstream, emturbovid) whose href sits in the raw HTML; the generic _embed_iframe extractor already resolves those via its anchor-hoster pattern, so javflix registers under sitetag "javflix" with a thin wrapper that drops the players.mp4 placeholder. Added emturbovid to the anchor-hoster host list. Verified end-to-end on prod: scrape a listing -> RawScene with metadata + code tag, resolve -> emturbovid/voe/doodstream hoster sources (all Goon-playable). NOT registered in ALL_BROWSE_SCRAPERS yet. JAV is a separate vertical (user decision) that must be gated out of the main scenes feed before ingest so it does not flood the western catalog. Next: feed gating (exclude JAV origins by default) + a mobile JAV tab + on-device playback test, then enable scheduled ingest. jav.guru / vjav / supjav follow the same pattern. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
parent
2d3831bd53
commit
d8d00295e1
4 changed files with 120 additions and 1 deletions
94
app/connectors/direct_scrapers/javflix.py
Normal file
94
app/connectors/direct_scrapers/javflix.py
Normal file
|
|
@ -0,0 +1,94 @@
|
||||||
|
"""javflix.cc — JAV browse scraper (WordPress, English-subbed JAV).
|
||||||
|
|
||||||
|
Osobna pula JAV (kody typu BKD-368, tytuły azjatyckie), origin `tube:javflix`.
|
||||||
|
NIE deduplikuje się z zachodnim katalogiem — świadomie orphan vertical (sekcja JAV).
|
||||||
|
|
||||||
|
Struktura (RE 2026-07-10):
|
||||||
|
- listing: `/page/N/` (WordPress archive), posty pod `/<slug>/` (np. /bkd-368-english-subtitle/)
|
||||||
|
- detail: metadane w `itemprop` (name/thumbnailUrl/uploadDate; BRAK duration),
|
||||||
|
playback = przyciski serwerów `<a class="myLink" name="<hoster>" href="<embed>">`
|
||||||
|
(streamtape/voe/doodstream/emturbovid). Href jest w SUROWYM HTML (theme strippuje go
|
||||||
|
po renderze JS, ale scraper widzi surowy). Embed-hostery obsługuje generyczny
|
||||||
|
ekstraktor `_embed_iframe` (anchor-hoster pattern) → rejestrujemy go pod `javflix`.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import re
|
||||||
|
|
||||||
|
from app.connectors.base import RawPlaybackSource, RawScene, RawTag
|
||||||
|
from app.connectors.direct_scrapers._browse_base import BaseBrowseScraper
|
||||||
|
from app.connectors.direct_scrapers._playtube import _parse_iso_date
|
||||||
|
|
||||||
|
_BASE = "https://javflix.cc"
|
||||||
|
# Posty to `javflix.cc/<slug>/`. Odsiewamy strony nie-postowe (taksonomie, statyczne).
|
||||||
|
_POST_RE = re.compile(r'href="(https://javflix\.cc/[a-z0-9][a-z0-9\-]{4,}/)"', re.IGNORECASE)
|
||||||
|
_NON_POST = (
|
||||||
|
"/page/", "/category/", "/categories/", "/genre/", "/maker/", "/actress/",
|
||||||
|
"/actors/", "/tag/", "/tags/", "/studio/", "/label/", "/series/", "/wp-",
|
||||||
|
"/18-usc", "/dmca", "/contact", "/privacy", "/about", "/2257",
|
||||||
|
)
|
||||||
|
_CODE_RE = re.compile(r"^([a-z]+-?\d+[a-z]?)", re.IGNORECASE)
|
||||||
|
|
||||||
|
|
||||||
|
def _itemprop(html: str, name: str) -> str | None:
|
||||||
|
m = re.search(
|
||||||
|
rf'itemprop="{name}"\s+content="([^"]+)"', html, re.IGNORECASE
|
||||||
|
)
|
||||||
|
return m.group(1).strip() if m else None
|
||||||
|
|
||||||
|
|
||||||
|
class JavflixScraper(BaseBrowseScraper):
|
||||||
|
sitetag = "javflix"
|
||||||
|
|
||||||
|
def _listing_url(self, page: int) -> str:
|
||||||
|
return f"{_BASE}/" if page <= 1 else f"{_BASE}/page/{page}/"
|
||||||
|
|
||||||
|
def _extract_scene_urls(self, listing_html: str) -> list[str]:
|
||||||
|
seen: set[str] = set()
|
||||||
|
out: list[str] = []
|
||||||
|
for m in _POST_RE.finditer(listing_html):
|
||||||
|
url = m.group(1)
|
||||||
|
if any(x in url for x in _NON_POST):
|
||||||
|
continue
|
||||||
|
if url.rstrip("/") == _BASE:
|
||||||
|
continue
|
||||||
|
if url not in seen:
|
||||||
|
seen.add(url)
|
||||||
|
out.append(url)
|
||||||
|
return out
|
||||||
|
|
||||||
|
def _parse_detail(self, scene_url: str, detail_html: str) -> RawScene | None:
|
||||||
|
title = _itemprop(detail_html, "name")
|
||||||
|
if not title:
|
||||||
|
tm = re.search(r"<title>([^<]+)</title>", detail_html)
|
||||||
|
title = tm.group(1).split(" – ")[0].strip() if tm else None
|
||||||
|
if not title:
|
||||||
|
return None
|
||||||
|
|
||||||
|
thumb = _itemprop(detail_html, "thumbnailUrl")
|
||||||
|
up = _itemprop(detail_html, "uploadDate")
|
||||||
|
release_date = _parse_iso_date(up) if up else None
|
||||||
|
|
||||||
|
# JAV code (BKD-368) ze sluga — kanoniczny identyfikator, dodajemy jako tag
|
||||||
|
# (searchable) bo javflix nie ma osobnego pola kodu.
|
||||||
|
slug = scene_url.rstrip("/").rsplit("/", 1)[-1]
|
||||||
|
cm = _CODE_RE.match(slug)
|
||||||
|
tags: list[RawTag] = []
|
||||||
|
if cm:
|
||||||
|
code = cm.group(1).upper()
|
||||||
|
tags.append(RawTag(external_id=f"javcode:{code}", name=code, slug=cm.group(1).lower()))
|
||||||
|
|
||||||
|
return RawScene(
|
||||||
|
external_id=f"{self.sitetag}:{scene_url}",
|
||||||
|
title=title,
|
||||||
|
release_date=release_date,
|
||||||
|
url=scene_url,
|
||||||
|
tags=tags,
|
||||||
|
playback_sources=[
|
||||||
|
RawPlaybackSource(
|
||||||
|
origin=f"tube:{self.sitetag}",
|
||||||
|
page_url=scene_url,
|
||||||
|
thumbnail_url=thumb,
|
||||||
|
)
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
@ -33,6 +33,7 @@ from app.extractors.tubes import (
|
||||||
hdporngg,
|
hdporngg,
|
||||||
hqfap,
|
hqfap,
|
||||||
hqporner,
|
hqporner,
|
||||||
|
javflix,
|
||||||
neporn,
|
neporn,
|
||||||
latestpornvideo,
|
latestpornvideo,
|
||||||
paradisehill,
|
paradisehill,
|
||||||
|
|
@ -170,6 +171,11 @@ _REGISTRY: dict[str, Callable[[str], list[StreamSource] | None]] = {
|
||||||
# strona wróciła na CDN vstor.top z realnymi plikami (portable cross-IP, zweryfikowane),
|
# strona wróciła na CDN vstor.top z realnymi plikami (portable cross-IP, zweryfikowane),
|
||||||
# user request. 4k69 zostaje usunięty (nie sprawdzany ponownie).
|
# user request. 4k69 zostaje usunięty (nie sprawdzany ponownie).
|
||||||
"hqfapcom": hqfap.extract,
|
"hqfapcom": hqfap.extract,
|
||||||
|
# javflix (JAV, WordPress) — przyciski serwerów to `<a class="myLink" href="<embed>">`
|
||||||
|
# (streamtape/voe/doodstream/emturbovid). Generyczny _embed_iframe łapie je anchor-hoster
|
||||||
|
# patternem → type='hoster', telefon resolwuje (voe backend, dood/filemoon phone-side).
|
||||||
|
# Wrapper javflix.extract odsiewa placeholder players.mp4.
|
||||||
|
"javflix": javflix.extract,
|
||||||
# neporn — KVS function/0 + license (jak freshporno). Server-side _kvs resolve →
|
# neporn — KVS function/0 + license (jak freshporno). Server-side _kvs resolve →
|
||||||
# data001.neporn.com/remote_control.php portable (cross-IP 206, 2026-06-10).
|
# data001.neporn.com/remote_control.php portable (cross-IP 206, 2026-06-10).
|
||||||
"neporncom": neporn.extract,
|
"neporncom": neporn.extract,
|
||||||
|
|
|
||||||
|
|
@ -124,7 +124,7 @@ _JS_SERVER_URL_RE = re.compile(
|
||||||
_ANCHOR_HOSTER_RE = re.compile(
|
_ANCHOR_HOSTER_RE = re.compile(
|
||||||
r'<a\s+[^>]*href=["\'](?P<url>https?://(?:'
|
r'<a\s+[^>]*href=["\'](?P<url>https?://(?:'
|
||||||
r'playmogo|luluvid|doodporn|doodstream|dood\.[a-z]+|streamtape|streamta\.pe|'
|
r'playmogo|luluvid|doodporn|doodstream|dood\.[a-z]+|streamtape|streamta\.pe|'
|
||||||
r'filemoon|streamwish|sdefx|veev|turbovidhls|gounlimited|iceyfile|hlswish|'
|
r'filemoon|emturbovid|streamwish|sdefx|veev|turbovidhls|gounlimited|iceyfile|hlswish|'
|
||||||
r'mixdrop|voe|vidoza|mediafire|asnwish|obeywish|streamruby|hqq\.[a-z]+|'
|
r'mixdrop|voe|vidoza|mediafire|asnwish|obeywish|streamruby|hqq\.[a-z]+|'
|
||||||
r'feurl|streamhide|krakenfiles|earnvids|jollytuna|peekvids|playerwish'
|
r'feurl|streamhide|krakenfiles|earnvids|jollytuna|peekvids|playerwish'
|
||||||
r')\.[a-z]{2,8}/[^"\']+)["\']',
|
r')\.[a-z]{2,8}/[^"\']+)["\']',
|
||||||
|
|
|
||||||
19
app/extractors/tubes/javflix.py
Normal file
19
app/extractors/tubes/javflix.py
Normal file
|
|
@ -0,0 +1,19 @@
|
||||||
|
"""javflix.cc extractor — cienki wrapper na generyczny _embed_iframe.
|
||||||
|
|
||||||
|
javflix trzyma hostery w `<a class="myLink" href="<embed>">` (streamtape/voe/doodstream/
|
||||||
|
emturbovid), które _embed_iframe łapie anchor-hoster patternem. Wrapper odsiewa tylko
|
||||||
|
placeholder `players.mp4` (pusty iframe zanim JS podmieni src) — bez tego trafiał jako
|
||||||
|
martwe pierwsze źródło type='mp4'.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from app.extractors._models import StreamSource
|
||||||
|
from app.extractors.tubes import _embed_iframe
|
||||||
|
|
||||||
|
|
||||||
|
def extract(page_url: str, *, timeout: float = 60.0) -> list[StreamSource] | None:
|
||||||
|
srcs = _embed_iframe.extract(page_url, timeout=timeout)
|
||||||
|
if not srcs:
|
||||||
|
return None
|
||||||
|
srcs = [s for s in srcs if "players.mp4" not in s.link]
|
||||||
|
return srcs or None
|
||||||
Loading…
Add table
Reference in a new issue