4df1e01b31
JAV is a distinct vertical (user decision): Asian codes/titles that do not dedup
against the western catalog, so they must not flood the main feed.
Backend (scenes.py): JAV_ORIGINS = {tube:javflix, tube:javguru, tube:vjav,
tube:supjav}; list_scenes gains a `jav` param. Default (jav=false) excludes any
scene with a live JAV-origin source; jav=true returns only those. The cached
default-count and _is_pure_default also exclude JAV so the main feed count matches.
JavflixScraper is now registered in ALL_BROWSE_SCRAPERS (scheduled ingest lands in
the JAV section, gated). Scraper hardened: requires a real server button
(class="myLink") so static pages (Terms/FAQ) are skipped, and unescapes HTML
entities in the title.
Mobile: a "JAV" top tab reuses ScenesScreen with { jav: true } (route param ->
listScenes jav=true). The 60s minimum-duration default is disabled in the JAV tab
because javflix does not expose duration (NULL >= 60 would hide the whole section).
Verified on prod: 16 javflix scenes appear only in the JAV feed and are excluded
from the 2.29M main feed; playback resolves to voe/doodstream/emturbovid.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
101 lines
4 KiB
Python
101 lines
4 KiB
Python
"""javflix.cc — JAV browse scraper (WordPress, English-subbed JAV).
|
||
|
||
Osobna pula JAV (kody typu BKD-368, tytuły azjatyckie), origin `tube:javflix`.
|
||
NIE deduplikuje się z zachodnim katalogiem — świadomie orphan vertical (sekcja JAV).
|
||
|
||
Struktura (RE 2026-07-10):
|
||
- listing: `/page/N/` (WordPress archive), posty pod `/<slug>/` (np. /bkd-368-english-subtitle/)
|
||
- detail: metadane w `itemprop` (name/thumbnailUrl/uploadDate; BRAK duration),
|
||
playback = przyciski serwerów `<a class="myLink" name="<hoster>" href="<embed>">`
|
||
(streamtape/voe/doodstream/emturbovid). Href jest w SUROWYM HTML (theme strippuje go
|
||
po renderze JS, ale scraper widzi surowy). Embed-hostery obsługuje generyczny
|
||
ekstraktor `_embed_iframe` (anchor-hoster pattern) → rejestrujemy go pod `javflix`.
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import html
|
||
import re
|
||
|
||
from app.connectors.base import RawPlaybackSource, RawScene, RawTag
|
||
from app.connectors.direct_scrapers._browse_base import BaseBrowseScraper
|
||
from app.connectors.direct_scrapers._playtube import _parse_iso_date
|
||
|
||
_BASE = "https://javflix.cc"
|
||
# Posty to `javflix.cc/<slug>/`. Odsiewamy strony nie-postowe (taksonomie, statyczne).
|
||
_POST_RE = re.compile(r'href="(https://javflix\.cc/[a-z0-9][a-z0-9\-]{4,}/)"', re.IGNORECASE)
|
||
_NON_POST = (
|
||
"/page/", "/category/", "/categories/", "/genre/", "/maker/", "/actress/",
|
||
"/actors/", "/tag/", "/tags/", "/studio/", "/label/", "/series/", "/wp-",
|
||
"/18-usc", "/dmca", "/contact", "/privacy", "/about", "/2257",
|
||
"/terms", "/faq", "/policy", "/disclaimer", "/sitemap",
|
||
)
|
||
_CODE_RE = re.compile(r"^([a-z]+-?\d+[a-z]?)", re.IGNORECASE)
|
||
|
||
|
||
def _itemprop(html: str, name: str) -> str | None:
|
||
m = re.search(
|
||
rf'itemprop="{name}"\s+content="([^"]+)"', html, re.IGNORECASE
|
||
)
|
||
return m.group(1).strip() if m else None
|
||
|
||
|
||
class JavflixScraper(BaseBrowseScraper):
|
||
sitetag = "javflix"
|
||
|
||
def _listing_url(self, page: int) -> str:
|
||
return f"{_BASE}/" if page <= 1 else f"{_BASE}/page/{page}/"
|
||
|
||
def _extract_scene_urls(self, listing_html: str) -> list[str]:
|
||
seen: set[str] = set()
|
||
out: list[str] = []
|
||
for m in _POST_RE.finditer(listing_html):
|
||
url = m.group(1)
|
||
if any(x in url for x in _NON_POST):
|
||
continue
|
||
if url.rstrip("/") == _BASE:
|
||
continue
|
||
if url not in seen:
|
||
seen.add(url)
|
||
out.append(url)
|
||
return out
|
||
|
||
def _parse_detail(self, scene_url: str, detail_html: str) -> RawScene | None:
|
||
# Prawdziwy post video ma przyciski serwerów (`class="myLink"`). Strony statyczne
|
||
# (Terms/FAQ/DMCA) ich nie mają → pomijamy (URL-filter nie łapie wszystkich).
|
||
if 'class="myLink"' not in detail_html:
|
||
return None
|
||
title = _itemprop(detail_html, "name")
|
||
if not title:
|
||
tm = re.search(r"<title>([^<]+)</title>", detail_html)
|
||
title = tm.group(1).split(" – ")[0].strip() if tm else None
|
||
if not title:
|
||
return None
|
||
title = html.unescape(title).strip()
|
||
|
||
thumb = _itemprop(detail_html, "thumbnailUrl")
|
||
up = _itemprop(detail_html, "uploadDate")
|
||
release_date = _parse_iso_date(up) if up else None
|
||
|
||
# JAV code (BKD-368) ze sluga — kanoniczny identyfikator, dodajemy jako tag
|
||
# (searchable) bo javflix nie ma osobnego pola kodu.
|
||
slug = scene_url.rstrip("/").rsplit("/", 1)[-1]
|
||
cm = _CODE_RE.match(slug)
|
||
tags: list[RawTag] = []
|
||
if cm:
|
||
code = cm.group(1).upper()
|
||
tags.append(RawTag(external_id=f"javcode:{code}", name=code, slug=cm.group(1).lower()))
|
||
|
||
return RawScene(
|
||
external_id=f"{self.sitetag}:{scene_url}",
|
||
title=title,
|
||
release_date=release_date,
|
||
url=scene_url,
|
||
tags=tags,
|
||
playback_sources=[
|
||
RawPlaybackSource(
|
||
origin=f"tube:{self.sitetag}",
|
||
page_url=scene_url,
|
||
thumbnail_url=thumb,
|
||
)
|
||
],
|
||
)
|