feat(jav): separate JAV section (tab + feed gating) + enable javflix ingest
Some checks are pending
Backend tests / test (push) Waiting to run
Some checks are pending
Backend tests / test (push) Waiting to run
JAV is a distinct vertical (user decision): Asian codes/titles that do not dedup
against the western catalog, so they must not flood the main feed.
Backend (scenes.py): JAV_ORIGINS = {tube:javflix, tube:javguru, tube:vjav,
tube:supjav}; list_scenes gains a `jav` param. Default (jav=false) excludes any
scene with a live JAV-origin source; jav=true returns only those. The cached
default-count and _is_pure_default also exclude JAV so the main feed count matches.
JavflixScraper is now registered in ALL_BROWSE_SCRAPERS (scheduled ingest lands in
the JAV section, gated). Scraper hardened: requires a real server button
(class="myLink") so static pages (Terms/FAQ) are skipped, and unescapes HTML
entities in the title.
Mobile: a "JAV" top tab reuses ScenesScreen with { jav: true } (route param ->
listScenes jav=true). The 60s minimum-duration default is disabled in the JAV tab
because javflix does not expose duration (NULL >= 60 would hide the whole section).
Verified on prod: 16 javflix scenes appear only in the JAV feed and are excluded
from the 2.29M main feed; playback resolves to voe/doodstream/emturbovid.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
parent
18057e46d9
commit
7828cfcfa2
8 changed files with 90 additions and 7 deletions
|
|
@ -59,7 +59,9 @@ def _default_scene_count(session: Session) -> int:
|
||||||
PlaybackSource.scene_id == Scene.id,
|
PlaybackSource.scene_id == Scene.id,
|
||||||
PlaybackSource.dead_at.is_(None),
|
PlaybackSource.dead_at.is_(None),
|
||||||
)
|
)
|
||||||
)
|
),
|
||||||
|
# Domyślny feed wyklucza JAV (osobna sekcja) → licznik też.
|
||||||
|
~_jav_source_exists(),
|
||||||
).subquery()
|
).subquery()
|
||||||
)
|
)
|
||||||
total = session.execute(count_query).scalar_one()
|
total = session.execute(count_query).scalar_one()
|
||||||
|
|
@ -68,6 +70,23 @@ def _default_scene_count(session: Session) -> int:
|
||||||
return total
|
return total
|
||||||
|
|
||||||
|
|
||||||
|
# JAV vertical — osobna sekcja. Sceny z tych originów NIE wchodzą do głównego feedu
|
||||||
|
# (domyślnie wykluczone), tylko do zakładki JAV (?jav=true). JAV to osobny namespace
|
||||||
|
# (kody typu BKD-368, azjatyckie tytuły), nie deduplikuje się z zachodnim katalogiem,
|
||||||
|
# więc scena JAV ma WYŁĄCZNIE origin JAV → wykluczenie/inkluzja po tym originie jest pewna.
|
||||||
|
JAV_ORIGINS = ("tube:javflix", "tube:javguru", "tube:vjav", "tube:supjav")
|
||||||
|
|
||||||
|
|
||||||
|
def _jav_source_exists():
|
||||||
|
return exists(
|
||||||
|
select(1).where(
|
||||||
|
PlaybackSource.scene_id == Scene.id,
|
||||||
|
PlaybackSource.dead_at.is_(None),
|
||||||
|
PlaybackSource.origin.in_(JAV_ORIGINS),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
# Blacklisty (performer/studio/tag) są zwykle PUSTE (self-hosted, single-user). Mimo to
|
# Blacklisty (performer/studio/tag) są zwykle PUSTE (self-hosted, single-user). Mimo to
|
||||||
# 3 NOT EXISTS klauzule doklejały się do KAŻDEJ filtrowanej listy scen i były ewaluowane
|
# 3 NOT EXISTS klauzule doklejały się do KAŻDEJ filtrowanej listy scen i były ewaluowane
|
||||||
# per-row — przy filtrze typu duży-tag/has_playback planer chodzi po ~176k scen, więc te
|
# per-row — przy filtrze typu duży-tag/has_playback planer chodzi po ~176k scen, więc te
|
||||||
|
|
@ -222,6 +241,13 @@ def list_scenes(
|
||||||
"z jedynym playback z hqporner (~7-min Brazzers trailer clipy zalewają katalog)."
|
"z jedynym playback z hqporner (~7-min Brazzers trailer clipy zalewają katalog)."
|
||||||
),
|
),
|
||||||
),
|
),
|
||||||
|
jav: bool = Query(
|
||||||
|
default=False,
|
||||||
|
description=(
|
||||||
|
"False (default): ukrywa sceny JAV (osobny vertical). True: TYLKO sceny JAV. "
|
||||||
|
"JAV to osobna sekcja w apce, nie zalewa głównego feedu (origin javflix/javguru/vjav/supjav)."
|
||||||
|
),
|
||||||
|
),
|
||||||
sort: str = Query(default="created_at", description="created_at|release_date|title|studio"),
|
sort: str = Query(default="created_at", description="created_at|release_date|title|studio"),
|
||||||
page: int = Query(default=1, ge=1),
|
page: int = Query(default=1, ge=1),
|
||||||
per_page: int = Query(default=50, ge=1, le=200),
|
per_page: int = Query(default=50, ge=1, le=200),
|
||||||
|
|
@ -322,6 +348,13 @@ def list_scenes(
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# JAV vertical gate: domyślnie JAV wykluczone z głównego feedu; ?jav=true → TYLKO JAV.
|
||||||
|
# (origin=... diagnostyka nadal podlega bramie — do JAV użyj jav=true.)
|
||||||
|
if jav:
|
||||||
|
base = base.where(_jav_source_exists())
|
||||||
|
else:
|
||||||
|
base = base.where(~_jav_source_exists())
|
||||||
|
|
||||||
# Blacklisty device (performer/studio/tag) — globalne wykluczenia, współdzielone z
|
# Blacklisty device (performer/studio/tag) — globalne wykluczenia, współdzielone z
|
||||||
# licznikiem +N ulubionych. Puste blacklisty → [] (typowy single-user, zero kosztu).
|
# licznikiem +N ulubionych. Puste blacklisty → [] (typowy single-user, zero kosztu).
|
||||||
for _bl_clause in blacklist_clauses(session, device_id):
|
for _bl_clause in blacklist_clauses(session, device_id):
|
||||||
|
|
@ -373,7 +406,7 @@ def list_scenes(
|
||||||
and not perf_id_strings and origin is None and has_playback is None
|
and not perf_id_strings and origin is None and has_playback is None
|
||||||
and min_duration_sec is None
|
and min_duration_sec is None
|
||||||
and max_duration_sec is None and released_within_days is None
|
and max_duration_sec is None and released_within_days is None
|
||||||
and min_quality_p is None
|
and min_quality_p is None and not jav
|
||||||
)
|
)
|
||||||
# Count strategy:
|
# Count strategy:
|
||||||
# - PURE default: cached pełny licznik katalogu (TTL 10 min).
|
# - PURE default: cached pełny licznik katalogu (TTL 10 min).
|
||||||
|
|
|
||||||
|
|
@ -135,6 +135,7 @@ from app.connectors.direct_scrapers.yesporn import YesPornVipScraper # noqa: E4
|
||||||
from app.connectors.direct_scrapers.fullmovies import FullmoviesScraper # noqa: E402
|
from app.connectors.direct_scrapers.fullmovies import FullmoviesScraper # noqa: E402
|
||||||
from app.connectors.direct_scrapers.hdporngg import HDPornGGScraper # noqa: E402
|
from app.connectors.direct_scrapers.hdporngg import HDPornGGScraper # noqa: E402
|
||||||
from app.connectors.direct_scrapers.hqfap import HQFapScraper # noqa: E402
|
from app.connectors.direct_scrapers.hqfap import HQFapScraper # noqa: E402
|
||||||
|
from app.connectors.direct_scrapers.javflix import JavflixScraper # noqa: E402
|
||||||
from app.connectors.direct_scrapers.neporn import NepornScraper # noqa: E402
|
from app.connectors.direct_scrapers.neporn import NepornScraper # noqa: E402
|
||||||
from app.connectors.direct_scrapers.superporn import SuperpornScraper # noqa: E402
|
from app.connectors.direct_scrapers.superporn import SuperpornScraper # noqa: E402
|
||||||
from app.connectors.direct_scrapers.eporner_api import EpornerApiScraper # noqa: E402
|
from app.connectors.direct_scrapers.eporner_api import EpornerApiScraper # noqa: E402
|
||||||
|
|
@ -218,6 +219,9 @@ ALL_BROWSE_SCRAPERS: list[type[BaseBrowseScraper]] = [
|
||||||
# bo strona wróciła na CDN vstor.top z realnymi plikami (portable cross-IP), user request.
|
# bo strona wróciła na CDN vstor.top z realnymi plikami (portable cross-IP), user request.
|
||||||
HQFapScraper,
|
HQFapScraper,
|
||||||
# FourK69Scraper — USUNIĘTY 2026-06-25 (ten sam stub), NIE sprawdzany ponownie.
|
# FourK69Scraper — USUNIĘTY 2026-06-25 (ten sam stub), NIE sprawdzany ponownie.
|
||||||
|
# JavflixScraper — JAV vertical (osobna sekcja). origin tube:javflix jest w
|
||||||
|
# JAV_ORIGINS → list_scenes wyklucza je z głównego feedu (tylko zakładka JAV).
|
||||||
|
JavflixScraper,
|
||||||
# NepornScraper — dołączony 2026-06-10 (user request). KVS engine (jak freshporno/
|
# NepornScraper — dołączony 2026-06-10 (user request). KVS engine (jak freshporno/
|
||||||
# porn00), /latest-updates/N/. JSON-LD (title+desc+uploadDate+thumb) + video:duration
|
# porn00), /latest-updates/N/. JSON-LD (title+desc+uploadDate+thumb) + video:duration
|
||||||
# meta + /models/ performerzy + /categories/ tagi. Brak studio (tytuł bywa
|
# meta + /models/ performerzy + /categories/ tagi. Brak studio (tytuł bywa
|
||||||
|
|
|
||||||
|
|
@ -13,6 +13,7 @@ Struktura (RE 2026-07-10):
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import html
|
||||||
import re
|
import re
|
||||||
|
|
||||||
from app.connectors.base import RawPlaybackSource, RawScene, RawTag
|
from app.connectors.base import RawPlaybackSource, RawScene, RawTag
|
||||||
|
|
@ -26,6 +27,7 @@ _NON_POST = (
|
||||||
"/page/", "/category/", "/categories/", "/genre/", "/maker/", "/actress/",
|
"/page/", "/category/", "/categories/", "/genre/", "/maker/", "/actress/",
|
||||||
"/actors/", "/tag/", "/tags/", "/studio/", "/label/", "/series/", "/wp-",
|
"/actors/", "/tag/", "/tags/", "/studio/", "/label/", "/series/", "/wp-",
|
||||||
"/18-usc", "/dmca", "/contact", "/privacy", "/about", "/2257",
|
"/18-usc", "/dmca", "/contact", "/privacy", "/about", "/2257",
|
||||||
|
"/terms", "/faq", "/policy", "/disclaimer", "/sitemap",
|
||||||
)
|
)
|
||||||
_CODE_RE = re.compile(r"^([a-z]+-?\d+[a-z]?)", re.IGNORECASE)
|
_CODE_RE = re.compile(r"^([a-z]+-?\d+[a-z]?)", re.IGNORECASE)
|
||||||
|
|
||||||
|
|
@ -58,12 +60,17 @@ class JavflixScraper(BaseBrowseScraper):
|
||||||
return out
|
return out
|
||||||
|
|
||||||
def _parse_detail(self, scene_url: str, detail_html: str) -> RawScene | None:
|
def _parse_detail(self, scene_url: str, detail_html: str) -> RawScene | None:
|
||||||
|
# Prawdziwy post video ma przyciski serwerów (`class="myLink"`). Strony statyczne
|
||||||
|
# (Terms/FAQ/DMCA) ich nie mają → pomijamy (URL-filter nie łapie wszystkich).
|
||||||
|
if 'class="myLink"' not in detail_html:
|
||||||
|
return None
|
||||||
title = _itemprop(detail_html, "name")
|
title = _itemprop(detail_html, "name")
|
||||||
if not title:
|
if not title:
|
||||||
tm = re.search(r"<title>([^<]+)</title>", detail_html)
|
tm = re.search(r"<title>([^<]+)</title>", detail_html)
|
||||||
title = tm.group(1).split(" – ")[0].strip() if tm else None
|
title = tm.group(1).split(" – ")[0].strip() if tm else None
|
||||||
if not title:
|
if not title:
|
||||||
return None
|
return None
|
||||||
|
title = html.unescape(title).strip()
|
||||||
|
|
||||||
thumb = _itemprop(detail_html, "thumbnailUrl")
|
thumb = _itemprop(detail_html, "thumbnailUrl")
|
||||||
up = _itemprop(detail_html, "uploadDate")
|
up = _itemprop(detail_html, "uploadDate")
|
||||||
|
|
|
||||||
|
|
@ -134,6 +134,7 @@ export class GoonClient {
|
||||||
qs.set('released_within_days', String(params.released_within_days));
|
qs.set('released_within_days', String(params.released_within_days));
|
||||||
if (params.min_quality_p !== undefined) qs.set('min_quality_p', String(params.min_quality_p));
|
if (params.min_quality_p !== undefined) qs.set('min_quality_p', String(params.min_quality_p));
|
||||||
if (params.include_stubs !== undefined) qs.set('include_stubs', String(params.include_stubs));
|
if (params.include_stubs !== undefined) qs.set('include_stubs', String(params.include_stubs));
|
||||||
|
if (params.jav) qs.set('jav', 'true');
|
||||||
if (params.origin) qs.set('origin', params.origin);
|
if (params.origin) qs.set('origin', params.origin);
|
||||||
if (params.sort) qs.set('sort', params.sort);
|
if (params.sort) qs.set('sort', params.sort);
|
||||||
qs.set('page', String(params.page ?? 1));
|
qs.set('page', String(params.page ?? 1));
|
||||||
|
|
|
||||||
|
|
@ -16,6 +16,13 @@ export type ChangelogEntry = {
|
||||||
};
|
};
|
||||||
|
|
||||||
export const CHANGELOG: ChangelogEntry[] = [
|
export const CHANGELOG: ChangelogEntry[] = [
|
||||||
|
{
|
||||||
|
id: '2026-07-10b',
|
||||||
|
date: 'July 2026',
|
||||||
|
items: [
|
||||||
|
'New JAV tab: a separate section for Japanese adult video (English-subtitled). It is kept out of the main Scenes feed, so the main list stays as it was.',
|
||||||
|
],
|
||||||
|
},
|
||||||
{
|
{
|
||||||
id: '2026-07-10',
|
id: '2026-07-10',
|
||||||
date: 'July 2026',
|
date: 'July 2026',
|
||||||
|
|
|
||||||
|
|
@ -33,6 +33,8 @@ export type RootStackParamList = {
|
||||||
Scenes: undefined;
|
Scenes: undefined;
|
||||||
Movies: undefined;
|
Movies: undefined;
|
||||||
Sites: undefined;
|
Sites: undefined;
|
||||||
|
// JAV: osobna sekcja — ScenesScreen z paramem jav → backend ?jav=true.
|
||||||
|
JAV: { jav: true } | undefined;
|
||||||
// `origin`: raw playback_source.origin (np. 'tube:hqpornercom'). Idzie do
|
// `origin`: raw playback_source.origin (np. 'tube:hqpornercom'). Idzie do
|
||||||
// listScenes({origin}) — backend robi substring match. `name`: display name
|
// listScenes({origin}) — backend robi substring match. `name`: display name
|
||||||
// do title bara (np. 'hqporner.com').
|
// do title bara (np. 'hqporner.com').
|
||||||
|
|
@ -97,7 +99,7 @@ export type RootStackParamList = {
|
||||||
|
|
||||||
const Stack = createNativeStackNavigator<RootStackParamList>();
|
const Stack = createNativeStackNavigator<RootStackParamList>();
|
||||||
|
|
||||||
type TopTab = 'Scenes' | 'Movies' | 'Sites';
|
type TopTab = 'Scenes' | 'Movies' | 'Sites' | 'JAV';
|
||||||
|
|
||||||
function TopTabs({
|
function TopTabs({
|
||||||
current,
|
current,
|
||||||
|
|
@ -106,7 +108,7 @@ function TopTabs({
|
||||||
current: TopTab;
|
current: TopTab;
|
||||||
onNavigate: (tab: TopTab) => void;
|
onNavigate: (tab: TopTab) => void;
|
||||||
}) {
|
}) {
|
||||||
const tabs: TopTab[] = ['Scenes', 'Movies', 'Sites'];
|
const tabs: TopTab[] = ['Scenes', 'Movies', 'Sites', 'JAV'];
|
||||||
return (
|
return (
|
||||||
<View style={{ flexDirection: 'row', gap: 10, paddingHorizontal: 10, alignItems: 'center' }}>
|
<View style={{ flexDirection: 'row', gap: 10, paddingHorizontal: 10, alignItems: 'center' }}>
|
||||||
{/* Logo (mark) — spójne z ekranem logowania; węższe od wordmarku, więc Scenes/Movies/Sites
|
{/* Logo (mark) — spójne z ekranem logowania; węższe od wordmarku, więc Scenes/Movies/Sites
|
||||||
|
|
@ -241,6 +243,28 @@ export function AppNavigator({ onLogout, client, appVersion }: AppNavigatorProps
|
||||||
),
|
),
|
||||||
})}
|
})}
|
||||||
/>
|
/>
|
||||||
|
<Stack.Screen
|
||||||
|
name="JAV"
|
||||||
|
component={ScenesScreen}
|
||||||
|
initialParams={{ jav: true }}
|
||||||
|
options={({ navigation }) => ({
|
||||||
|
title: '',
|
||||||
|
headerBackVisible: false,
|
||||||
|
headerLeft: () => (
|
||||||
|
<TopTabs
|
||||||
|
current="JAV"
|
||||||
|
onNavigate={(t) => navigation.replace(t)}
|
||||||
|
/>
|
||||||
|
),
|
||||||
|
headerRight: () => (
|
||||||
|
<View style={{ flexDirection: 'row', gap: 14, alignItems: 'center' }}>
|
||||||
|
<Pressable onPress={onLogout} hitSlop={12}>
|
||||||
|
<Text style={{ fontSize: 17 }}>🚪</Text>
|
||||||
|
</Pressable>
|
||||||
|
</View>
|
||||||
|
),
|
||||||
|
})}
|
||||||
|
/>
|
||||||
<Stack.Screen
|
<Stack.Screen
|
||||||
name="SiteScenes"
|
name="SiteScenes"
|
||||||
component={SiteScenesScreen}
|
component={SiteScenesScreen}
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
import { useNavigation } from '@react-navigation/native';
|
import { useNavigation, useRoute } from '@react-navigation/native';
|
||||||
import type { NativeStackNavigationProp } from '@react-navigation/native-stack';
|
import type { NativeStackNavigationProp } from '@react-navigation/native-stack';
|
||||||
import { useInfiniteQuery, useMutation, useQuery, useQueryClient } from '@tanstack/react-query';
|
import { useInfiniteQuery, useMutation, useQuery, useQueryClient } from '@tanstack/react-query';
|
||||||
import React, { useState } from 'react';
|
import React, { useState } from 'react';
|
||||||
|
|
@ -29,6 +29,9 @@ export function ScenesScreen() {
|
||||||
const client = useClient();
|
const client = useClient();
|
||||||
const { gridColumns } = usePreferences();
|
const { gridColumns } = usePreferences();
|
||||||
const navigation = useNavigation<NativeStackNavigationProp<RootStackParamList, 'Scenes'>>();
|
const navigation = useNavigation<NativeStackNavigationProp<RootStackParamList, 'Scenes'>>();
|
||||||
|
// Zakładka JAV reużywa ten ekran z paramem jav → backend ?jav=true (osobny vertical).
|
||||||
|
const route = useRoute();
|
||||||
|
const jav = (route.params as { jav?: boolean } | undefined)?.jav === true;
|
||||||
const [q, setQ] = useState('');
|
const [q, setQ] = useState('');
|
||||||
const [debouncedQ, setDebouncedQ] = useState('');
|
const [debouncedQ, setDebouncedQ] = useState('');
|
||||||
const [filter, setFilter] = useState<FilterState>(DEFAULT_FILTER);
|
const [filter, setFilter] = useState<FilterState>(DEFAULT_FILTER);
|
||||||
|
|
@ -50,7 +53,7 @@ export function ScenesScreen() {
|
||||||
hasNextPage,
|
hasNextPage,
|
||||||
isFetchingNextPage,
|
isFetchingNextPage,
|
||||||
} = useInfiniteQuery({
|
} = useInfiniteQuery({
|
||||||
queryKey: ['scenes', debouncedQ, filter],
|
queryKey: ['scenes', debouncedQ, filter, jav],
|
||||||
queryFn: ({ pageParam = 1 }) =>
|
queryFn: ({ pageParam = 1 }) =>
|
||||||
client.listScenes({
|
client.listScenes({
|
||||||
q: debouncedQ || undefined,
|
q: debouncedQ || undefined,
|
||||||
|
|
@ -61,8 +64,11 @@ export function ScenesScreen() {
|
||||||
sort: filter.sort,
|
sort: filter.sort,
|
||||||
include_stubs: filter.includeStubs || undefined,
|
include_stubs: filter.includeStubs || undefined,
|
||||||
origin: filter.origin.trim() || undefined,
|
origin: filter.origin.trim() || undefined,
|
||||||
|
jav: jav || undefined,
|
||||||
// 0 = "Any" → undefined → backend stosuje bazowy próg 60s. >0 podnosi minimum.
|
// 0 = "Any" → undefined → backend stosuje bazowy próg 60s. >0 podnosi minimum.
|
||||||
min_duration_sec: filter.minDurationSec > 0 ? filter.minDurationSec : undefined,
|
// JAV nie ma duration (javflix nie podaje) → w sekcji JAV wyłączamy próg 60s
|
||||||
|
// (explicit 0), inaczej NULL >= 60 = false i cała sekcja byłaby pusta.
|
||||||
|
min_duration_sec: jav ? 0 : filter.minDurationSec > 0 ? filter.minDurationSec : undefined,
|
||||||
page: pageParam,
|
page: pageParam,
|
||||||
per_page: PER_PAGE,
|
per_page: PER_PAGE,
|
||||||
}),
|
}),
|
||||||
|
|
|
||||||
|
|
@ -151,6 +151,7 @@ export interface ScenesListParams {
|
||||||
min_quality_p?: number;
|
min_quality_p?: number;
|
||||||
include_stubs?: boolean;
|
include_stubs?: boolean;
|
||||||
origin?: string;
|
origin?: string;
|
||||||
|
jav?: boolean;
|
||||||
sort?: ScenesSort;
|
sort?: ScenesSort;
|
||||||
page?: number;
|
page?: number;
|
||||||
per_page?: number;
|
per_page?: number;
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue