feat(jav): separate JAV section (tab + feed gating) + enable javflix ingest
JAV is a distinct vertical (user decision): Asian codes/titles that do not dedup
against the western catalog, so they must not flood the main feed.
Backend (scenes.py): JAV_ORIGINS = {tube:javflix, tube:javguru, tube:vjav,
tube:supjav}; list_scenes gains a `jav` param. Default (jav=false) excludes any
scene with a live JAV-origin source; jav=true returns only those. The cached
default-count and _is_pure_default also exclude JAV so the main feed count matches.
JavflixScraper is now registered in ALL_BROWSE_SCRAPERS (scheduled ingest lands in
the JAV section, gated). Scraper hardened: requires a real server button
(class="myLink") so static pages (Terms/FAQ) are skipped, and unescapes HTML
entities in the title.
Mobile: a "JAV" top tab reuses ScenesScreen with { jav: true } (route param ->
listScenes jav=true). The 60s minimum-duration default is disabled in the JAV tab
because javflix does not expose duration (NULL >= 60 would hide the whole section).
Verified on prod: 16 javflix scenes appear only in the JAV feed and are excluded
from the 2.29M main feed; playback resolves to voe/doodstream/emturbovid.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
goon-foss
parent
d8d00295e1
commit
4df1e01b31
8 changed files with 90 additions and 7 deletions
|
|
@ -59,7 +59,9 @@ def _default_scene_count(session: Session) -> int:
|
|||
PlaybackSource.scene_id == Scene.id,
|
||||
PlaybackSource.dead_at.is_(None),
|
||||
)
|
||||
)
|
||||
),
|
||||
# Domyślny feed wyklucza JAV (osobna sekcja) → licznik też.
|
||||
~_jav_source_exists(),
|
||||
).subquery()
|
||||
)
|
||||
total = session.execute(count_query).scalar_one()
|
||||
|
|
@ -68,6 +70,23 @@ def _default_scene_count(session: Session) -> int:
|
|||
return total
|
||||
|
||||
|
||||
# JAV vertical — osobna sekcja. Sceny z tych originów NIE wchodzą do głównego feedu
|
||||
# (domyślnie wykluczone), tylko do zakładki JAV (?jav=true). JAV to osobny namespace
|
||||
# (kody typu BKD-368, azjatyckie tytuły), nie deduplikuje się z zachodnim katalogiem,
|
||||
# więc scena JAV ma WYŁĄCZNIE origin JAV → wykluczenie/inkluzja po tym originie jest pewna.
|
||||
JAV_ORIGINS = ("tube:javflix", "tube:javguru", "tube:vjav", "tube:supjav")
|
||||
|
||||
|
||||
def _jav_source_exists():
|
||||
return exists(
|
||||
select(1).where(
|
||||
PlaybackSource.scene_id == Scene.id,
|
||||
PlaybackSource.dead_at.is_(None),
|
||||
PlaybackSource.origin.in_(JAV_ORIGINS),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
# Blacklisty (performer/studio/tag) są zwykle PUSTE (self-hosted, single-user). Mimo to
|
||||
# 3 NOT EXISTS klauzule doklejały się do KAŻDEJ filtrowanej listy scen i były ewaluowane
|
||||
# per-row — przy filtrze typu duży-tag/has_playback planer chodzi po ~176k scen, więc te
|
||||
|
|
@ -222,6 +241,13 @@ def list_scenes(
|
|||
"z jedynym playback z hqporner (~7-min Brazzers trailer clipy zalewają katalog)."
|
||||
),
|
||||
),
|
||||
jav: bool = Query(
|
||||
default=False,
|
||||
description=(
|
||||
"False (default): ukrywa sceny JAV (osobny vertical). True: TYLKO sceny JAV. "
|
||||
"JAV to osobna sekcja w apce, nie zalewa głównego feedu (origin javflix/javguru/vjav/supjav)."
|
||||
),
|
||||
),
|
||||
sort: str = Query(default="created_at", description="created_at|release_date|title|studio"),
|
||||
page: int = Query(default=1, ge=1),
|
||||
per_page: int = Query(default=50, ge=1, le=200),
|
||||
|
|
@ -322,6 +348,13 @@ def list_scenes(
|
|||
)
|
||||
)
|
||||
|
||||
# JAV vertical gate: domyślnie JAV wykluczone z głównego feedu; ?jav=true → TYLKO JAV.
|
||||
# (origin=... diagnostyka nadal podlega bramie — do JAV użyj jav=true.)
|
||||
if jav:
|
||||
base = base.where(_jav_source_exists())
|
||||
else:
|
||||
base = base.where(~_jav_source_exists())
|
||||
|
||||
# Blacklisty device (performer/studio/tag) — globalne wykluczenia, współdzielone z
|
||||
# licznikiem +N ulubionych. Puste blacklisty → [] (typowy single-user, zero kosztu).
|
||||
for _bl_clause in blacklist_clauses(session, device_id):
|
||||
|
|
@ -373,7 +406,7 @@ def list_scenes(
|
|||
and not perf_id_strings and origin is None and has_playback is None
|
||||
and min_duration_sec is None
|
||||
and max_duration_sec is None and released_within_days is None
|
||||
and min_quality_p is None
|
||||
and min_quality_p is None and not jav
|
||||
)
|
||||
# Count strategy:
|
||||
# - PURE default: cached pełny licznik katalogu (TTL 10 min).
|
||||
|
|
|
|||
|
|
@ -135,6 +135,7 @@ from app.connectors.direct_scrapers.yesporn import YesPornVipScraper # noqa: E4
|
|||
from app.connectors.direct_scrapers.fullmovies import FullmoviesScraper # noqa: E402
|
||||
from app.connectors.direct_scrapers.hdporngg import HDPornGGScraper # noqa: E402
|
||||
from app.connectors.direct_scrapers.hqfap import HQFapScraper # noqa: E402
|
||||
from app.connectors.direct_scrapers.javflix import JavflixScraper # noqa: E402
|
||||
from app.connectors.direct_scrapers.neporn import NepornScraper # noqa: E402
|
||||
from app.connectors.direct_scrapers.superporn import SuperpornScraper # noqa: E402
|
||||
from app.connectors.direct_scrapers.eporner_api import EpornerApiScraper # noqa: E402
|
||||
|
|
@ -218,6 +219,9 @@ ALL_BROWSE_SCRAPERS: list[type[BaseBrowseScraper]] = [
|
|||
# bo strona wróciła na CDN vstor.top z realnymi plikami (portable cross-IP), user request.
|
||||
HQFapScraper,
|
||||
# FourK69Scraper — USUNIĘTY 2026-06-25 (ten sam stub), NIE sprawdzany ponownie.
|
||||
# JavflixScraper — JAV vertical (osobna sekcja). origin tube:javflix jest w
|
||||
# JAV_ORIGINS → list_scenes wyklucza je z głównego feedu (tylko zakładka JAV).
|
||||
JavflixScraper,
|
||||
# NepornScraper — dołączony 2026-06-10 (user request). KVS engine (jak freshporno/
|
||||
# porn00), /latest-updates/N/. JSON-LD (title+desc+uploadDate+thumb) + video:duration
|
||||
# meta + /models/ performerzy + /categories/ tagi. Brak studio (tytuł bywa
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ Struktura (RE 2026-07-10):
|
|||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import re
|
||||
|
||||
from app.connectors.base import RawPlaybackSource, RawScene, RawTag
|
||||
|
|
@ -26,6 +27,7 @@ _NON_POST = (
|
|||
"/page/", "/category/", "/categories/", "/genre/", "/maker/", "/actress/",
|
||||
"/actors/", "/tag/", "/tags/", "/studio/", "/label/", "/series/", "/wp-",
|
||||
"/18-usc", "/dmca", "/contact", "/privacy", "/about", "/2257",
|
||||
"/terms", "/faq", "/policy", "/disclaimer", "/sitemap",
|
||||
)
|
||||
_CODE_RE = re.compile(r"^([a-z]+-?\d+[a-z]?)", re.IGNORECASE)
|
||||
|
||||
|
|
@ -58,12 +60,17 @@ class JavflixScraper(BaseBrowseScraper):
|
|||
return out
|
||||
|
||||
def _parse_detail(self, scene_url: str, detail_html: str) -> RawScene | None:
|
||||
# Prawdziwy post video ma przyciski serwerów (`class="myLink"`). Strony statyczne
|
||||
# (Terms/FAQ/DMCA) ich nie mają → pomijamy (URL-filter nie łapie wszystkich).
|
||||
if 'class="myLink"' not in detail_html:
|
||||
return None
|
||||
title = _itemprop(detail_html, "name")
|
||||
if not title:
|
||||
tm = re.search(r"<title>([^<]+)</title>", detail_html)
|
||||
title = tm.group(1).split(" – ")[0].strip() if tm else None
|
||||
if not title:
|
||||
return None
|
||||
title = html.unescape(title).strip()
|
||||
|
||||
thumb = _itemprop(detail_html, "thumbnailUrl")
|
||||
up = _itemprop(detail_html, "uploadDate")
|
||||
|
|
|
|||
|
|
@ -134,6 +134,7 @@ export class GoonClient {
|
|||
qs.set('released_within_days', String(params.released_within_days));
|
||||
if (params.min_quality_p !== undefined) qs.set('min_quality_p', String(params.min_quality_p));
|
||||
if (params.include_stubs !== undefined) qs.set('include_stubs', String(params.include_stubs));
|
||||
if (params.jav) qs.set('jav', 'true');
|
||||
if (params.origin) qs.set('origin', params.origin);
|
||||
if (params.sort) qs.set('sort', params.sort);
|
||||
qs.set('page', String(params.page ?? 1));
|
||||
|
|
|
|||
|
|
@ -16,6 +16,13 @@ export type ChangelogEntry = {
|
|||
};
|
||||
|
||||
export const CHANGELOG: ChangelogEntry[] = [
|
||||
{
|
||||
id: '2026-07-10b',
|
||||
date: 'July 2026',
|
||||
items: [
|
||||
'New JAV tab: a separate section for Japanese adult video (English-subtitled). It is kept out of the main Scenes feed, so the main list stays as it was.',
|
||||
],
|
||||
},
|
||||
{
|
||||
id: '2026-07-10',
|
||||
date: 'July 2026',
|
||||
|
|
|
|||
|
|
@ -33,6 +33,8 @@ export type RootStackParamList = {
|
|||
Scenes: undefined;
|
||||
Movies: undefined;
|
||||
Sites: undefined;
|
||||
// JAV: osobna sekcja — ScenesScreen z paramem jav → backend ?jav=true.
|
||||
JAV: { jav: true } | undefined;
|
||||
// `origin`: raw playback_source.origin (np. 'tube:hqpornercom'). Idzie do
|
||||
// listScenes({origin}) — backend robi substring match. `name`: display name
|
||||
// do title bara (np. 'hqporner.com').
|
||||
|
|
@ -97,7 +99,7 @@ export type RootStackParamList = {
|
|||
|
||||
const Stack = createNativeStackNavigator<RootStackParamList>();
|
||||
|
||||
type TopTab = 'Scenes' | 'Movies' | 'Sites';
|
||||
type TopTab = 'Scenes' | 'Movies' | 'Sites' | 'JAV';
|
||||
|
||||
function TopTabs({
|
||||
current,
|
||||
|
|
@ -106,7 +108,7 @@ function TopTabs({
|
|||
current: TopTab;
|
||||
onNavigate: (tab: TopTab) => void;
|
||||
}) {
|
||||
const tabs: TopTab[] = ['Scenes', 'Movies', 'Sites'];
|
||||
const tabs: TopTab[] = ['Scenes', 'Movies', 'Sites', 'JAV'];
|
||||
return (
|
||||
<View style={{ flexDirection: 'row', gap: 10, paddingHorizontal: 10, alignItems: 'center' }}>
|
||||
{/* Logo (mark) — spójne z ekranem logowania; węższe od wordmarku, więc Scenes/Movies/Sites
|
||||
|
|
@ -241,6 +243,28 @@ export function AppNavigator({ onLogout, client, appVersion }: AppNavigatorProps
|
|||
),
|
||||
})}
|
||||
/>
|
||||
<Stack.Screen
|
||||
name="JAV"
|
||||
component={ScenesScreen}
|
||||
initialParams={{ jav: true }}
|
||||
options={({ navigation }) => ({
|
||||
title: '',
|
||||
headerBackVisible: false,
|
||||
headerLeft: () => (
|
||||
<TopTabs
|
||||
current="JAV"
|
||||
onNavigate={(t) => navigation.replace(t)}
|
||||
/>
|
||||
),
|
||||
headerRight: () => (
|
||||
<View style={{ flexDirection: 'row', gap: 14, alignItems: 'center' }}>
|
||||
<Pressable onPress={onLogout} hitSlop={12}>
|
||||
<Text style={{ fontSize: 17 }}>🚪</Text>
|
||||
</Pressable>
|
||||
</View>
|
||||
),
|
||||
})}
|
||||
/>
|
||||
<Stack.Screen
|
||||
name="SiteScenes"
|
||||
component={SiteScenesScreen}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import { useNavigation } from '@react-navigation/native';
|
||||
import { useNavigation, useRoute } from '@react-navigation/native';
|
||||
import type { NativeStackNavigationProp } from '@react-navigation/native-stack';
|
||||
import { useInfiniteQuery, useMutation, useQuery, useQueryClient } from '@tanstack/react-query';
|
||||
import React, { useState } from 'react';
|
||||
|
|
@ -29,6 +29,9 @@ export function ScenesScreen() {
|
|||
const client = useClient();
|
||||
const { gridColumns } = usePreferences();
|
||||
const navigation = useNavigation<NativeStackNavigationProp<RootStackParamList, 'Scenes'>>();
|
||||
// Zakładka JAV reużywa ten ekran z paramem jav → backend ?jav=true (osobny vertical).
|
||||
const route = useRoute();
|
||||
const jav = (route.params as { jav?: boolean } | undefined)?.jav === true;
|
||||
const [q, setQ] = useState('');
|
||||
const [debouncedQ, setDebouncedQ] = useState('');
|
||||
const [filter, setFilter] = useState<FilterState>(DEFAULT_FILTER);
|
||||
|
|
@ -50,7 +53,7 @@ export function ScenesScreen() {
|
|||
hasNextPage,
|
||||
isFetchingNextPage,
|
||||
} = useInfiniteQuery({
|
||||
queryKey: ['scenes', debouncedQ, filter],
|
||||
queryKey: ['scenes', debouncedQ, filter, jav],
|
||||
queryFn: ({ pageParam = 1 }) =>
|
||||
client.listScenes({
|
||||
q: debouncedQ || undefined,
|
||||
|
|
@ -61,8 +64,11 @@ export function ScenesScreen() {
|
|||
sort: filter.sort,
|
||||
include_stubs: filter.includeStubs || undefined,
|
||||
origin: filter.origin.trim() || undefined,
|
||||
jav: jav || undefined,
|
||||
// 0 = "Any" → undefined → backend stosuje bazowy próg 60s. >0 podnosi minimum.
|
||||
min_duration_sec: filter.minDurationSec > 0 ? filter.minDurationSec : undefined,
|
||||
// JAV nie ma duration (javflix nie podaje) → w sekcji JAV wyłączamy próg 60s
|
||||
// (explicit 0), inaczej NULL >= 60 = false i cała sekcja byłaby pusta.
|
||||
min_duration_sec: jav ? 0 : filter.minDurationSec > 0 ? filter.minDurationSec : undefined,
|
||||
page: pageParam,
|
||||
per_page: PER_PAGE,
|
||||
}),
|
||||
|
|
|
|||
|
|
@ -151,6 +151,7 @@ export interface ScenesListParams {
|
|||
min_quality_p?: number;
|
||||
include_stubs?: boolean;
|
||||
origin?: string;
|
||||
jav?: boolean;
|
||||
sort?: ScenesSort;
|
||||
page?: number;
|
||||
per_page?: number;
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue