feat(sites): loga stron (favicon -> PNG w static, job co 24h) + sortowanie w arkuszu pod ikona
- scripts/site_icons.py: apple-touch-icon / icon / favicon.ico, PIL do 64x64 PNG, proxy Bright Data w odwodzie, mapa domen dla sitetagow bez TLD - /sources: icon_url gdy plik istnieje (BACKEND_PUBLIC_URL albo wzgledne) - SitesScreen: SiteLogo z literowym fallbackiem, SortSheet (most scenes, recently updated, best rated, name) zapamietany w SecureStore, koniec wiersza Top/Recent - changelog 2026-09-07g, OTA 1e90f427 na 1.3
This commit is contained in:
parent
8c75b12066
commit
e13463cba4
6 changed files with 427 additions and 18 deletions
|
|
@ -14,8 +14,10 @@ Sortowanie: scene_count DESC (najbardziej "wypełnione" tubey na górze).
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
import os
|
||||||
import re
|
import re
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
from pathlib import Path
|
||||||
from typing import Annotated
|
from typing import Annotated
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends
|
from fastapi import APIRouter, Depends
|
||||||
|
|
@ -75,6 +77,22 @@ class SourceOut(BaseModel):
|
||||||
"""Ocena 0-5★ (freshness/richness/health) z source_stats, None gdy jeszcze
|
"""Ocena 0-5★ (freshness/richness/health) z source_stats, None gdy jeszcze
|
||||||
nie policzona (job source-stats leci co kilka h)."""
|
nie policzona (job source-stats leci co kilka h)."""
|
||||||
|
|
||||||
|
icon_url: str | None = None
|
||||||
|
"""Logo strony (favicon 64x64 PNG) z app/static/site-icons, pobierane przez
|
||||||
|
scripts/site_icons.py (job raz na dobe). None, gdy jeszcze nie ma pliku. Adres
|
||||||
|
absolutny przez BACKEND_PUBLIC_URL, inaczej wzgledny '/static/...' (apka dokleja
|
||||||
|
baseUrl)."""
|
||||||
|
|
||||||
|
|
||||||
|
_ICON_DIR = Path(__file__).resolve().parent.parent / "static" / "site-icons"
|
||||||
|
|
||||||
|
|
||||||
|
def _icon_url(sitetag: str) -> str | None:
|
||||||
|
if not (_ICON_DIR / f"{sitetag}.png").is_file():
|
||||||
|
return None
|
||||||
|
base = os.environ.get("BACKEND_PUBLIC_URL", "").rstrip("/")
|
||||||
|
return f"{base}/static/site-icons/{sitetag}.png"
|
||||||
|
|
||||||
|
|
||||||
class SourceListOut(BaseModel):
|
class SourceListOut(BaseModel):
|
||||||
items: list[SourceOut]
|
items: list[SourceOut]
|
||||||
|
|
@ -159,6 +177,7 @@ def list_sources(
|
||||||
scene_count=scene_count,
|
scene_count=scene_count,
|
||||||
last_scraped_at=last_scraped_at,
|
last_scraped_at=last_scraped_at,
|
||||||
rating=rating,
|
rating=rating,
|
||||||
|
icon_url=_icon_url(sitetag),
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -193,6 +193,18 @@ def _job_refresh_taxonomy_counts() -> None:
|
||||||
log.exception("[scheduler] taxonomy counts refresh failed")
|
log.exception("[scheduler] taxonomy counts refresh failed")
|
||||||
|
|
||||||
|
|
||||||
|
def _job_site_icons() -> None:
|
||||||
|
"""Loga stron do zakladki Sites (favicon -> PNG 64x64 w app/static/site-icons).
|
||||||
|
Patrz scripts/site_icons.py; pliki mlodsze niz 14 dni zostaja."""
|
||||||
|
from scripts.site_icons import live_sitetags, refresh_site_icons
|
||||||
|
|
||||||
|
try:
|
||||||
|
stats = refresh_site_icons(live_sitetags())
|
||||||
|
log.info("[scheduler] site-icons: %s", stats)
|
||||||
|
except Exception: # noqa: BLE001
|
||||||
|
log.exception("[scheduler] site-icons failed")
|
||||||
|
|
||||||
|
|
||||||
def _job_refresh_sxyprn_thumbs(batch: int = 1200) -> None:
|
def _job_refresh_sxyprn_thumbs(batch: int = 1200) -> None:
|
||||||
"""Odświeża wygasłe sxyprn miniaturki z żywych stron /post/<id> (bug 2026-06-10).
|
"""Odświeża wygasłe sxyprn miniaturki z żywych stron /post/<id> (bug 2026-06-10).
|
||||||
|
|
||||||
|
|
@ -478,6 +490,15 @@ def build_scheduler(cfg: dict[str, Any]) -> BlockingScheduler:
|
||||||
"""
|
"""
|
||||||
sched = BlockingScheduler(timezone="UTC")
|
sched = BlockingScheduler(timezone="UTC")
|
||||||
|
|
||||||
|
sched.add_job(
|
||||||
|
_job_site_icons,
|
||||||
|
IntervalTrigger(hours=24, start_date=INTERVAL_ANCHOR),
|
||||||
|
id="site-icons",
|
||||||
|
replace_existing=True,
|
||||||
|
max_instances=1,
|
||||||
|
coalesce=True,
|
||||||
|
)
|
||||||
|
|
||||||
if cfg.get("tpdb_hours"):
|
if cfg.get("tpdb_hours"):
|
||||||
sched.add_job(
|
sched.add_job(
|
||||||
_job_tpdb,
|
_job_tpdb,
|
||||||
|
|
|
||||||
|
|
@ -29,6 +29,13 @@ export type ChangelogEntry = {
|
||||||
};
|
};
|
||||||
|
|
||||||
export const CHANGELOG: ChangelogEntry[] = [
|
export const CHANGELOG: ChangelogEntry[] = [
|
||||||
|
{
|
||||||
|
id: '2026-09-07g',
|
||||||
|
date: 'September 2026',
|
||||||
|
items: [
|
||||||
|
'Sites now show each site\'s logo. Sorting (most scenes, recently updated, best rated, name) moved from the Top/Recent row into a sort icon next to the search box and is remembered between sessions.',
|
||||||
|
],
|
||||||
|
},
|
||||||
{
|
{
|
||||||
id: '2026-09-07f',
|
id: '2026-09-07f',
|
||||||
date: 'September 2026',
|
date: 'September 2026',
|
||||||
|
|
|
||||||
|
|
@ -4,6 +4,7 @@
|
||||||
//
|
//
|
||||||
// Layout: chip-grid analogiczny do TagsScreen, krótkie nazwy (domena.tld)
|
// Layout: chip-grid analogiczny do TagsScreen, krótkie nazwy (domena.tld)
|
||||||
// plus scene_count + relative-time "Xh temu" scraped, jeśli świeży.
|
// plus scene_count + relative-time "Xh temu" scraped, jeśli świeży.
|
||||||
|
import { Image } from 'expo-image';
|
||||||
import { useNavigation } from '@react-navigation/native';
|
import { useNavigation } from '@react-navigation/native';
|
||||||
import type { NativeStackNavigationProp } from '@react-navigation/native-stack';
|
import type { NativeStackNavigationProp } from '@react-navigation/native-stack';
|
||||||
import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query';
|
import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query';
|
||||||
|
|
@ -21,11 +22,20 @@ import {
|
||||||
import { useClient } from '../ClientContext';
|
import { useClient } from '../ClientContext';
|
||||||
import type { RootStackParamList } from '../navigation';
|
import type { RootStackParamList } from '../navigation';
|
||||||
import { theme } from '../theme';
|
import { theme } from '../theme';
|
||||||
import { PillButton, SCREEN_PAD_H, SCREEN_PAD_TOP, SearchInput, SettingsButton, Toolbar } from '../ui';
|
import { IconButton, SCREEN_PAD_H, SCREEN_PAD_TOP, SearchInput, SettingsButton, Toolbar } from '../ui';
|
||||||
|
import { usePersistedState } from '../lib/persistedState';
|
||||||
import type { SourceOut, SourceRating } from '../types';
|
import type { SourceOut, SourceRating } from '../types';
|
||||||
import { useAutoHideOnScroll, useBottomBarInset, useScrollTopListener } from '../lib/bottomBar';
|
import { useAutoHideOnScroll, useBottomBarInset, useScrollTopListener } from '../lib/bottomBar';
|
||||||
|
|
||||||
type Order = 'popular' | 'recent';
|
type Order = 'popular' | 'recent' | 'rated' | 'name';
|
||||||
|
|
||||||
|
const ORDER_OPTIONS: { value: Order; label: string; hint: string }[] = [
|
||||||
|
{ value: 'popular', label: 'Most scenes', hint: 'biggest catalogues first' },
|
||||||
|
{ value: 'recent', label: 'Recently updated', hint: 'freshest scrape first' },
|
||||||
|
{ value: 'rated', label: 'Best rated', hint: 'by the star rating' },
|
||||||
|
{ value: 'name', label: 'Name', hint: 'A to Z' },
|
||||||
|
];
|
||||||
|
const DEFAULT_SITES_FILTER = { order: 'popular' as Order };
|
||||||
|
|
||||||
export function SitesScreen() {
|
export function SitesScreen() {
|
||||||
const client = useClient();
|
const client = useClient();
|
||||||
|
|
@ -33,7 +43,11 @@ export function SitesScreen() {
|
||||||
useNavigation<NativeStackNavigationProp<RootStackParamList, 'Sites'>>();
|
useNavigation<NativeStackNavigationProp<RootStackParamList, 'Sites'>>();
|
||||||
const [q, setQ] = useState('');
|
const [q, setQ] = useState('');
|
||||||
const [debouncedQ, setDebouncedQ] = useState('');
|
const [debouncedQ, setDebouncedQ] = useState('');
|
||||||
const [order, setOrder] = useState<Order>('popular');
|
// Sortowanie w arkuszu pod ikona, jak filtry na Scenes i Movies, zapamietane
|
||||||
|
// miedzy sesjami (uwaga 2026-09-07: Top/Recent wisialo osobnym wierszem).
|
||||||
|
const [sitesFilter, setSitesFilter] = usePersistedState('filters_sites', DEFAULT_SITES_FILTER);
|
||||||
|
const order = sitesFilter.order;
|
||||||
|
const [sortOpen, setSortOpen] = useState(false);
|
||||||
const [searchFocused, setSearchFocused] = useState(false);
|
const [searchFocused, setSearchFocused] = useState(false);
|
||||||
// Źródło którego rozkład oceny pokazujemy w modalu (tap w gwiazdki).
|
// Źródło którego rozkład oceny pokazujemy w modalu (tap w gwiazdki).
|
||||||
const [detail, setDetail] = useState<SourceOut | null>(null);
|
const [detail, setDetail] = useState<SourceOut | null>(null);
|
||||||
|
|
@ -85,15 +99,23 @@ export function SitesScreen() {
|
||||||
s.sitetag.toLowerCase().includes(debouncedQ.toLowerCase()),
|
s.sitetag.toLowerCase().includes(debouncedQ.toLowerCase()),
|
||||||
)
|
)
|
||||||
: all;
|
: all;
|
||||||
const sorted =
|
let sorted = filtered;
|
||||||
order === 'recent'
|
if (order === 'recent') {
|
||||||
? [...filtered].sort((a, b) => {
|
sorted = [...filtered].sort((a, b) => {
|
||||||
if (!a.last_scraped_at && !b.last_scraped_at) return 0;
|
if (!a.last_scraped_at && !b.last_scraped_at) return 0;
|
||||||
if (!a.last_scraped_at) return 1;
|
if (!a.last_scraped_at) return 1;
|
||||||
if (!b.last_scraped_at) return -1;
|
if (!b.last_scraped_at) return -1;
|
||||||
return b.last_scraped_at.localeCompare(a.last_scraped_at);
|
return b.last_scraped_at.localeCompare(a.last_scraped_at);
|
||||||
})
|
});
|
||||||
: filtered;
|
} else if (order === 'rated') {
|
||||||
|
sorted = [...filtered].sort(
|
||||||
|
(a, b) => (b.rating?.stars ?? -1) - (a.rating?.stars ?? -1) || b.scene_count - a.scene_count,
|
||||||
|
);
|
||||||
|
} else if (order === 'name') {
|
||||||
|
sorted = [...filtered].sort((a, b) =>
|
||||||
|
prettySiteName(a.display_name).localeCompare(prettySiteName(b.display_name)),
|
||||||
|
);
|
||||||
|
}
|
||||||
if (pinned.size === 0) return sorted;
|
if (pinned.size === 0) return sorted;
|
||||||
return [...sorted.filter((s) => pinned.has(s.origin)), ...sorted.filter((s) => !pinned.has(s.origin))];
|
return [...sorted.filter((s) => pinned.has(s.origin)), ...sorted.filter((s) => !pinned.has(s.origin))];
|
||||||
}, [data?.items, debouncedQ, order, pinned]);
|
}, [data?.items, debouncedQ, order, pinned]);
|
||||||
|
|
@ -102,13 +124,23 @@ export function SitesScreen() {
|
||||||
<View style={styles.container}>
|
<View style={styles.container}>
|
||||||
<Toolbar>
|
<Toolbar>
|
||||||
<SearchInput value={q} onChangeText={setQ} placeholder="search site…" />
|
<SearchInput value={q} onChangeText={setQ} placeholder="search site…" />
|
||||||
|
<IconButton
|
||||||
|
icon="sort-variant"
|
||||||
|
accessibilityLabel="Sort"
|
||||||
|
active={order !== 'popular'}
|
||||||
|
onPress={() => setSortOpen(true)}
|
||||||
|
/>
|
||||||
<SettingsButton />
|
<SettingsButton />
|
||||||
</Toolbar>
|
</Toolbar>
|
||||||
|
<SortSheet
|
||||||
<View style={styles.segment}>
|
visible={sortOpen}
|
||||||
<PillButton label="Top" compact active={order === 'popular'} onPress={() => setOrder('popular')} />
|
value={order}
|
||||||
<PillButton label="Recent" compact active={order === 'recent'} onPress={() => setOrder('recent')} />
|
onSelect={(o) => {
|
||||||
</View>
|
setSitesFilter({ order: o });
|
||||||
|
setSortOpen(false);
|
||||||
|
}}
|
||||||
|
onClose={() => setSortOpen(false)}
|
||||||
|
/>
|
||||||
{favQuery.data && pinned.size === 0 ? (
|
{favQuery.data && pinned.size === 0 ? (
|
||||||
<Text style={styles.hint}>long-press a site to pin it to the top</Text>
|
<Text style={styles.hint}>long-press a site to pin it to the top</Text>
|
||||||
) : null}
|
) : null}
|
||||||
|
|
@ -127,6 +159,7 @@ export function SitesScreen() {
|
||||||
renderItem={({ item }) => (
|
renderItem={({ item }) => (
|
||||||
<SiteChip
|
<SiteChip
|
||||||
source={item}
|
source={item}
|
||||||
|
iconUrl={absoluteUrl(client.baseUrl, item.icon_url)}
|
||||||
pinned={pinned.has(item.origin)}
|
pinned={pinned.has(item.origin)}
|
||||||
onPress={() =>
|
onPress={() =>
|
||||||
navigation.navigate('SiteScenes', {
|
navigation.navigate('SiteScenes', {
|
||||||
|
|
@ -151,6 +184,76 @@ export function SitesScreen() {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function SortSheet({
|
||||||
|
visible,
|
||||||
|
value,
|
||||||
|
onSelect,
|
||||||
|
onClose,
|
||||||
|
}: {
|
||||||
|
visible: boolean;
|
||||||
|
value: Order;
|
||||||
|
onSelect: (o: Order) => void;
|
||||||
|
onClose: () => void;
|
||||||
|
}) {
|
||||||
|
return (
|
||||||
|
<Modal visible={visible} transparent animationType="slide" onRequestClose={onClose}>
|
||||||
|
<View style={styles.backdrop}>
|
||||||
|
<Pressable style={{ flex: 1 }} onPress={onClose} />
|
||||||
|
<View style={styles.sheet}>
|
||||||
|
<Text style={styles.sheetTitle}>Sort sites</Text>
|
||||||
|
{ORDER_OPTIONS.map((opt) => {
|
||||||
|
const active = opt.value === value;
|
||||||
|
return (
|
||||||
|
<Pressable
|
||||||
|
key={opt.value}
|
||||||
|
style={({ pressed }) => [styles.sortRow, pressed && { opacity: 0.7 }]}
|
||||||
|
onPress={() => onSelect(opt.value)}
|
||||||
|
accessibilityRole="radio"
|
||||||
|
accessibilityState={{ selected: active }}
|
||||||
|
>
|
||||||
|
<View style={[styles.radio, active && styles.radioActive]} />
|
||||||
|
<View style={{ flex: 1 }}>
|
||||||
|
<Text style={[styles.sortLabel, active && styles.sortLabelActive]}>{opt.label}</Text>
|
||||||
|
<Text style={styles.sortHint}>{opt.hint}</Text>
|
||||||
|
</View>
|
||||||
|
</Pressable>
|
||||||
|
);
|
||||||
|
})}
|
||||||
|
</View>
|
||||||
|
</View>
|
||||||
|
</Modal>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Logo z backendu bywa wzgledne ('/static/...'), gdy BACKEND_PUBLIC_URL nie jest ustawiony. */
|
||||||
|
function absoluteUrl(base: string, raw: string | null | undefined): string | null {
|
||||||
|
if (!raw) return null;
|
||||||
|
return raw.startsWith('/') ? `${base}${raw}` : raw;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Logo strony albo pierwsza litera nazwy, gdy loga nie ma lub nie chce sie wczytac. */
|
||||||
|
function SiteLogo({ url, name }: { url: string | null; name: string }) {
|
||||||
|
const [failed, setFailed] = useState(false);
|
||||||
|
React.useEffect(() => setFailed(false), [url]);
|
||||||
|
if (url && !failed) {
|
||||||
|
return (
|
||||||
|
<Image
|
||||||
|
source={{ uri: url }}
|
||||||
|
style={styles.logo}
|
||||||
|
contentFit="contain"
|
||||||
|
transition={120}
|
||||||
|
cachePolicy="disk"
|
||||||
|
onError={() => setFailed(true)}
|
||||||
|
/>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return (
|
||||||
|
<View style={[styles.logo, styles.logoFallback]}>
|
||||||
|
<Text style={styles.logoLetter}>{(name.trim()[0] || '?').toUpperCase()}</Text>
|
||||||
|
</View>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
// Lista Sites pokazuje display_name = domena (hqporner.com). User-report 18105d14:
|
// Lista Sites pokazuje display_name = domena (hqporner.com). User-report 18105d14:
|
||||||
// pozbyć się suffixów com/org/itp. Strip końcowego TLD → czysta nazwa.
|
// pozbyć się suffixów com/org/itp. Strip końcowego TLD → czysta nazwa.
|
||||||
function prettySiteName(name: string): string {
|
function prettySiteName(name: string): string {
|
||||||
|
|
@ -186,12 +289,14 @@ function Stars({ value, size = 13 }: { value: number | null | undefined; size?:
|
||||||
|
|
||||||
function SiteChip({
|
function SiteChip({
|
||||||
source,
|
source,
|
||||||
|
iconUrl,
|
||||||
pinned,
|
pinned,
|
||||||
onPress,
|
onPress,
|
||||||
onLongPress,
|
onLongPress,
|
||||||
onShowRating,
|
onShowRating,
|
||||||
}: {
|
}: {
|
||||||
source: SourceOut;
|
source: SourceOut;
|
||||||
|
iconUrl: string | null;
|
||||||
pinned: boolean;
|
pinned: boolean;
|
||||||
onPress: () => void;
|
onPress: () => void;
|
||||||
onLongPress: () => void;
|
onLongPress: () => void;
|
||||||
|
|
@ -207,6 +312,7 @@ function SiteChip({
|
||||||
delayLongPress={350}
|
delayLongPress={350}
|
||||||
>
|
>
|
||||||
<View style={styles.chipTopRow}>
|
<View style={styles.chipTopRow}>
|
||||||
|
<SiteLogo url={iconUrl} name={prettySiteName(source.display_name)} />
|
||||||
<View style={styles.chipMain}>
|
<View style={styles.chipMain}>
|
||||||
<Text style={styles.chipName} numberOfLines={1}>
|
<Text style={styles.chipName} numberOfLines={1}>
|
||||||
{pinned ? '★ ' : ''}
|
{pinned ? '★ ' : ''}
|
||||||
|
|
@ -304,7 +410,37 @@ function RatingModal({ source, onClose }: { source: SourceOut | null; onClose: (
|
||||||
const styles = StyleSheet.create({
|
const styles = StyleSheet.create({
|
||||||
container: { flex: 1, backgroundColor: theme.bg, paddingHorizontal: SCREEN_PAD_H, paddingTop: SCREEN_PAD_TOP },
|
container: { flex: 1, backgroundColor: theme.bg, paddingHorizontal: SCREEN_PAD_H, paddingTop: SCREEN_PAD_TOP },
|
||||||
|
|
||||||
segment: { flexDirection: 'row', gap: 8, marginBottom: 12 },
|
backdrop: { flex: 1, backgroundColor: 'rgba(0,0,0,0.6)', justifyContent: 'flex-end' },
|
||||||
|
sheet: {
|
||||||
|
backgroundColor: theme.bgElevated,
|
||||||
|
borderTopLeftRadius: 20,
|
||||||
|
borderTopRightRadius: 20,
|
||||||
|
paddingHorizontal: 20,
|
||||||
|
paddingTop: 16,
|
||||||
|
paddingBottom: 28,
|
||||||
|
gap: 4,
|
||||||
|
},
|
||||||
|
sheetTitle: { color: theme.fg, fontSize: 18, fontWeight: '600', marginBottom: 8 },
|
||||||
|
sortRow: { flexDirection: 'row', alignItems: 'center', gap: 14, paddingVertical: 10 },
|
||||||
|
radio: {
|
||||||
|
width: 18,
|
||||||
|
height: 18,
|
||||||
|
borderRadius: 9,
|
||||||
|
borderWidth: 2,
|
||||||
|
borderColor: theme.borderStrong,
|
||||||
|
},
|
||||||
|
radioActive: { borderColor: theme.accentText, backgroundColor: theme.accent },
|
||||||
|
sortLabel: { color: theme.fg, fontSize: 15 },
|
||||||
|
sortLabelActive: { color: theme.accentText, fontWeight: '600' },
|
||||||
|
sortHint: { color: theme.mutedDim, fontSize: 12 },
|
||||||
|
logo: { width: 30, height: 30, borderRadius: 7, backgroundColor: theme.bgElevated },
|
||||||
|
logoFallback: {
|
||||||
|
alignItems: 'center',
|
||||||
|
justifyContent: 'center',
|
||||||
|
borderWidth: 1,
|
||||||
|
borderColor: theme.border,
|
||||||
|
},
|
||||||
|
logoLetter: { color: theme.muted, fontSize: 14, fontWeight: '700' },
|
||||||
|
|
||||||
|
|
||||||
gridRow: { gap: 10, marginBottom: 10 },
|
gridRow: { gap: 10, marginBottom: 10 },
|
||||||
|
|
|
||||||
|
|
@ -130,6 +130,8 @@ export interface SourceOut {
|
||||||
scene_count: number;
|
scene_count: number;
|
||||||
last_scraped_at: string | null;
|
last_scraped_at: string | null;
|
||||||
rating?: SourceRating | null;
|
rating?: SourceRating | null;
|
||||||
|
/** Logo strony (favicon jako PNG z backendu). Wzgledne '/static/...' dokleja apka. */
|
||||||
|
icon_url?: string | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface SourceListOut {
|
export interface SourceListOut {
|
||||||
|
|
|
||||||
224
scripts/site_icons.py
Normal file
224
scripts/site_icons.py
Normal file
|
|
@ -0,0 +1,224 @@
|
||||||
|
"""Loga stron do zakladki Sites: favicon kazdego tube'a jako PNG 64x64 w app/static.
|
||||||
|
|
||||||
|
Po co przez backend, a nie z telefonu: apka nie ma uderzac do 44 domen NSFW ani do
|
||||||
|
zewnetrznego serwisu ikon (Google/DDG) z adresem IP uzytkownika. VPS pobiera raz,
|
||||||
|
normalizuje (PIL: ico/png/svg-bez, przezroczystosc zachowana) i serwuje statycznie,
|
||||||
|
a API `/sources` dokleja `icon_url`, gdy plik istnieje.
|
||||||
|
|
||||||
|
Zrodlo ikony, w kolejnosci: `<link rel="apple-touch-icon">` (zwykle najwiekszy PNG),
|
||||||
|
`<link rel="icon"|"shortcut icon">`, na koncu `/favicon.ico`. Strony za Cloudflare,
|
||||||
|
ktore blokuja VPS, idą przez proxy Bright Data, jak w ingest HTML.
|
||||||
|
|
||||||
|
Odswiezanie: plik mlodszy niz `_TTL_DNI` zostaje, brak/porazka nie zostawia sladu
|
||||||
|
(nastepny przebieg sprobuje znowu). Job w schedulerze raz na dobe, recznie:
|
||||||
|
docker compose run --rm worker python3 -u /srv/scripts/site_icons.py [--force] [--only sitetag]
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import html as html_mod
|
||||||
|
import io
|
||||||
|
import logging
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
from urllib.parse import urljoin
|
||||||
|
|
||||||
|
from PIL import Image
|
||||||
|
|
||||||
|
from app.api.sources import _sitetag_to_display
|
||||||
|
from app.config import get_settings
|
||||||
|
from app.extractors._fetch import browser_get
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
ICON_DIR = Path(__file__).resolve().parent.parent / "app" / "static" / "site-icons"
|
||||||
|
ICON_PX = 64
|
||||||
|
_TTL_DNI = 14
|
||||||
|
|
||||||
|
_LINK_RE = re.compile(r"<link\b[^>]*>", re.IGNORECASE)
|
||||||
|
_ATTR_RE = re.compile(r"""([a-zA-Z-]+)\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s>]+))""")
|
||||||
|
|
||||||
|
|
||||||
|
def _attrs(tag: str) -> dict[str, str]:
|
||||||
|
out: dict[str, str] = {}
|
||||||
|
for m in _ATTR_RE.finditer(tag):
|
||||||
|
out[m.group(1).lower()] = html_mod.unescape(m.group(2) or m.group(3) or m.group(4) or "")
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _icon_candidates(page_url: str, page_html: str) -> list[str]:
|
||||||
|
"""Adresy ikon z <link rel=...icon...>, najpierw apple-touch-icon, potem reszta."""
|
||||||
|
apple: list[str] = []
|
||||||
|
plain: list[str] = []
|
||||||
|
for tag in _LINK_RE.findall(page_html):
|
||||||
|
a = _attrs(tag)
|
||||||
|
rel = a.get("rel", "").lower()
|
||||||
|
href = a.get("href", "").strip()
|
||||||
|
if not href or "icon" not in rel:
|
||||||
|
continue
|
||||||
|
if href.lower().endswith(".svg"):
|
||||||
|
continue # PIL nie czyta SVG, a to i tak zwykle logo, nie favicon
|
||||||
|
full = urljoin(page_url, href)
|
||||||
|
(apple if "apple-touch" in rel else plain).append(full)
|
||||||
|
seen: set[str] = set()
|
||||||
|
out: list[str] = []
|
||||||
|
for u in apple + plain + [urljoin(page_url, "/favicon.ico")]:
|
||||||
|
if u not in seen:
|
||||||
|
seen.add(u)
|
||||||
|
out.append(u)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _fetch(url: str, *, proxy: str | None, timeout: float = 20.0):
|
||||||
|
"""HTML strony glownej. Najpierw bez proxy, przy blokadzie przez Bright Data."""
|
||||||
|
try:
|
||||||
|
r = browser_get(url, timeout=timeout)
|
||||||
|
if r.status_code == 200:
|
||||||
|
return r
|
||||||
|
except Exception as e: # noqa: BLE001
|
||||||
|
log.debug("site-icons: %s bez proxy: %s", url, e)
|
||||||
|
if not proxy:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
r = browser_get(url, timeout=timeout, proxy=proxy)
|
||||||
|
return r if r.status_code == 200 else None
|
||||||
|
except Exception as e: # noqa: BLE001
|
||||||
|
log.debug("site-icons: %s przez proxy: %s", url, e)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _fetch_bytes(url: str, *, proxy: str | None, timeout: float = 15.0) -> bytes | None:
|
||||||
|
"""Binarka ikony. `browser_get` oddaje tylko tekst, wiec ten sam klient (curl_cffi z
|
||||||
|
odciskiem Chrome) wolany wprost, z tym samym proxy w odwodzie."""
|
||||||
|
from curl_cffi import requests as cf
|
||||||
|
|
||||||
|
for px in ([None, proxy] if proxy else [None]):
|
||||||
|
try:
|
||||||
|
r = cf.get(
|
||||||
|
url,
|
||||||
|
timeout=timeout,
|
||||||
|
impersonate="chrome120",
|
||||||
|
allow_redirects=True,
|
||||||
|
proxies={"http": px, "https": px} if px else None,
|
||||||
|
verify=not px,
|
||||||
|
)
|
||||||
|
if r.status_code == 200 and r.content:
|
||||||
|
return r.content
|
||||||
|
except Exception as e: # noqa: BLE001
|
||||||
|
log.debug("site-icons: %s (%s): %s", url, "proxy" if px else "direct", e)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _to_png(data: bytes) -> bytes | None:
|
||||||
|
try:
|
||||||
|
im = Image.open(io.BytesIO(data))
|
||||||
|
# ICO trzyma kilka rozmiarow, PIL otwiera najwiekszy dopiero po .size sprawdzeniu
|
||||||
|
if getattr(im, "format", None) == "ICO":
|
||||||
|
sizes = sorted(getattr(im, "ico", None).sizes()) if hasattr(im, "ico") else []
|
||||||
|
if sizes:
|
||||||
|
im.size = sizes[-1]
|
||||||
|
im = im.convert("RGBA")
|
||||||
|
except Exception: # noqa: BLE001
|
||||||
|
return None
|
||||||
|
if im.width < 8 or im.height < 8:
|
||||||
|
return None
|
||||||
|
im.thumbnail((ICON_PX, ICON_PX), Image.LANCZOS)
|
||||||
|
canvas = Image.new("RGBA", (ICON_PX, ICON_PX), (0, 0, 0, 0))
|
||||||
|
canvas.paste(im, ((ICON_PX - im.width) // 2, (ICON_PX - im.height) // 2))
|
||||||
|
buf = io.BytesIO()
|
||||||
|
canvas.save(buf, format="PNG", optimize=True)
|
||||||
|
return buf.getvalue()
|
||||||
|
|
||||||
|
|
||||||
|
# Sitetagi bez TLD w nazwie (display name = sama nazwa) albo z inna domena niz sugeruje
|
||||||
|
# nazwa (galaxyporn.net, nie .com). Zrodlo: _BASE w scraperach.
|
||||||
|
_DOMAIN_OVERRIDES: dict[str, str] = {
|
||||||
|
"fullporner": "fullporner.com",
|
||||||
|
"fullvideosporn": "fullvideosporn.com",
|
||||||
|
"galaxyporncom": "galaxyporn.net",
|
||||||
|
"hentaigasm": "hentaigasm.com",
|
||||||
|
"hentaimama": "hentaimama.io",
|
||||||
|
"hentaiplay": "hentaiplay.net",
|
||||||
|
"javflix": "javflix.cc",
|
||||||
|
"naughtymachinima": "www.naughtymachinima.com",
|
||||||
|
"porndishcom": "www.porndish.com",
|
||||||
|
"porndupe": "www.porndupe.com",
|
||||||
|
"pornmike": "pornmike.com",
|
||||||
|
"rule34video": "rule34video.com",
|
||||||
|
"supjav": "supjav.com",
|
||||||
|
"tubepornclassic": "tubepornclassic.com",
|
||||||
|
"vjav": "vjav.com",
|
||||||
|
"xanimeporn": "xanimeporn.com",
|
||||||
|
"yespornvip": "yesporn.vip",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def fetch_icon(sitetag: str, *, proxy: str | None) -> bytes | None:
|
||||||
|
domain = _DOMAIN_OVERRIDES.get(sitetag) or _sitetag_to_display(sitetag)
|
||||||
|
if "." not in domain:
|
||||||
|
return None
|
||||||
|
home = f"https://{domain}/"
|
||||||
|
page = _fetch(home, proxy=proxy)
|
||||||
|
candidates = _icon_candidates(home, page.text) if page is not None else [home + "favicon.ico"]
|
||||||
|
for url in candidates[:4]:
|
||||||
|
raw = _fetch_bytes(url, proxy=proxy)
|
||||||
|
if not raw:
|
||||||
|
continue
|
||||||
|
png = _to_png(raw)
|
||||||
|
if png:
|
||||||
|
return png
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def refresh_site_icons(sitetags: list[str], *, force: bool = False) -> dict[str, int]:
|
||||||
|
ICON_DIR.mkdir(parents=True, exist_ok=True)
|
||||||
|
proxy = get_settings().brightdata_proxy_url
|
||||||
|
stats = {"ok": 0, "fresh": 0, "miss": 0}
|
||||||
|
now = time.time()
|
||||||
|
for tag in sitetags:
|
||||||
|
out = ICON_DIR / f"{tag}.png"
|
||||||
|
if not force and out.exists() and now - out.stat().st_mtime < _TTL_DNI * 86400:
|
||||||
|
stats["fresh"] += 1
|
||||||
|
continue
|
||||||
|
png = fetch_icon(tag, proxy=proxy)
|
||||||
|
if png:
|
||||||
|
out.write_bytes(png)
|
||||||
|
stats["ok"] += 1
|
||||||
|
log.info("site-icons: %s ok (%d B)", tag, len(png))
|
||||||
|
else:
|
||||||
|
stats["miss"] += 1
|
||||||
|
log.info("site-icons: %s brak ikony", tag)
|
||||||
|
return stats
|
||||||
|
|
||||||
|
|
||||||
|
def live_sitetags() -> list[str]:
|
||||||
|
from sqlalchemy import text
|
||||||
|
|
||||||
|
from app.db import session_scope
|
||||||
|
|
||||||
|
with session_scope() as s:
|
||||||
|
rows = s.execute(
|
||||||
|
text(
|
||||||
|
"SELECT DISTINCT origin FROM playback_sources "
|
||||||
|
"WHERE dead_at IS NULL AND origin LIKE 'tube:%'"
|
||||||
|
)
|
||||||
|
).all()
|
||||||
|
return sorted(r[0].split(":", 1)[1] for r in rows)
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||||||
|
ap = argparse.ArgumentParser()
|
||||||
|
ap.add_argument("--force", action="store_true")
|
||||||
|
ap.add_argument("--only", help="jeden sitetag")
|
||||||
|
args = ap.parse_args()
|
||||||
|
tags = [args.only] if args.only else live_sitetags()
|
||||||
|
stats = refresh_site_icons(tags, force=args.force)
|
||||||
|
print(f"site-icons: {stats} z {len(tags)} stron; katalog {ICON_DIR}")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
Loading…
Add table
Reference in a new issue