feat(sites): loga stron (favicon -> PNG w static, job co 24h) + sortowanie w arkuszu pod ikona
- scripts/site_icons.py: apple-touch-icon / icon / favicon.ico, PIL do 64x64 PNG, proxy Bright Data w odwodzie, mapa domen dla sitetagow bez TLD - /sources: icon_url gdy plik istnieje (BACKEND_PUBLIC_URL albo wzgledne) - SitesScreen: SiteLogo z literowym fallbackiem, SortSheet (most scenes, recently updated, best rated, name) zapamietany w SecureStore, koniec wiersza Top/Recent - changelog 2026-09-07g, OTA 1e90f427 na 1.3
This commit is contained in:
parent
8c75b12066
commit
e13463cba4
6 changed files with 427 additions and 18 deletions
|
|
@ -14,8 +14,10 @@ Sortowanie: scene_count DESC (najbardziej "wypełnione" tubey na górze).
|
|||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
|
|
@ -75,6 +77,22 @@ class SourceOut(BaseModel):
|
|||
"""Ocena 0-5★ (freshness/richness/health) z source_stats, None gdy jeszcze
|
||||
nie policzona (job source-stats leci co kilka h)."""
|
||||
|
||||
icon_url: str | None = None
|
||||
"""Logo strony (favicon 64x64 PNG) z app/static/site-icons, pobierane przez
|
||||
scripts/site_icons.py (job raz na dobe). None, gdy jeszcze nie ma pliku. Adres
|
||||
absolutny przez BACKEND_PUBLIC_URL, inaczej wzgledny '/static/...' (apka dokleja
|
||||
baseUrl)."""
|
||||
|
||||
|
||||
_ICON_DIR = Path(__file__).resolve().parent.parent / "static" / "site-icons"
|
||||
|
||||
|
||||
def _icon_url(sitetag: str) -> str | None:
|
||||
if not (_ICON_DIR / f"{sitetag}.png").is_file():
|
||||
return None
|
||||
base = os.environ.get("BACKEND_PUBLIC_URL", "").rstrip("/")
|
||||
return f"{base}/static/site-icons/{sitetag}.png"
|
||||
|
||||
|
||||
class SourceListOut(BaseModel):
|
||||
items: list[SourceOut]
|
||||
|
|
@ -159,6 +177,7 @@ def list_sources(
|
|||
scene_count=scene_count,
|
||||
last_scraped_at=last_scraped_at,
|
||||
rating=rating,
|
||||
icon_url=_icon_url(sitetag),
|
||||
)
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -193,6 +193,18 @@ def _job_refresh_taxonomy_counts() -> None:
|
|||
log.exception("[scheduler] taxonomy counts refresh failed")
|
||||
|
||||
|
||||
def _job_site_icons() -> None:
|
||||
"""Loga stron do zakladki Sites (favicon -> PNG 64x64 w app/static/site-icons).
|
||||
Patrz scripts/site_icons.py; pliki mlodsze niz 14 dni zostaja."""
|
||||
from scripts.site_icons import live_sitetags, refresh_site_icons
|
||||
|
||||
try:
|
||||
stats = refresh_site_icons(live_sitetags())
|
||||
log.info("[scheduler] site-icons: %s", stats)
|
||||
except Exception: # noqa: BLE001
|
||||
log.exception("[scheduler] site-icons failed")
|
||||
|
||||
|
||||
def _job_refresh_sxyprn_thumbs(batch: int = 1200) -> None:
|
||||
"""Odświeża wygasłe sxyprn miniaturki z żywych stron /post/<id> (bug 2026-06-10).
|
||||
|
||||
|
|
@ -478,6 +490,15 @@ def build_scheduler(cfg: dict[str, Any]) -> BlockingScheduler:
|
|||
"""
|
||||
sched = BlockingScheduler(timezone="UTC")
|
||||
|
||||
sched.add_job(
|
||||
_job_site_icons,
|
||||
IntervalTrigger(hours=24, start_date=INTERVAL_ANCHOR),
|
||||
id="site-icons",
|
||||
replace_existing=True,
|
||||
max_instances=1,
|
||||
coalesce=True,
|
||||
)
|
||||
|
||||
if cfg.get("tpdb_hours"):
|
||||
sched.add_job(
|
||||
_job_tpdb,
|
||||
|
|
|
|||
|
|
@ -29,6 +29,13 @@ export type ChangelogEntry = {
|
|||
};
|
||||
|
||||
export const CHANGELOG: ChangelogEntry[] = [
|
||||
{
|
||||
id: '2026-09-07g',
|
||||
date: 'September 2026',
|
||||
items: [
|
||||
'Sites now show each site\'s logo. Sorting (most scenes, recently updated, best rated, name) moved from the Top/Recent row into a sort icon next to the search box and is remembered between sessions.',
|
||||
],
|
||||
},
|
||||
{
|
||||
id: '2026-09-07f',
|
||||
date: 'September 2026',
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@
|
|||
//
|
||||
// Layout: chip-grid analogiczny do TagsScreen, krótkie nazwy (domena.tld)
|
||||
// plus scene_count + relative-time "Xh temu" scraped, jeśli świeży.
|
||||
import { Image } from 'expo-image';
|
||||
import { useNavigation } from '@react-navigation/native';
|
||||
import type { NativeStackNavigationProp } from '@react-navigation/native-stack';
|
||||
import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query';
|
||||
|
|
@ -21,11 +22,20 @@ import {
|
|||
import { useClient } from '../ClientContext';
|
||||
import type { RootStackParamList } from '../navigation';
|
||||
import { theme } from '../theme';
|
||||
import { PillButton, SCREEN_PAD_H, SCREEN_PAD_TOP, SearchInput, SettingsButton, Toolbar } from '../ui';
|
||||
import { IconButton, SCREEN_PAD_H, SCREEN_PAD_TOP, SearchInput, SettingsButton, Toolbar } from '../ui';
|
||||
import { usePersistedState } from '../lib/persistedState';
|
||||
import type { SourceOut, SourceRating } from '../types';
|
||||
import { useAutoHideOnScroll, useBottomBarInset, useScrollTopListener } from '../lib/bottomBar';
|
||||
|
||||
type Order = 'popular' | 'recent';
|
||||
type Order = 'popular' | 'recent' | 'rated' | 'name';
|
||||
|
||||
const ORDER_OPTIONS: { value: Order; label: string; hint: string }[] = [
|
||||
{ value: 'popular', label: 'Most scenes', hint: 'biggest catalogues first' },
|
||||
{ value: 'recent', label: 'Recently updated', hint: 'freshest scrape first' },
|
||||
{ value: 'rated', label: 'Best rated', hint: 'by the star rating' },
|
||||
{ value: 'name', label: 'Name', hint: 'A to Z' },
|
||||
];
|
||||
const DEFAULT_SITES_FILTER = { order: 'popular' as Order };
|
||||
|
||||
export function SitesScreen() {
|
||||
const client = useClient();
|
||||
|
|
@ -33,7 +43,11 @@ export function SitesScreen() {
|
|||
useNavigation<NativeStackNavigationProp<RootStackParamList, 'Sites'>>();
|
||||
const [q, setQ] = useState('');
|
||||
const [debouncedQ, setDebouncedQ] = useState('');
|
||||
const [order, setOrder] = useState<Order>('popular');
|
||||
// Sortowanie w arkuszu pod ikona, jak filtry na Scenes i Movies, zapamietane
|
||||
// miedzy sesjami (uwaga 2026-09-07: Top/Recent wisialo osobnym wierszem).
|
||||
const [sitesFilter, setSitesFilter] = usePersistedState('filters_sites', DEFAULT_SITES_FILTER);
|
||||
const order = sitesFilter.order;
|
||||
const [sortOpen, setSortOpen] = useState(false);
|
||||
const [searchFocused, setSearchFocused] = useState(false);
|
||||
// Źródło którego rozkład oceny pokazujemy w modalu (tap w gwiazdki).
|
||||
const [detail, setDetail] = useState<SourceOut | null>(null);
|
||||
|
|
@ -85,15 +99,23 @@ export function SitesScreen() {
|
|||
s.sitetag.toLowerCase().includes(debouncedQ.toLowerCase()),
|
||||
)
|
||||
: all;
|
||||
const sorted =
|
||||
order === 'recent'
|
||||
? [...filtered].sort((a, b) => {
|
||||
let sorted = filtered;
|
||||
if (order === 'recent') {
|
||||
sorted = [...filtered].sort((a, b) => {
|
||||
if (!a.last_scraped_at && !b.last_scraped_at) return 0;
|
||||
if (!a.last_scraped_at) return 1;
|
||||
if (!b.last_scraped_at) return -1;
|
||||
return b.last_scraped_at.localeCompare(a.last_scraped_at);
|
||||
})
|
||||
: filtered;
|
||||
});
|
||||
} else if (order === 'rated') {
|
||||
sorted = [...filtered].sort(
|
||||
(a, b) => (b.rating?.stars ?? -1) - (a.rating?.stars ?? -1) || b.scene_count - a.scene_count,
|
||||
);
|
||||
} else if (order === 'name') {
|
||||
sorted = [...filtered].sort((a, b) =>
|
||||
prettySiteName(a.display_name).localeCompare(prettySiteName(b.display_name)),
|
||||
);
|
||||
}
|
||||
if (pinned.size === 0) return sorted;
|
||||
return [...sorted.filter((s) => pinned.has(s.origin)), ...sorted.filter((s) => !pinned.has(s.origin))];
|
||||
}, [data?.items, debouncedQ, order, pinned]);
|
||||
|
|
@ -102,13 +124,23 @@ export function SitesScreen() {
|
|||
<View style={styles.container}>
|
||||
<Toolbar>
|
||||
<SearchInput value={q} onChangeText={setQ} placeholder="search site…" />
|
||||
<IconButton
|
||||
icon="sort-variant"
|
||||
accessibilityLabel="Sort"
|
||||
active={order !== 'popular'}
|
||||
onPress={() => setSortOpen(true)}
|
||||
/>
|
||||
<SettingsButton />
|
||||
</Toolbar>
|
||||
|
||||
<View style={styles.segment}>
|
||||
<PillButton label="Top" compact active={order === 'popular'} onPress={() => setOrder('popular')} />
|
||||
<PillButton label="Recent" compact active={order === 'recent'} onPress={() => setOrder('recent')} />
|
||||
</View>
|
||||
<SortSheet
|
||||
visible={sortOpen}
|
||||
value={order}
|
||||
onSelect={(o) => {
|
||||
setSitesFilter({ order: o });
|
||||
setSortOpen(false);
|
||||
}}
|
||||
onClose={() => setSortOpen(false)}
|
||||
/>
|
||||
{favQuery.data && pinned.size === 0 ? (
|
||||
<Text style={styles.hint}>long-press a site to pin it to the top</Text>
|
||||
) : null}
|
||||
|
|
@ -127,6 +159,7 @@ export function SitesScreen() {
|
|||
renderItem={({ item }) => (
|
||||
<SiteChip
|
||||
source={item}
|
||||
iconUrl={absoluteUrl(client.baseUrl, item.icon_url)}
|
||||
pinned={pinned.has(item.origin)}
|
||||
onPress={() =>
|
||||
navigation.navigate('SiteScenes', {
|
||||
|
|
@ -151,6 +184,76 @@ export function SitesScreen() {
|
|||
);
|
||||
}
|
||||
|
||||
function SortSheet({
|
||||
visible,
|
||||
value,
|
||||
onSelect,
|
||||
onClose,
|
||||
}: {
|
||||
visible: boolean;
|
||||
value: Order;
|
||||
onSelect: (o: Order) => void;
|
||||
onClose: () => void;
|
||||
}) {
|
||||
return (
|
||||
<Modal visible={visible} transparent animationType="slide" onRequestClose={onClose}>
|
||||
<View style={styles.backdrop}>
|
||||
<Pressable style={{ flex: 1 }} onPress={onClose} />
|
||||
<View style={styles.sheet}>
|
||||
<Text style={styles.sheetTitle}>Sort sites</Text>
|
||||
{ORDER_OPTIONS.map((opt) => {
|
||||
const active = opt.value === value;
|
||||
return (
|
||||
<Pressable
|
||||
key={opt.value}
|
||||
style={({ pressed }) => [styles.sortRow, pressed && { opacity: 0.7 }]}
|
||||
onPress={() => onSelect(opt.value)}
|
||||
accessibilityRole="radio"
|
||||
accessibilityState={{ selected: active }}
|
||||
>
|
||||
<View style={[styles.radio, active && styles.radioActive]} />
|
||||
<View style={{ flex: 1 }}>
|
||||
<Text style={[styles.sortLabel, active && styles.sortLabelActive]}>{opt.label}</Text>
|
||||
<Text style={styles.sortHint}>{opt.hint}</Text>
|
||||
</View>
|
||||
</Pressable>
|
||||
);
|
||||
})}
|
||||
</View>
|
||||
</View>
|
||||
</Modal>
|
||||
);
|
||||
}
|
||||
|
||||
/** Logo z backendu bywa wzgledne ('/static/...'), gdy BACKEND_PUBLIC_URL nie jest ustawiony. */
|
||||
function absoluteUrl(base: string, raw: string | null | undefined): string | null {
|
||||
if (!raw) return null;
|
||||
return raw.startsWith('/') ? `${base}${raw}` : raw;
|
||||
}
|
||||
|
||||
/** Logo strony albo pierwsza litera nazwy, gdy loga nie ma lub nie chce sie wczytac. */
|
||||
function SiteLogo({ url, name }: { url: string | null; name: string }) {
|
||||
const [failed, setFailed] = useState(false);
|
||||
React.useEffect(() => setFailed(false), [url]);
|
||||
if (url && !failed) {
|
||||
return (
|
||||
<Image
|
||||
source={{ uri: url }}
|
||||
style={styles.logo}
|
||||
contentFit="contain"
|
||||
transition={120}
|
||||
cachePolicy="disk"
|
||||
onError={() => setFailed(true)}
|
||||
/>
|
||||
);
|
||||
}
|
||||
return (
|
||||
<View style={[styles.logo, styles.logoFallback]}>
|
||||
<Text style={styles.logoLetter}>{(name.trim()[0] || '?').toUpperCase()}</Text>
|
||||
</View>
|
||||
);
|
||||
}
|
||||
|
||||
// Lista Sites pokazuje display_name = domena (hqporner.com). User-report 18105d14:
|
||||
// pozbyć się suffixów com/org/itp. Strip końcowego TLD → czysta nazwa.
|
||||
function prettySiteName(name: string): string {
|
||||
|
|
@ -186,12 +289,14 @@ function Stars({ value, size = 13 }: { value: number | null | undefined; size?:
|
|||
|
||||
function SiteChip({
|
||||
source,
|
||||
iconUrl,
|
||||
pinned,
|
||||
onPress,
|
||||
onLongPress,
|
||||
onShowRating,
|
||||
}: {
|
||||
source: SourceOut;
|
||||
iconUrl: string | null;
|
||||
pinned: boolean;
|
||||
onPress: () => void;
|
||||
onLongPress: () => void;
|
||||
|
|
@ -207,6 +312,7 @@ function SiteChip({
|
|||
delayLongPress={350}
|
||||
>
|
||||
<View style={styles.chipTopRow}>
|
||||
<SiteLogo url={iconUrl} name={prettySiteName(source.display_name)} />
|
||||
<View style={styles.chipMain}>
|
||||
<Text style={styles.chipName} numberOfLines={1}>
|
||||
{pinned ? '★ ' : ''}
|
||||
|
|
@ -304,7 +410,37 @@ function RatingModal({ source, onClose }: { source: SourceOut | null; onClose: (
|
|||
const styles = StyleSheet.create({
|
||||
container: { flex: 1, backgroundColor: theme.bg, paddingHorizontal: SCREEN_PAD_H, paddingTop: SCREEN_PAD_TOP },
|
||||
|
||||
segment: { flexDirection: 'row', gap: 8, marginBottom: 12 },
|
||||
backdrop: { flex: 1, backgroundColor: 'rgba(0,0,0,0.6)', justifyContent: 'flex-end' },
|
||||
sheet: {
|
||||
backgroundColor: theme.bgElevated,
|
||||
borderTopLeftRadius: 20,
|
||||
borderTopRightRadius: 20,
|
||||
paddingHorizontal: 20,
|
||||
paddingTop: 16,
|
||||
paddingBottom: 28,
|
||||
gap: 4,
|
||||
},
|
||||
sheetTitle: { color: theme.fg, fontSize: 18, fontWeight: '600', marginBottom: 8 },
|
||||
sortRow: { flexDirection: 'row', alignItems: 'center', gap: 14, paddingVertical: 10 },
|
||||
radio: {
|
||||
width: 18,
|
||||
height: 18,
|
||||
borderRadius: 9,
|
||||
borderWidth: 2,
|
||||
borderColor: theme.borderStrong,
|
||||
},
|
||||
radioActive: { borderColor: theme.accentText, backgroundColor: theme.accent },
|
||||
sortLabel: { color: theme.fg, fontSize: 15 },
|
||||
sortLabelActive: { color: theme.accentText, fontWeight: '600' },
|
||||
sortHint: { color: theme.mutedDim, fontSize: 12 },
|
||||
logo: { width: 30, height: 30, borderRadius: 7, backgroundColor: theme.bgElevated },
|
||||
logoFallback: {
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
borderWidth: 1,
|
||||
borderColor: theme.border,
|
||||
},
|
||||
logoLetter: { color: theme.muted, fontSize: 14, fontWeight: '700' },
|
||||
|
||||
|
||||
gridRow: { gap: 10, marginBottom: 10 },
|
||||
|
|
|
|||
|
|
@ -130,6 +130,8 @@ export interface SourceOut {
|
|||
scene_count: number;
|
||||
last_scraped_at: string | null;
|
||||
rating?: SourceRating | null;
|
||||
/** Logo strony (favicon jako PNG z backendu). Wzgledne '/static/...' dokleja apka. */
|
||||
icon_url?: string | null;
|
||||
}
|
||||
|
||||
export interface SourceListOut {
|
||||
|
|
|
|||
224
scripts/site_icons.py
Normal file
224
scripts/site_icons.py
Normal file
|
|
@ -0,0 +1,224 @@
|
|||
"""Loga stron do zakladki Sites: favicon kazdego tube'a jako PNG 64x64 w app/static.
|
||||
|
||||
Po co przez backend, a nie z telefonu: apka nie ma uderzac do 44 domen NSFW ani do
|
||||
zewnetrznego serwisu ikon (Google/DDG) z adresem IP uzytkownika. VPS pobiera raz,
|
||||
normalizuje (PIL: ico/png/svg-bez, przezroczystosc zachowana) i serwuje statycznie,
|
||||
a API `/sources` dokleja `icon_url`, gdy plik istnieje.
|
||||
|
||||
Zrodlo ikony, w kolejnosci: `<link rel="apple-touch-icon">` (zwykle najwiekszy PNG),
|
||||
`<link rel="icon"|"shortcut icon">`, na koncu `/favicon.ico`. Strony za Cloudflare,
|
||||
ktore blokuja VPS, idą przez proxy Bright Data, jak w ingest HTML.
|
||||
|
||||
Odswiezanie: plik mlodszy niz `_TTL_DNI` zostaje, brak/porazka nie zostawia sladu
|
||||
(nastepny przebieg sprobuje znowu). Job w schedulerze raz na dobe, recznie:
|
||||
docker compose run --rm worker python3 -u /srv/scripts/site_icons.py [--force] [--only sitetag]
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import html as html_mod
|
||||
import io
|
||||
import logging
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from urllib.parse import urljoin
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from app.api.sources import _sitetag_to_display
|
||||
from app.config import get_settings
|
||||
from app.extractors._fetch import browser_get
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
ICON_DIR = Path(__file__).resolve().parent.parent / "app" / "static" / "site-icons"
|
||||
ICON_PX = 64
|
||||
_TTL_DNI = 14
|
||||
|
||||
_LINK_RE = re.compile(r"<link\b[^>]*>", re.IGNORECASE)
|
||||
_ATTR_RE = re.compile(r"""([a-zA-Z-]+)\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s>]+))""")
|
||||
|
||||
|
||||
def _attrs(tag: str) -> dict[str, str]:
|
||||
out: dict[str, str] = {}
|
||||
for m in _ATTR_RE.finditer(tag):
|
||||
out[m.group(1).lower()] = html_mod.unescape(m.group(2) or m.group(3) or m.group(4) or "")
|
||||
return out
|
||||
|
||||
|
||||
def _icon_candidates(page_url: str, page_html: str) -> list[str]:
|
||||
"""Adresy ikon z <link rel=...icon...>, najpierw apple-touch-icon, potem reszta."""
|
||||
apple: list[str] = []
|
||||
plain: list[str] = []
|
||||
for tag in _LINK_RE.findall(page_html):
|
||||
a = _attrs(tag)
|
||||
rel = a.get("rel", "").lower()
|
||||
href = a.get("href", "").strip()
|
||||
if not href or "icon" not in rel:
|
||||
continue
|
||||
if href.lower().endswith(".svg"):
|
||||
continue # PIL nie czyta SVG, a to i tak zwykle logo, nie favicon
|
||||
full = urljoin(page_url, href)
|
||||
(apple if "apple-touch" in rel else plain).append(full)
|
||||
seen: set[str] = set()
|
||||
out: list[str] = []
|
||||
for u in apple + plain + [urljoin(page_url, "/favicon.ico")]:
|
||||
if u not in seen:
|
||||
seen.add(u)
|
||||
out.append(u)
|
||||
return out
|
||||
|
||||
|
||||
def _fetch(url: str, *, proxy: str | None, timeout: float = 20.0):
|
||||
"""HTML strony glownej. Najpierw bez proxy, przy blokadzie przez Bright Data."""
|
||||
try:
|
||||
r = browser_get(url, timeout=timeout)
|
||||
if r.status_code == 200:
|
||||
return r
|
||||
except Exception as e: # noqa: BLE001
|
||||
log.debug("site-icons: %s bez proxy: %s", url, e)
|
||||
if not proxy:
|
||||
return None
|
||||
try:
|
||||
r = browser_get(url, timeout=timeout, proxy=proxy)
|
||||
return r if r.status_code == 200 else None
|
||||
except Exception as e: # noqa: BLE001
|
||||
log.debug("site-icons: %s przez proxy: %s", url, e)
|
||||
return None
|
||||
|
||||
|
||||
def _fetch_bytes(url: str, *, proxy: str | None, timeout: float = 15.0) -> bytes | None:
|
||||
"""Binarka ikony. `browser_get` oddaje tylko tekst, wiec ten sam klient (curl_cffi z
|
||||
odciskiem Chrome) wolany wprost, z tym samym proxy w odwodzie."""
|
||||
from curl_cffi import requests as cf
|
||||
|
||||
for px in ([None, proxy] if proxy else [None]):
|
||||
try:
|
||||
r = cf.get(
|
||||
url,
|
||||
timeout=timeout,
|
||||
impersonate="chrome120",
|
||||
allow_redirects=True,
|
||||
proxies={"http": px, "https": px} if px else None,
|
||||
verify=not px,
|
||||
)
|
||||
if r.status_code == 200 and r.content:
|
||||
return r.content
|
||||
except Exception as e: # noqa: BLE001
|
||||
log.debug("site-icons: %s (%s): %s", url, "proxy" if px else "direct", e)
|
||||
return None
|
||||
|
||||
|
||||
def _to_png(data: bytes) -> bytes | None:
|
||||
try:
|
||||
im = Image.open(io.BytesIO(data))
|
||||
# ICO trzyma kilka rozmiarow, PIL otwiera najwiekszy dopiero po .size sprawdzeniu
|
||||
if getattr(im, "format", None) == "ICO":
|
||||
sizes = sorted(getattr(im, "ico", None).sizes()) if hasattr(im, "ico") else []
|
||||
if sizes:
|
||||
im.size = sizes[-1]
|
||||
im = im.convert("RGBA")
|
||||
except Exception: # noqa: BLE001
|
||||
return None
|
||||
if im.width < 8 or im.height < 8:
|
||||
return None
|
||||
im.thumbnail((ICON_PX, ICON_PX), Image.LANCZOS)
|
||||
canvas = Image.new("RGBA", (ICON_PX, ICON_PX), (0, 0, 0, 0))
|
||||
canvas.paste(im, ((ICON_PX - im.width) // 2, (ICON_PX - im.height) // 2))
|
||||
buf = io.BytesIO()
|
||||
canvas.save(buf, format="PNG", optimize=True)
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
# Sitetagi bez TLD w nazwie (display name = sama nazwa) albo z inna domena niz sugeruje
|
||||
# nazwa (galaxyporn.net, nie .com). Zrodlo: _BASE w scraperach.
|
||||
_DOMAIN_OVERRIDES: dict[str, str] = {
|
||||
"fullporner": "fullporner.com",
|
||||
"fullvideosporn": "fullvideosporn.com",
|
||||
"galaxyporncom": "galaxyporn.net",
|
||||
"hentaigasm": "hentaigasm.com",
|
||||
"hentaimama": "hentaimama.io",
|
||||
"hentaiplay": "hentaiplay.net",
|
||||
"javflix": "javflix.cc",
|
||||
"naughtymachinima": "www.naughtymachinima.com",
|
||||
"porndishcom": "www.porndish.com",
|
||||
"porndupe": "www.porndupe.com",
|
||||
"pornmike": "pornmike.com",
|
||||
"rule34video": "rule34video.com",
|
||||
"supjav": "supjav.com",
|
||||
"tubepornclassic": "tubepornclassic.com",
|
||||
"vjav": "vjav.com",
|
||||
"xanimeporn": "xanimeporn.com",
|
||||
"yespornvip": "yesporn.vip",
|
||||
}
|
||||
|
||||
|
||||
def fetch_icon(sitetag: str, *, proxy: str | None) -> bytes | None:
|
||||
domain = _DOMAIN_OVERRIDES.get(sitetag) or _sitetag_to_display(sitetag)
|
||||
if "." not in domain:
|
||||
return None
|
||||
home = f"https://{domain}/"
|
||||
page = _fetch(home, proxy=proxy)
|
||||
candidates = _icon_candidates(home, page.text) if page is not None else [home + "favicon.ico"]
|
||||
for url in candidates[:4]:
|
||||
raw = _fetch_bytes(url, proxy=proxy)
|
||||
if not raw:
|
||||
continue
|
||||
png = _to_png(raw)
|
||||
if png:
|
||||
return png
|
||||
return None
|
||||
|
||||
|
||||
def refresh_site_icons(sitetags: list[str], *, force: bool = False) -> dict[str, int]:
|
||||
ICON_DIR.mkdir(parents=True, exist_ok=True)
|
||||
proxy = get_settings().brightdata_proxy_url
|
||||
stats = {"ok": 0, "fresh": 0, "miss": 0}
|
||||
now = time.time()
|
||||
for tag in sitetags:
|
||||
out = ICON_DIR / f"{tag}.png"
|
||||
if not force and out.exists() and now - out.stat().st_mtime < _TTL_DNI * 86400:
|
||||
stats["fresh"] += 1
|
||||
continue
|
||||
png = fetch_icon(tag, proxy=proxy)
|
||||
if png:
|
||||
out.write_bytes(png)
|
||||
stats["ok"] += 1
|
||||
log.info("site-icons: %s ok (%d B)", tag, len(png))
|
||||
else:
|
||||
stats["miss"] += 1
|
||||
log.info("site-icons: %s brak ikony", tag)
|
||||
return stats
|
||||
|
||||
|
||||
def live_sitetags() -> list[str]:
|
||||
from sqlalchemy import text
|
||||
|
||||
from app.db import session_scope
|
||||
|
||||
with session_scope() as s:
|
||||
rows = s.execute(
|
||||
text(
|
||||
"SELECT DISTINCT origin FROM playback_sources "
|
||||
"WHERE dead_at IS NULL AND origin LIKE 'tube:%'"
|
||||
)
|
||||
).all()
|
||||
return sorted(r[0].split(":", 1)[1] for r in rows)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--force", action="store_true")
|
||||
ap.add_argument("--only", help="jeden sitetag")
|
||||
args = ap.parse_args()
|
||||
tags = [args.only] if args.only else live_sitetags()
|
||||
stats = refresh_site_icons(tags, force=args.force)
|
||||
print(f"site-icons: {stats} z {len(tags)} stron; katalog {ICON_DIR}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Loading…
Add table
Reference in a new issue