feat(sites): loga stron (favicon -> PNG w static, job co 24h) + sortowanie w arkuszu pod ikona

- scripts/site_icons.py: apple-touch-icon / icon / favicon.ico, PIL do 64x64 PNG,
  proxy Bright Data w odwodzie, mapa domen dla sitetagow bez TLD
- /sources: icon_url gdy plik istnieje (BACKEND_PUBLIC_URL albo wzgledne)
- SitesScreen: SiteLogo z literowym fallbackiem, SortSheet (most scenes, recently
  updated, best rated, name) zapamietany w SecureStore, koniec wiersza Top/Recent
- changelog 2026-09-07g, OTA 1e90f427 na 1.3
This commit is contained in:
goon-foss 2026-09-07 12:13:58 +02:00
parent 8c75b12066
commit e13463cba4
6 changed files with 427 additions and 18 deletions

View file

@ -14,8 +14,10 @@ Sortowanie: scene_count DESC (najbardziej "wypełnione" tubey na górze).
from __future__ import annotations
import logging
import os
import re
from datetime import datetime
from pathlib import Path
from typing import Annotated
from fastapi import APIRouter, Depends
@ -75,6 +77,22 @@ class SourceOut(BaseModel):
"""Ocena 0-5★ (freshness/richness/health) z source_stats, None gdy jeszcze
nie policzona (job source-stats leci co kilka h)."""
icon_url: str | None = None
"""Logo strony (favicon 64x64 PNG) z app/static/site-icons, pobierane przez
scripts/site_icons.py (job raz na dobe). None, gdy jeszcze nie ma pliku. Adres
absolutny przez BACKEND_PUBLIC_URL, inaczej wzgledny '/static/...' (apka dokleja
baseUrl)."""
_ICON_DIR = Path(__file__).resolve().parent.parent / "static" / "site-icons"
def _icon_url(sitetag: str) -> str | None:
if not (_ICON_DIR / f"{sitetag}.png").is_file():
return None
base = os.environ.get("BACKEND_PUBLIC_URL", "").rstrip("/")
return f"{base}/static/site-icons/{sitetag}.png"
class SourceListOut(BaseModel):
items: list[SourceOut]
@ -159,6 +177,7 @@ def list_sources(
scene_count=scene_count,
last_scraped_at=last_scraped_at,
rating=rating,
icon_url=_icon_url(sitetag),
)
)

View file

@ -193,6 +193,18 @@ def _job_refresh_taxonomy_counts() -> None:
log.exception("[scheduler] taxonomy counts refresh failed")
def _job_site_icons() -> None:
"""Loga stron do zakladki Sites (favicon -> PNG 64x64 w app/static/site-icons).
Patrz scripts/site_icons.py; pliki mlodsze niz 14 dni zostaja."""
from scripts.site_icons import live_sitetags, refresh_site_icons
try:
stats = refresh_site_icons(live_sitetags())
log.info("[scheduler] site-icons: %s", stats)
except Exception: # noqa: BLE001
log.exception("[scheduler] site-icons failed")
def _job_refresh_sxyprn_thumbs(batch: int = 1200) -> None:
"""Odświeża wygasłe sxyprn miniaturki z żywych stron /post/<id> (bug 2026-06-10).
@ -478,6 +490,15 @@ def build_scheduler(cfg: dict[str, Any]) -> BlockingScheduler:
"""
sched = BlockingScheduler(timezone="UTC")
sched.add_job(
_job_site_icons,
IntervalTrigger(hours=24, start_date=INTERVAL_ANCHOR),
id="site-icons",
replace_existing=True,
max_instances=1,
coalesce=True,
)
if cfg.get("tpdb_hours"):
sched.add_job(
_job_tpdb,

View file

@ -29,6 +29,13 @@ export type ChangelogEntry = {
};
export const CHANGELOG: ChangelogEntry[] = [
{
id: '2026-09-07g',
date: 'September 2026',
items: [
'Sites now show each site\'s logo. Sorting (most scenes, recently updated, best rated, name) moved from the Top/Recent row into a sort icon next to the search box and is remembered between sessions.',
],
},
{
id: '2026-09-07f',
date: 'September 2026',

View file

@ -4,6 +4,7 @@
//
// Layout: chip-grid analogiczny do TagsScreen, krótkie nazwy (domena.tld)
// plus scene_count + relative-time "Xh temu" scraped, jeśli świeży.
import { Image } from 'expo-image';
import { useNavigation } from '@react-navigation/native';
import type { NativeStackNavigationProp } from '@react-navigation/native-stack';
import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query';
@ -21,11 +22,20 @@ import {
import { useClient } from '../ClientContext';
import type { RootStackParamList } from '../navigation';
import { theme } from '../theme';
import { PillButton, SCREEN_PAD_H, SCREEN_PAD_TOP, SearchInput, SettingsButton, Toolbar } from '../ui';
import { IconButton, SCREEN_PAD_H, SCREEN_PAD_TOP, SearchInput, SettingsButton, Toolbar } from '../ui';
import { usePersistedState } from '../lib/persistedState';
import type { SourceOut, SourceRating } from '../types';
import { useAutoHideOnScroll, useBottomBarInset, useScrollTopListener } from '../lib/bottomBar';
type Order = 'popular' | 'recent';
type Order = 'popular' | 'recent' | 'rated' | 'name';
const ORDER_OPTIONS: { value: Order; label: string; hint: string }[] = [
{ value: 'popular', label: 'Most scenes', hint: 'biggest catalogues first' },
{ value: 'recent', label: 'Recently updated', hint: 'freshest scrape first' },
{ value: 'rated', label: 'Best rated', hint: 'by the star rating' },
{ value: 'name', label: 'Name', hint: 'A to Z' },
];
const DEFAULT_SITES_FILTER = { order: 'popular' as Order };
export function SitesScreen() {
const client = useClient();
@ -33,7 +43,11 @@ export function SitesScreen() {
useNavigation<NativeStackNavigationProp<RootStackParamList, 'Sites'>>();
const [q, setQ] = useState('');
const [debouncedQ, setDebouncedQ] = useState('');
const [order, setOrder] = useState<Order>('popular');
// Sortowanie w arkuszu pod ikona, jak filtry na Scenes i Movies, zapamietane
// miedzy sesjami (uwaga 2026-09-07: Top/Recent wisialo osobnym wierszem).
const [sitesFilter, setSitesFilter] = usePersistedState('filters_sites', DEFAULT_SITES_FILTER);
const order = sitesFilter.order;
const [sortOpen, setSortOpen] = useState(false);
const [searchFocused, setSearchFocused] = useState(false);
// Źródło którego rozkład oceny pokazujemy w modalu (tap w gwiazdki).
const [detail, setDetail] = useState<SourceOut | null>(null);
@ -85,15 +99,23 @@ export function SitesScreen() {
s.sitetag.toLowerCase().includes(debouncedQ.toLowerCase()),
)
: all;
const sorted =
order === 'recent'
? [...filtered].sort((a, b) => {
if (!a.last_scraped_at && !b.last_scraped_at) return 0;
if (!a.last_scraped_at) return 1;
if (!b.last_scraped_at) return -1;
return b.last_scraped_at.localeCompare(a.last_scraped_at);
})
: filtered;
let sorted = filtered;
if (order === 'recent') {
sorted = [...filtered].sort((a, b) => {
if (!a.last_scraped_at && !b.last_scraped_at) return 0;
if (!a.last_scraped_at) return 1;
if (!b.last_scraped_at) return -1;
return b.last_scraped_at.localeCompare(a.last_scraped_at);
});
} else if (order === 'rated') {
sorted = [...filtered].sort(
(a, b) => (b.rating?.stars ?? -1) - (a.rating?.stars ?? -1) || b.scene_count - a.scene_count,
);
} else if (order === 'name') {
sorted = [...filtered].sort((a, b) =>
prettySiteName(a.display_name).localeCompare(prettySiteName(b.display_name)),
);
}
if (pinned.size === 0) return sorted;
return [...sorted.filter((s) => pinned.has(s.origin)), ...sorted.filter((s) => !pinned.has(s.origin))];
}, [data?.items, debouncedQ, order, pinned]);
@ -102,13 +124,23 @@ export function SitesScreen() {
<View style={styles.container}>
<Toolbar>
<SearchInput value={q} onChangeText={setQ} placeholder="search site…" />
<IconButton
icon="sort-variant"
accessibilityLabel="Sort"
active={order !== 'popular'}
onPress={() => setSortOpen(true)}
/>
<SettingsButton />
</Toolbar>
<View style={styles.segment}>
<PillButton label="Top" compact active={order === 'popular'} onPress={() => setOrder('popular')} />
<PillButton label="Recent" compact active={order === 'recent'} onPress={() => setOrder('recent')} />
</View>
<SortSheet
visible={sortOpen}
value={order}
onSelect={(o) => {
setSitesFilter({ order: o });
setSortOpen(false);
}}
onClose={() => setSortOpen(false)}
/>
{favQuery.data && pinned.size === 0 ? (
<Text style={styles.hint}>long-press a site to pin it to the top</Text>
) : null}
@ -127,6 +159,7 @@ export function SitesScreen() {
renderItem={({ item }) => (
<SiteChip
source={item}
iconUrl={absoluteUrl(client.baseUrl, item.icon_url)}
pinned={pinned.has(item.origin)}
onPress={() =>
navigation.navigate('SiteScenes', {
@ -151,6 +184,76 @@ export function SitesScreen() {
);
}
function SortSheet({
visible,
value,
onSelect,
onClose,
}: {
visible: boolean;
value: Order;
onSelect: (o: Order) => void;
onClose: () => void;
}) {
return (
<Modal visible={visible} transparent animationType="slide" onRequestClose={onClose}>
<View style={styles.backdrop}>
<Pressable style={{ flex: 1 }} onPress={onClose} />
<View style={styles.sheet}>
<Text style={styles.sheetTitle}>Sort sites</Text>
{ORDER_OPTIONS.map((opt) => {
const active = opt.value === value;
return (
<Pressable
key={opt.value}
style={({ pressed }) => [styles.sortRow, pressed && { opacity: 0.7 }]}
onPress={() => onSelect(opt.value)}
accessibilityRole="radio"
accessibilityState={{ selected: active }}
>
<View style={[styles.radio, active && styles.radioActive]} />
<View style={{ flex: 1 }}>
<Text style={[styles.sortLabel, active && styles.sortLabelActive]}>{opt.label}</Text>
<Text style={styles.sortHint}>{opt.hint}</Text>
</View>
</Pressable>
);
})}
</View>
</View>
</Modal>
);
}
/** Logo z backendu bywa wzgledne ('/static/...'), gdy BACKEND_PUBLIC_URL nie jest ustawiony. */
function absoluteUrl(base: string, raw: string | null | undefined): string | null {
if (!raw) return null;
return raw.startsWith('/') ? `${base}${raw}` : raw;
}
/** Logo strony albo pierwsza litera nazwy, gdy loga nie ma lub nie chce sie wczytac. */
function SiteLogo({ url, name }: { url: string | null; name: string }) {
const [failed, setFailed] = useState(false);
React.useEffect(() => setFailed(false), [url]);
if (url && !failed) {
return (
<Image
source={{ uri: url }}
style={styles.logo}
contentFit="contain"
transition={120}
cachePolicy="disk"
onError={() => setFailed(true)}
/>
);
}
return (
<View style={[styles.logo, styles.logoFallback]}>
<Text style={styles.logoLetter}>{(name.trim()[0] || '?').toUpperCase()}</Text>
</View>
);
}
// Lista Sites pokazuje display_name = domena (hqporner.com). User-report 18105d14:
// pozbyć się suffixów com/org/itp. Strip końcowego TLD → czysta nazwa.
function prettySiteName(name: string): string {
@ -186,12 +289,14 @@ function Stars({ value, size = 13 }: { value: number | null | undefined; size?:
function SiteChip({
source,
iconUrl,
pinned,
onPress,
onLongPress,
onShowRating,
}: {
source: SourceOut;
iconUrl: string | null;
pinned: boolean;
onPress: () => void;
onLongPress: () => void;
@ -207,6 +312,7 @@ function SiteChip({
delayLongPress={350}
>
<View style={styles.chipTopRow}>
<SiteLogo url={iconUrl} name={prettySiteName(source.display_name)} />
<View style={styles.chipMain}>
<Text style={styles.chipName} numberOfLines={1}>
{pinned ? '★ ' : ''}
@ -304,7 +410,37 @@ function RatingModal({ source, onClose }: { source: SourceOut | null; onClose: (
const styles = StyleSheet.create({
container: { flex: 1, backgroundColor: theme.bg, paddingHorizontal: SCREEN_PAD_H, paddingTop: SCREEN_PAD_TOP },
segment: { flexDirection: 'row', gap: 8, marginBottom: 12 },
backdrop: { flex: 1, backgroundColor: 'rgba(0,0,0,0.6)', justifyContent: 'flex-end' },
sheet: {
backgroundColor: theme.bgElevated,
borderTopLeftRadius: 20,
borderTopRightRadius: 20,
paddingHorizontal: 20,
paddingTop: 16,
paddingBottom: 28,
gap: 4,
},
sheetTitle: { color: theme.fg, fontSize: 18, fontWeight: '600', marginBottom: 8 },
sortRow: { flexDirection: 'row', alignItems: 'center', gap: 14, paddingVertical: 10 },
radio: {
width: 18,
height: 18,
borderRadius: 9,
borderWidth: 2,
borderColor: theme.borderStrong,
},
radioActive: { borderColor: theme.accentText, backgroundColor: theme.accent },
sortLabel: { color: theme.fg, fontSize: 15 },
sortLabelActive: { color: theme.accentText, fontWeight: '600' },
sortHint: { color: theme.mutedDim, fontSize: 12 },
logo: { width: 30, height: 30, borderRadius: 7, backgroundColor: theme.bgElevated },
logoFallback: {
alignItems: 'center',
justifyContent: 'center',
borderWidth: 1,
borderColor: theme.border,
},
logoLetter: { color: theme.muted, fontSize: 14, fontWeight: '700' },
gridRow: { gap: 10, marginBottom: 10 },

View file

@ -130,6 +130,8 @@ export interface SourceOut {
scene_count: number;
last_scraped_at: string | null;
rating?: SourceRating | null;
/** Logo strony (favicon jako PNG z backendu). Wzgledne '/static/...' dokleja apka. */
icon_url?: string | null;
}
export interface SourceListOut {

224
scripts/site_icons.py Normal file
View file

@ -0,0 +1,224 @@
"""Loga stron do zakladki Sites: favicon kazdego tube'a jako PNG 64x64 w app/static.
Po co przez backend, a nie z telefonu: apka nie ma uderzac do 44 domen NSFW ani do
zewnetrznego serwisu ikon (Google/DDG) z adresem IP uzytkownika. VPS pobiera raz,
normalizuje (PIL: ico/png/svg-bez, przezroczystosc zachowana) i serwuje statycznie,
a API `/sources` dokleja `icon_url`, gdy plik istnieje.
Zrodlo ikony, w kolejnosci: `<link rel="apple-touch-icon">` (zwykle najwiekszy PNG),
`<link rel="icon"|"shortcut icon">`, na koncu `/favicon.ico`. Strony za Cloudflare,
ktore blokuja VPS, idą przez proxy Bright Data, jak w ingest HTML.
Odswiezanie: plik mlodszy niz `_TTL_DNI` zostaje, brak/porazka nie zostawia sladu
(nastepny przebieg sprobuje znowu). Job w schedulerze raz na dobe, recznie:
docker compose run --rm worker python3 -u /srv/scripts/site_icons.py [--force] [--only sitetag]
"""
from __future__ import annotations
import argparse
import html as html_mod
import io
import logging
import re
import sys
import time
from pathlib import Path
from urllib.parse import urljoin
from PIL import Image
from app.api.sources import _sitetag_to_display
from app.config import get_settings
from app.extractors._fetch import browser_get
log = logging.getLogger(__name__)
ICON_DIR = Path(__file__).resolve().parent.parent / "app" / "static" / "site-icons"
ICON_PX = 64
_TTL_DNI = 14
_LINK_RE = re.compile(r"<link\b[^>]*>", re.IGNORECASE)
_ATTR_RE = re.compile(r"""([a-zA-Z-]+)\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s>]+))""")
def _attrs(tag: str) -> dict[str, str]:
out: dict[str, str] = {}
for m in _ATTR_RE.finditer(tag):
out[m.group(1).lower()] = html_mod.unescape(m.group(2) or m.group(3) or m.group(4) or "")
return out
def _icon_candidates(page_url: str, page_html: str) -> list[str]:
"""Adresy ikon z <link rel=...icon...>, najpierw apple-touch-icon, potem reszta."""
apple: list[str] = []
plain: list[str] = []
for tag in _LINK_RE.findall(page_html):
a = _attrs(tag)
rel = a.get("rel", "").lower()
href = a.get("href", "").strip()
if not href or "icon" not in rel:
continue
if href.lower().endswith(".svg"):
continue # PIL nie czyta SVG, a to i tak zwykle logo, nie favicon
full = urljoin(page_url, href)
(apple if "apple-touch" in rel else plain).append(full)
seen: set[str] = set()
out: list[str] = []
for u in apple + plain + [urljoin(page_url, "/favicon.ico")]:
if u not in seen:
seen.add(u)
out.append(u)
return out
def _fetch(url: str, *, proxy: str | None, timeout: float = 20.0):
"""HTML strony glownej. Najpierw bez proxy, przy blokadzie przez Bright Data."""
try:
r = browser_get(url, timeout=timeout)
if r.status_code == 200:
return r
except Exception as e: # noqa: BLE001
log.debug("site-icons: %s bez proxy: %s", url, e)
if not proxy:
return None
try:
r = browser_get(url, timeout=timeout, proxy=proxy)
return r if r.status_code == 200 else None
except Exception as e: # noqa: BLE001
log.debug("site-icons: %s przez proxy: %s", url, e)
return None
def _fetch_bytes(url: str, *, proxy: str | None, timeout: float = 15.0) -> bytes | None:
"""Binarka ikony. `browser_get` oddaje tylko tekst, wiec ten sam klient (curl_cffi z
odciskiem Chrome) wolany wprost, z tym samym proxy w odwodzie."""
from curl_cffi import requests as cf
for px in ([None, proxy] if proxy else [None]):
try:
r = cf.get(
url,
timeout=timeout,
impersonate="chrome120",
allow_redirects=True,
proxies={"http": px, "https": px} if px else None,
verify=not px,
)
if r.status_code == 200 and r.content:
return r.content
except Exception as e: # noqa: BLE001
log.debug("site-icons: %s (%s): %s", url, "proxy" if px else "direct", e)
return None
def _to_png(data: bytes) -> bytes | None:
try:
im = Image.open(io.BytesIO(data))
# ICO trzyma kilka rozmiarow, PIL otwiera najwiekszy dopiero po .size sprawdzeniu
if getattr(im, "format", None) == "ICO":
sizes = sorted(getattr(im, "ico", None).sizes()) if hasattr(im, "ico") else []
if sizes:
im.size = sizes[-1]
im = im.convert("RGBA")
except Exception: # noqa: BLE001
return None
if im.width < 8 or im.height < 8:
return None
im.thumbnail((ICON_PX, ICON_PX), Image.LANCZOS)
canvas = Image.new("RGBA", (ICON_PX, ICON_PX), (0, 0, 0, 0))
canvas.paste(im, ((ICON_PX - im.width) // 2, (ICON_PX - im.height) // 2))
buf = io.BytesIO()
canvas.save(buf, format="PNG", optimize=True)
return buf.getvalue()
# Sitetagi bez TLD w nazwie (display name = sama nazwa) albo z inna domena niz sugeruje
# nazwa (galaxyporn.net, nie .com). Zrodlo: _BASE w scraperach.
_DOMAIN_OVERRIDES: dict[str, str] = {
"fullporner": "fullporner.com",
"fullvideosporn": "fullvideosporn.com",
"galaxyporncom": "galaxyporn.net",
"hentaigasm": "hentaigasm.com",
"hentaimama": "hentaimama.io",
"hentaiplay": "hentaiplay.net",
"javflix": "javflix.cc",
"naughtymachinima": "www.naughtymachinima.com",
"porndishcom": "www.porndish.com",
"porndupe": "www.porndupe.com",
"pornmike": "pornmike.com",
"rule34video": "rule34video.com",
"supjav": "supjav.com",
"tubepornclassic": "tubepornclassic.com",
"vjav": "vjav.com",
"xanimeporn": "xanimeporn.com",
"yespornvip": "yesporn.vip",
}
def fetch_icon(sitetag: str, *, proxy: str | None) -> bytes | None:
domain = _DOMAIN_OVERRIDES.get(sitetag) or _sitetag_to_display(sitetag)
if "." not in domain:
return None
home = f"https://{domain}/"
page = _fetch(home, proxy=proxy)
candidates = _icon_candidates(home, page.text) if page is not None else [home + "favicon.ico"]
for url in candidates[:4]:
raw = _fetch_bytes(url, proxy=proxy)
if not raw:
continue
png = _to_png(raw)
if png:
return png
return None
def refresh_site_icons(sitetags: list[str], *, force: bool = False) -> dict[str, int]:
ICON_DIR.mkdir(parents=True, exist_ok=True)
proxy = get_settings().brightdata_proxy_url
stats = {"ok": 0, "fresh": 0, "miss": 0}
now = time.time()
for tag in sitetags:
out = ICON_DIR / f"{tag}.png"
if not force and out.exists() and now - out.stat().st_mtime < _TTL_DNI * 86400:
stats["fresh"] += 1
continue
png = fetch_icon(tag, proxy=proxy)
if png:
out.write_bytes(png)
stats["ok"] += 1
log.info("site-icons: %s ok (%d B)", tag, len(png))
else:
stats["miss"] += 1
log.info("site-icons: %s brak ikony", tag)
return stats
def live_sitetags() -> list[str]:
from sqlalchemy import text
from app.db import session_scope
with session_scope() as s:
rows = s.execute(
text(
"SELECT DISTINCT origin FROM playback_sources "
"WHERE dead_at IS NULL AND origin LIKE 'tube:%'"
)
).all()
return sorted(r[0].split(":", 1)[1] for r in rows)
def main() -> int:
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
ap = argparse.ArgumentParser()
ap.add_argument("--force", action="store_true")
ap.add_argument("--only", help="jeden sitetag")
args = ap.parse_args()
tags = [args.only] if args.only else live_sitetags()
stats = refresh_site_icons(tags, force=args.force)
print(f"site-icons: {stats} z {len(tags)} stron; katalog {ICON_DIR}")
return 0
if __name__ == "__main__":
sys.exit(main())