Compare commits

...

5 commits

Author SHA1 Message Date
goon-foss
494b15aedd fix(siska): kafelki z apostrofem w tytule i kilka osob w jednym nazwisku
Some checks are pending
Backend tests / test (push) Waiting to run
Strona nie escapuje apostrofu w title/alt, wiec "Brenna's Mouths" urywal
dopasowanie i kafelek wypadal (3 z 18 na stronie). Pierwszy czlon alt z
kilkoma osobami ("A, B - Tytul") szedl jako jeden wykonawca.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-27 19:14:47 +02:00
goon-foss
9225ae95de fix(scrapers): krotka nazwa wykonawcy nie przypina go do wszystkich wynikow
Bramka nazwy w slugu zwracala True, gdy zaden czlon nazwy nie mial 3 znakow ("Wa A",
"Al B", "T P"), a wolajacy pomijali ja przy pustych tokenach. Performer-driven
dopinal wiec takiego wykonawce do kazdego wyniku wyszukiwarki tuba: 108 wykonawcow,
10,5 tys. blednych przypisan w 7,9 tys. scen. Teraz takie nazwy nic nie dopasowuja.
Na produkcji bledne przypisania usuniete (kopia w scene_performers_removed_20260924).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-24 13:03:11 +02:00
goon-foss
e8fae54724 docs: link do APK na goon-foss.org zamiast nieistniejacych GitHub Releases
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-24 12:59:30 +02:00
goon-foss
b64ddfd742 fix(hqporner): okladka z kafelka listingu, nie cudza ze strony filmu
Strona filmu hqporner nie ma wlasnej okladki, a pierwszy _main.jpg na niej to kafelek
polecanego filmu. Scraper go bral, wiec ~90% zrodel dostalo cudze miniatury (62 tys.
zrodel, 5,4 tys. roznych miniatur). Okladka jest tylko w kafelku listingu
(cover_<id>): browse zbiera je z listingu, wyszukiwarka z wynikow wyszukiwania.
scripts/hqporner_fix_thumbs.py poprawia zapisane miniatury przechodzac listing.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-24 12:58:41 +02:00
goon-foss
30dd83557d fix(mobile): expo-video bez ConcurrentModificationException przy liscie sluchaczy
close() odtwarzacza (watek JS albo sprzatajacy obiekty) wypisywal AudioFocusManager
z listy sluchaczy, a glowny watek w tym samym czasie rozsylal zdarzenie po prepare()
i iterowal ta liste (0.4.2, GOON-7A, Edge przy przewijaniu). Listy sluchaczy
odtwarzacza i odtwarzaczy w AudioFocusManager to teraz CopyOnWriteArrayList,
dopisywanie pod synchronized, usuwanie przez atomowe removeIf. Natywne, czeka na APK.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-24 08:58:37 +02:00
9 changed files with 286 additions and 27 deletions

View file

@ -63,9 +63,8 @@ Verify: `curl localhost:8000/health` → `{"status":"ok"}`.
### 2. Install the mobile app (Android) ### 2. Install the mobile app (Android)
Download the latest debug APK from Download the latest APK from [goon-foss.org](https://goon-foss.org)
[GitHub Releases](../../releases/latest) → `goon-vX.Y.Z-debug.apk`, install on (`goon-vX.Y.Z.apk`), install on your Android device (allow "Install from unknown sources" for the browser /
your Android device (allow "Install from unknown sources" for the browser /
file manager you used to download). file manager you used to download).
On first launch the app shows the age-gate disclaimer (must be accepted), then On first launch the app shows the age-gate disclaimer (must be accepted), then

View file

@ -144,7 +144,11 @@ class BaseSearchScraper(BaseDirectTubeScraper):
if len(tok) >= BaseSearchScraper._query_token_min_len if len(tok) >= BaseSearchScraper._query_token_min_len
] ]
if not toks: if not toks:
return True # Nazwa z samych krotkich czlonow ("Wa A", "Al B", "T P", "L A S") nie ma czego
# szukac w slugu. Kiedys zwracalismy tu True, wiec performer-driven doklejal
# takiego wykonawce do KAZDEGO wyniku wyszukiwarki tuba: "Wa A" 498 scen,
# "Al B" 485, "T P" 344 (zgloszenie 0982b000, 2026-09-24). Lepiej nic.
return False
phrase = r"[^a-z0-9]*".join(toks) phrase = r"[^a-z0-9]*".join(toks)
# Dozwolone końcowe „s": slug gubi apostrof, więc dzierżawcze „TiffaniTime's" # Dozwolone końcowe „s": slug gubi apostrof, więc dzierżawcze „TiffaniTime's"
# zapisuje jako „tiffanitimes". Nie luzuje to bramki, bo przed tym „s" i tak # zapisuje jako „tiffanitimes". Nie luzuje to bramki, bo przed tym „s" i tak
@ -173,10 +177,6 @@ class BaseSearchScraper(BaseDirectTubeScraper):
log.debug("%s search %s status=%d", self.sitetag, url, r.status_code) log.debug("%s search %s status=%d", self.sitetag, url, r.status_code)
return return
query_tokens = {
tok for tok in query.lower().split() if len(tok) >= self._query_token_min_len
}
seen: set[str] = set() seen: set[str] = set()
try: try:
thumb_map = self._thumbnail_map(r.text) thumb_map = self._thumbnail_map(r.text)
@ -216,7 +216,7 @@ class BaseSearchScraper(BaseDirectTubeScraper):
# To ma znaczenie, bo `all_performers` niżej dokleja szukanego performera # To ma znaczenie, bo `all_performers` niżej dokleja szukanego performera
# BEZWARUNKOWO do każdej sceny, która tu przejdzie. Ten warunek jest jedyną # BEZWARUNKOWO do każdej sceny, która tu przejdzie. Ten warunek jest jedyną
# bramką między wyszukiwarką tuba (dopasowuje rozmyto) a obsadą w bazie. # bramką między wyszukiwarką tuba (dopasowuje rozmyto) a obsadą w bazie.
if query_tokens and not self._slug_matches_query(slug_lower, query): if not self._slug_matches_query(slug_lower, query):
continue continue
title = self._title_from_slug(slug) title = self._title_from_slug(slug)

View file

@ -19,6 +19,7 @@ from collections.abc import Iterator
from app.connectors.base import RawPerformer, RawPlaybackSource, RawScene from app.connectors.base import RawPerformer, RawPlaybackSource, RawScene
from app.connectors.direct_scrapers._search_base import BaseSearchScraper from app.connectors.direct_scrapers._search_base import BaseSearchScraper
from app.connectors.direct_scrapers.base import BaseDirectTubeScraper from app.connectors.direct_scrapers.base import BaseDirectTubeScraper
from app.connectors.direct_scrapers.hqporner_browse import listing_covers
from app.extractors import browser_get from app.extractors import browser_get
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
@ -54,8 +55,7 @@ class HQPornerScraper(BaseDirectTubeScraper):
# token → sceny z setkami błędnych aktorek (do 503; hqporner = 5659/5897 takich scen, # token → sceny z setkami błędnych aktorek (do 503; hqporner = 5659/5897 takich scen,
# bug-report 2026-06-07). `all` wymaga pełnego dopasowania nazwy → precyzja. # bug-report 2026-06-07). `all` wymaga pełnego dopasowania nazwy → precyzja.
# Pojedyncze nazwy ("Belladonna") nadal działają (jeden token musi być). # Pojedyncze nazwy ("Belladonna") nadal działają (jeden token musi być).
query_tokens = {tok for tok in query.lower().split() if len(tok) >= 3} # Człon <3 znaki ("Lady Vi", "Mia Li", "Ai Uehara") wypada z tokenów bramki, więc
# Człon <3 znaki ("Lady Vi", "Mia Li", "Ai Uehara") wypada z `query_tokens`, więc
# zostaje sam pospolity człon i `all` degeneruje się do „slug zawiera 'lady'", # zostaje sam pospolity człon i `all` degeneruje się do „slug zawiera 'lady'",
# czyli KAŻDA scena z tym słowem (audit 2026-07-28: Lady Vi miała 116 przypisań # czyli KAŻDA scena z tym słowem (audit 2026-07-28: Lady Vi miała 116 przypisań
# z tube, z czego 115 tylko przez „lady", realnie o niej 1). Gdy nazwa ma taki # z tube, z czego 115 tylko przez „lady", realnie o niej 1). Gdy nazwa ma taki
@ -66,6 +66,7 @@ class HQPornerScraper(BaseDirectTubeScraper):
seen_urls: set[str] = set() seen_urls: set[str] = set()
yielded = 0 yielded = 0
covers = listing_covers(r.text)
for m in _SCENE_HREF_RE.finditer(r.text): for m in _SCENE_HREF_RE.finditer(r.text):
scene_id = m.group(1) scene_id = m.group(1)
slug_part = m.group(2) slug_part = m.group(2)
@ -80,7 +81,7 @@ class HQPornerScraper(BaseDirectTubeScraper):
# `all(tok in slug)` przepuszczał „tiffany" w „s-tiffany-love" i „time" # `all(tok in slug)` przepuszczał „tiffany" w „s-tiffany-love" i „time"
# w „first-time" — zgłoszenie 2026-08-13, strona Tiffani Time). # w „first-time" — zgłoszenie 2026-08-13, strona Tiffani Time).
slug_lower = slug_part.lower() slug_lower = slug_part.lower()
if query_tokens and not BaseSearchScraper._slug_matches_query(slug_lower, query): if not BaseSearchScraper._slug_matches_query(slug_lower, query):
continue continue
if _needs_compact and _compact not in re.sub(r"[^a-z0-9]", "", slug_lower): if _needs_compact and _compact not in re.sub(r"[^a-z0-9]", "", slug_lower):
continue continue
@ -95,6 +96,7 @@ class HQPornerScraper(BaseDirectTubeScraper):
RawPlaybackSource( RawPlaybackSource(
origin="tube:hqpornercom", origin="tube:hqpornercom",
page_url=scene_url, page_url=scene_url,
thumbnail_url=covers.get(scene_id),
) )
], ],
# Wymuszamy hint performera = query, search per performer name znaczy # Wymuszamy hint performera = query, search per performer name znaczy

View file

@ -12,7 +12,7 @@ Detail (`/hdporn/<id>-<slug>.html`) ma komplet metadanych:
- długość → meta description "Video duration is 46min 58sec", - długość → meta description "Video duration is 46min 58sec",
- performerki → `<a href="/actress/<slug>">Display Name</a>` (display, nie slug), - performerki → `<a href="/actress/<slug>">Display Name</a>` (display, nie slug),
- tagi → `/category/<slug>`, - tagi → `/category/<slug>`,
- miniaturka → `fastporndelivery.hqporner.com/imgs/.../_main.jpg`. - miniaturki NIE ma: okladka jest tylko w kafelku listingu (patrz _COVER_RE).
`external_id` MUSI być identyczny jak w `HQPornerScraper` (search): `hqpornercom:<url>`. `external_id` MUSI być identyczny jak w `HQPornerScraper` (search): `hqpornercom:<url>`.
Inaczej ta sama scena z obu ścieżek zrobiłaby się dwoma wpisami. Inaczej ta sama scena z obu ścieżek zrobiłaby się dwoma wpisami.
@ -52,7 +52,21 @@ _ACTRESS_RE = re.compile(
re.IGNORECASE | re.DOTALL, re.IGNORECASE | re.DOTALL,
) )
_CATEGORY_RE = re.compile(r'href=[\'"]/category/([a-z0-9\-_]+)[\'"]', re.IGNORECASE) _CATEGORY_RE = re.compile(r'href=[\'"]/category/([a-z0-9\-_]+)[\'"]', re.IGNORECASE)
_THUMB_RE = re.compile(r"fastporndelivery\.hqporner\.com/imgs/[^\"'\s]+_main\.jpg", re.IGNORECASE) # Okladka filmu jest TYLKO w kafelku listingu (`<img id="cover_<id>" src=...>`). Strona
# filmu wlasnej okladki nie ma, a pierwszy `_main.jpg` na niej to kafelek innego,
# polecanego filmu. Braniem go ~90% zrodel hqporner dostalo cudze miniatury (62 tys.
# zrodel, 5,4 tys. roznych miniatur; zgloszenie 0982b000, 2026-09-24).
_COVER_RE = re.compile(
r'id=["\']cover_(\d+)["\']\s+src=["\'](//fastporndelivery\.hqporner\.com/imgs/[^"\']+_main\.jpg)',
re.IGNORECASE,
)
def listing_covers(listing_html: str) -> dict[str, str]:
"""Okladki z kafelkow listingu (glowny, wyszukiwarka, kategorie): {id filmu: url}."""
return {vid: "https:" + src for vid, src in _COVER_RE.findall(listing_html)}
_TAG_STRIP = re.compile(r"<[^>]+>") _TAG_STRIP = re.compile(r"<[^>]+>")
# hqporner wrzuca do /category/ także jakość ("1080p-porn", "4k-porn"). To nie jest tag # hqporner wrzuca do /category/ także jakość ("1080p-porn", "4k-porn"). To nie jest tag
@ -71,6 +85,10 @@ class HQPornerBrowseScraper(BaseBrowseScraper):
return _BASE + "/" if page <= 1 else f"{_BASE}/?p={page}" return _BASE + "/" if page <= 1 else f"{_BASE}/?p={page}"
def _extract_scene_urls(self, listing_html: str) -> list[str]: def _extract_scene_urls(self, listing_html: str) -> list[str]:
# Okladki zapamietujemy tu, bo _parse_detail dostaje juz tylko strone filmu.
if not hasattr(self, "_covers"):
self._covers: dict[str, str] = {}
self._covers.update(listing_covers(listing_html))
seen: set[str] = set() seen: set[str] = set()
out: list[str] = [] out: list[str] = []
for m in _SCENE_HREF_RE.finditer(listing_html): for m in _SCENE_HREF_RE.finditer(listing_html):
@ -122,10 +140,10 @@ class HQPornerBrowseScraper(BaseBrowseScraper):
) )
) )
thumbnail_url: str | None = None # Tylko okladka z kafelka listingu. Brak kafelka = brak miniatury, nigdy obrazek ze
thm = _THUMB_RE.search(detail_html) # strony filmu (patrz _COVER_RE).
if thm: vid = _SCENE_HREF_RE.search(scene_url)
thumbnail_url = "https://" + thm.group(0) thumbnail_url = getattr(self, "_covers", {}).get(vid.group(1)) if vid else None
fingerprints: list[RawFingerprint] = [] fingerprints: list[RawFingerprint] = []
if thumbnail_url: if thumbnail_url:

View file

@ -45,14 +45,16 @@ _BASE = "https://siska.tv"
_HOST_RE = re.compile(r"^https://(?:www\.)?siska\.(?:tv|video)/", re.IGNORECASE) _HOST_RE = re.compile(r"^https://(?:www\.)?siska\.(?:tv|video)/", re.IGNORECASE)
# Kafelek: <a title='..' href='..videoID=N'>. Reszta pól w oknie po tym matchu. # Kafelek: <a title='..' href='..videoID=N'>. Reszta pól w oknie po tym matchu.
# Strona nie escapuje apostrofu w atrybutach (title='Brenna's Mouths'), wiec wartosc
# konczy sie na apostrofie, po ktorym idzie nastepny atrybut, a nie na pierwszym.
_A_RE = re.compile( _A_RE = re.compile(
r"<a\s+title='(?P<title>[^']*)'\s+" r"<a\s+title='(?P<title>[^<>]*?)'\s+"
r"href='(?P<url>https://(?:www\.)?siska\.(?:tv|video)/video\.php\?videoID=\d+)'", r"href='(?P<url>https://(?:www\.)?siska\.(?:tv|video)/video\.php\?videoID=\d+)'",
re.IGNORECASE, re.IGNORECASE,
) )
_DUR_RE = re.compile(r"th_video_duration'>\s*([\d]{1,2}(?:\s*:\s*[\d]{2}){1,2})\s*<") _DUR_RE = re.compile(r"th_video_duration'>\s*([\d]{1,2}(?:\s*:\s*[\d]{2}){1,2})\s*<")
_THUMB_RE = re.compile(r"data-src='([^']+\.(?:jpg|jpeg|webp|png))'", re.IGNORECASE) _THUMB_RE = re.compile(r"data-src='([^']+\.(?:jpg|jpeg|webp|png))'", re.IGNORECASE)
_ALT_RE = re.compile(r"alt='([^']*)'") _ALT_RE = re.compile(r"alt='([^<>]*?)'(?=\s+[\w-]+=|\s*/?>)")
_CAT_RE = re.compile(r"/category/([^/]+)/", re.IGNORECASE) _CAT_RE = re.compile(r"/category/([^/]+)/", re.IGNORECASE)
@ -122,21 +124,22 @@ class SiskaScraper(BaseBrowseScraper):
thumbnail_url = _canonical(tm.group(1)) if tm else None thumbnail_url = _canonical(tm.group(1)) if tm else None
# alt='Performer - Tytuł - Studio' → performer (pierwszy), studio (ostatni). # alt='Performer - Tytuł - Studio' → performer (pierwszy), studio (ostatni).
# Kilka osob stoi w pierwszym czlonie po przecinku ("A, B - Tytul").
performers: list[RawPerformer] = [] performers: list[RawPerformer] = []
studio: RawStudio | None = None studio: RawStudio | None = None
am = _ALT_RE.search(window) am = _ALT_RE.search(window)
if am and " - " in am.group(1): if am and " - " in am.group(1):
parts = [p.strip() for p in am.group(1).split(" - ") if p.strip()] parts = [p.strip() for p in am.group(1).split(" - ") if p.strip()]
if len(parts) >= 2: if len(parts) >= 2:
pname = parts[0]
sname = parts[-1] sname = parts[-1]
if pname: for pname in (n.strip() for n in parts[0].split(",")):
performers.append( if pname:
RawPerformer( performers.append(
external_id=f"{self.sitetag}:performer:{_slugify(pname)}", RawPerformer(
name=pname, external_id=f"{self.sitetag}:performer:{_slugify(pname)}",
name=pname,
)
) )
)
if sname and len(parts) >= 3: if sname and len(parts) >= 3:
studio = RawStudio( studio = RawStudio(
external_id=f"{self.sitetag}:studio:{_slugify(sname)}", external_id=f"{self.sitetag}:studio:{_slugify(sname)}",

View file

@ -1,3 +1,40 @@
diff --git a/node_modules/expo-video/android/src/main/java/expo/modules/video/AudioFocusManager.kt b/node_modules/expo-video/android/src/main/java/expo/modules/video/AudioFocusManager.kt
index aa911b1..671bf05 100644
--- a/node_modules/expo-video/android/src/main/java/expo/modules/video/AudioFocusManager.kt
+++ b/node_modules/expo-video/android/src/main/java/expo/modules/video/AudioFocusManager.kt
@@ -21,7 +21,9 @@ class AudioFocusManager(private val appContext: AppContext) : AudioManager.OnAud
}
}
- private var players: MutableList<WeakReference<VideoPlayer>> = mutableListOf()
+ // CopyOnWriteArrayList z tego samego powodu co listeners w VideoPlayer: register/unregister
+ // przychodza z watku JS i sprzatajacego, a forEach po liscie idzie na glownym (Goon, GOON-7A).
+ private val players = java.util.concurrent.CopyOnWriteArrayList<WeakReference<VideoPlayer>>()
private var currentFocusRequest: AudioFocusRequest? = null
private var currentMixingMode: AudioMixingMode = AudioMixingMode.MIX_WITH_OTHERS
private val anyPlayerRequiresFocus: Boolean
@@ -89,8 +91,10 @@ class AudioFocusManager(private val appContext: AppContext) : AudioManager.OnAud
}
fun registerPlayer(player: VideoPlayer) {
- players.find { it.get() == player } ?: run {
- players.add(WeakReference(player))
+ synchronized(players) {
+ players.find { it.get() == player } ?: run {
+ players.add(WeakReference(player))
+ }
}
player.addListener(this)
updateAudioFocus()
@@ -98,7 +102,7 @@ class AudioFocusManager(private val appContext: AppContext) : AudioManager.OnAud
fun unregisterPlayer(player: VideoPlayer) {
player.removeListener(this)
- players.removeAll { it.get() == player }
+ players.removeIf { it.get() == player }
updateAudioFocus()
}
diff --git a/node_modules/expo-video/android/src/main/java/expo/modules/video/playbackService/ExpoVideoPlaybackService.kt b/node_modules/expo-video/android/src/main/java/expo/modules/video/playbackService/ExpoVideoPlaybackService.kt diff --git a/node_modules/expo-video/android/src/main/java/expo/modules/video/playbackService/ExpoVideoPlaybackService.kt b/node_modules/expo-video/android/src/main/java/expo/modules/video/playbackService/ExpoVideoPlaybackService.kt
index cd798cf..8cb6dfb 100644 index cd798cf..8cb6dfb 100644
--- a/node_modules/expo-video/android/src/main/java/expo/modules/video/playbackService/ExpoVideoPlaybackService.kt --- a/node_modules/expo-video/android/src/main/java/expo/modules/video/playbackService/ExpoVideoPlaybackService.kt
@ -34,3 +71,40 @@ index cd798cf..8cb6dfb 100644
} }
private fun hidePlayerNotification(player: ExoPlayer) { private fun hidePlayerNotification(player: ExoPlayer) {
diff --git a/node_modules/expo-video/android/src/main/java/expo/modules/video/player/VideoPlayer.kt b/node_modules/expo-video/android/src/main/java/expo/modules/video/player/VideoPlayer.kt
index ec7accf..dd365e3 100644
--- a/node_modules/expo-video/android/src/main/java/expo/modules/video/player/VideoPlayer.kt
+++ b/node_modules/expo-video/android/src/main/java/expo/modules/video/player/VideoPlayer.kt
@@ -44,7 +44,11 @@ class VideoPlayer(val context: Context, appContext: AppContext, source: VideoSou
private var renderersFactory = DefaultRenderersFactory(context)
.forceEnableMediaCodecAsynchronousQueueing()
.setEnableDecoderFallback(true)
- private var listeners: MutableList<WeakReference<VideoPlayerListener>> = mutableListOf()
+ // CopyOnWriteArrayList, nie ArrayList: close() (watek JS albo sprzatajacy obiekty) wypisuje
+ // stad AudioFocusManager, a glowny watek w tym samym czasie rozsyla zdarzenie po prepare()
+ // i iteruje liste -> ConcurrentModificationException (Goon 0.4.2, Sentry GOON-7A, Edge
+ // tworzy i zwalnia odtwarzacze przy kazdym przewinieciu). Iteracja idzie po migawce.
+ private val listeners = java.util.concurrent.CopyOnWriteArrayList<WeakReference<VideoPlayerListener>>()
val loadControl: VideoPlayerLoadControl = VideoPlayerLoadControl.Builder().build()
val subtitles: VideoPlayerSubtitles = VideoPlayerSubtitles(this)
val trackSelector = DefaultTrackSelector(context)
@@ -320,13 +324,16 @@ class VideoPlayer(val context: Context, appContext: AppContext, source: VideoSou
}
fun addListener(videoPlayerListener: VideoPlayerListener) {
- if (listeners.all { it.get() != videoPlayerListener }) {
- listeners.add(WeakReference(videoPlayerListener))
+ synchronized(listeners) {
+ if (listeners.all { it.get() != videoPlayerListener }) {
+ listeners.add(WeakReference(videoPlayerListener))
+ }
}
}
fun removeListener(videoPlayerListener: VideoPlayerListener) {
- listeners.removeAll { it.get() == videoPlayerListener }
+ // removeIf listy (atomowe), nie kotlinowe removeAll { }, ktore usuwa po indeksach.
+ listeners.removeIf { it.get() == videoPlayerListener }
}
private fun sendEvent(event: PlayerEvent) {

View file

@ -0,0 +1,104 @@
"""Naprawa miniatur hqporner: okladki z kafelkow listingu zamiast cudzych z detali.
Scraper bral pierwszy `_main.jpg` ze strony filmu, a to kafelek polecanego filmu
(app/connectors/direct_scrapers/hqporner_browse.py, _COVER_RE). Skrypt przechodzi
listing `https://hqporner.com/?p=N`, zbiera {id filmu: okladka} i poprawia
playback_sources.thumbnail_url zrodel tube:hqpornercom.
Uzycie: python3 -u scripts/hqporner_fix_thumbs.py [--commit] [--max-pages N] [--start N]
Bez --commit tylko liczy. Postep co 50 stron; wznawianie przez --start.
"""
from __future__ import annotations
import argparse
import re
import time
from sqlalchemy import text
from app.connectors.direct_scrapers.hqporner_browse import listing_covers
from app.db import session_scope
from app.extractors import browser_get
_BASE = "https://hqporner.com"
_ID_RE = re.compile(r"/hdporn/(\d+)-")
def _load_sources() -> dict[str, list[tuple[str, str | None]]]:
"""{id filmu: [(id zrodla, obecna miniatura)]} dla zrodel hqporner."""
out: dict[str, list[tuple[str, str | None]]] = {}
with session_scope() as s:
for sid, page, thumb in s.execute(
text("SELECT id, page_url, thumbnail_url FROM playback_sources WHERE origin = 'tube:hqpornercom'")
):
m = _ID_RE.search(page or "")
if m:
out.setdefault(m.group(1), []).append((str(sid), thumb))
return out
def _apply(fixes: list[tuple[str, str]], commit: bool) -> None:
if not fixes or not commit:
return
with session_scope() as s:
s.execute(
text("UPDATE playback_sources SET thumbnail_url = :t, updated_at = now() WHERE id = CAST(:i AS uuid)"),
[{"i": i, "t": t} for i, t in fixes],
)
def main() -> None:
ap = argparse.ArgumentParser()
ap.add_argument("--commit", action="store_true")
ap.add_argument("--max-pages", type=int, default=5000)
ap.add_argument("--start", type=int, default=1)
ap.add_argument("--pause", type=float, default=0.3)
args = ap.parse_args()
sources = _load_sources()
print(f"zrodel hqporner: {sum(len(v) for v in sources.values())}, filmow: {len(sources)}", flush=True)
seen_ids: set[str] = set()
fixed = same = empty_streak = 0
pending: list[tuple[str, str]] = []
t0 = time.time()
for page in range(args.start, args.start + args.max_pages):
url = _BASE + "/" if page <= 1 else f"{_BASE}/?p={page}"
covers: dict[str, str] = {}
for attempt in range(3):
try:
covers = listing_covers(browser_get(url, timeout=40).text)
break
except Exception as e: # noqa: BLE001
print(f" strona {page} proba {attempt + 1}: {e}", flush=True)
time.sleep(5)
new = {k: v for k, v in covers.items() if k not in seen_ids}
if not new:
empty_streak += 1
if empty_streak >= 3:
print(f"koniec listingu na stronie {page}", flush=True)
break
else:
empty_streak = 0
seen_ids.update(new)
for vid, cover in new.items():
for sid, thumb in sources.get(vid, []):
if thumb == cover:
same += 1
else:
pending.append((sid, cover))
fixed += 1
if page % 50 == 0:
_apply(pending, args.commit)
pending.clear()
print(f"strona {page}: filmow w listingu {len(seen_ids)}, poprawionych {fixed}, "
f"juz dobrych {same}, {time.time() - t0:.0f} s", flush=True)
time.sleep(args.pause)
_apply(pending, args.commit)
ours = set(sources)
print(f"KONIEC: filmow w listingu {len(seen_ids)}, naszych filmow {len(ours)}, "
f"znalezionych {len(ours & seen_ids)}, poprawionych zrodel {fixed}, juz dobrych {same}, "
f"{'ZAPISANE' if args.commit else 'bez zapisu'}, {time.time() - t0:.0f} s", flush=True)
if __name__ == "__main__":
main()

View file

@ -0,0 +1,18 @@
from app.connectors.direct_scrapers._search_base import BaseSearchScraper
gate = BaseSearchScraper._slug_matches_query
def test_short_only_names_never_match():
# Kiedys True dla kazdego slugu: "Wa A" trafial do 498 scen, "Al B" do 485.
for name in ("Wa A", "Al B", "T P", "Ms P", "L A S", "Cl O"):
assert not gate("tidy-up", name)
assert not gate("wa-a-hot-scene", name)
def test_phrase_gate_unchanged_for_normal_names():
assert gate("tiffani-time-first-scene", "Tiffani Time")
assert not gate("first-time-with-tiffani", "Tiffani Time")
assert gate("violet_myers_prodigious", "Violet Myers")
# czlon < 3 znakow pomijany, reszta frazy dalej musi pasowac
assert gate("mia-li-in-the-shower", "Mia Li")

View file

@ -0,0 +1,41 @@
from app.connectors.direct_scrapers import siska
from app.connectors.direct_scrapers.siska import SiskaScraper
# Dwa kafelki jak na siska.tv 2026-09-27; pierwszy z nieescapowanym apostrofem.
_LISTING = """
<li class='pure-u-1-3'><div class='back'><div class='thumb'><div class='rel'>
<a title=' Filling Chanel and Brenna's Mouths' href='https://siska.tv/video.php?videoID=226988' class='video-thumb'><span class='icon-hd'></span><span class='th_video_duration'>36 : 58</span><span class='th_photo_count'><span class='ico-photo'></span></span><img src='data:image/png;base64,R0lGOD' data-src='https://siska.tv/category/Deep-Throat/1049123346.jpg' width='640' height='360' alt='Chanel Camryn, Brenna McKenna - Filling Chanel and Brenna's Mouths' class='pure-img padding' /></a></div>
</div></div></li>
<li class='pure-u-1-3'><div class='back'><div class='thumb'><div class='rel'>
<a title='Plain Title' href='https://www.siska.video/video.php?videoID=226987' class='video-thumb'><span class='th_video_duration'>1 : 02 : 03</span><img data-src='https://siska.tv/category/Anal/1049123345.jpg' alt='Jane Doe - Plain Title - Some Studio' class='pure-img' /></a></div>
</div></div></li>
"""
class _Res:
text = _LISTING
def _crawl(monkeypatch):
monkeypatch.setattr(siska, "browser_get", lambda url, timeout=None: _Res())
monkeypatch.setattr(siska, "compute_thumbnail_phash", lambda *a, **k: None)
return SiskaScraper().crawl_page(1)
def test_apostrophe_in_title_keeps_tile(monkeypatch):
scenes = _crawl(monkeypatch)
assert [s.url for s in scenes] == [
"https://siska.tv/video.php?videoID=226988",
"https://siska.tv/video.php?videoID=226987",
]
assert scenes[0].title == "Filling Chanel and Brenna's Mouths"
assert scenes[0].duration_sec == 36 * 60 + 58
def test_comma_separated_performers_are_split(monkeypatch):
first, second = _crawl(monkeypatch)
assert [p.name for p in first.performers] == ["Chanel Camryn", "Brenna McKenna"]
# dwa czlony alt = osoby + tytul, studia brak
assert first.studio is None
assert [p.name for p in second.performers] == ["Jane Doe"]
assert second.studio.name == "Some Studio"