7895148c0f
Some checks are pending
Backend tests / test (push) Waiting to run
Addresses the ultra-review findings on this branch:
Player (PlayerScreen.tsx): the new recoveryPending mirrored the fallback-chain guards
by hand and could deadlock into a permanent "Reconnecting" spinner with no way to Mark
broken — for gone (410) sources on IP-bound tubes (re-resolve bails before setting
reResolveDone) and for any post-load error on those tubes (re-resolve is initial-load
only). Derive one reResolveApplicable flag (IP-bound AND initial-load AND not-gone) and
use it for both the chain gate and the spinner, so gone/post-load errors fall through to
proxy/WebView or the terminal error card. Seek-recovery now falls through to the chain
when player.replace() throws instead of returning.
Quick-play (SceneDetail): the autoplay route param persisted and autoPlay={i===0} re-fired
when the source list reordered (e.g. after Mark broken drops the dead source), bouncing the
user into the player. Consume it once via onAutoPlayConsumed -> nav.setParams({autoplay:false}).
Backfill semantics: performer-driven direct-scraper "backward fill" now tags scenes
backfill=True (search-by-name pulls the whole old catalog); merge coalesces backfill
(keep AND drop) so a fresh scene merged into a dead dup keeps NEW; deep-crawl only tags
backfill on a tube's FIRST sweep (swept_once) so re-sweep catalog growth stays genuine;
pilot script tags backfill.
Perf/migration: migration 0026 is now idempotent (IF NOT EXISTS; prod got the column via
manual ALTER) and adds ix_scene_performers_performer_id (favorites count filtered
performer_id with no index); index also created on prod.
Cleanup: deleted dead FavoriteSceneRow (unused import in two screens, stale isNew without
the backfill guard); removed em-dashes from all lines this branch added (user CLAUDE.md
rule), including the user-facing changelog / Settings / player-overlay strings.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
78 lines
2.9 KiB
Python
78 lines
2.9 KiB
Python
"""Pilot (Faza 1) — deep-crawl porndoe poza najnowsze strony, żeby zmierzyć WARTOŚĆ
|
|
pełnego crawlu tube'a (vs obecne search+top-N). Mamy ~3% katalogu porndoe (1959/62k+).
|
|
|
|
Crawluje strony START..END (domyślnie 64+, czyli ogon którego jeszcze nie mamy),
|
|
przepuszcza przez normalny `_process_scene` (resolver: match canonical / orphan + tagi
|
|
+ duration). Mierzy counters. NIE modyfikuje produkcyjnych jobów — to ad-hoc pomiar.
|
|
|
|
Użycie: python scripts/pilot_porndoe_deepcrawl.py --start 64 --end 110
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import logging
|
|
import sys
|
|
import time
|
|
|
|
from app.connectors.direct_scrapers.porndoe import PornDoeScraper
|
|
from app.db import session_scope
|
|
from app.extractors import browser_get
|
|
from app.ingest import _process_scene, get_or_create_source
|
|
from app.models.source import SourceKind
|
|
|
|
logging.basicConfig(level=logging.WARNING, format="%(asctime)s %(levelname)s %(message)s")
|
|
log = logging.getLogger("pilot_porndoe")
|
|
|
|
|
|
def main() -> int:
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--start", type=int, default=64)
|
|
ap.add_argument("--end", type=int, default=110)
|
|
args = ap.parse_args()
|
|
|
|
s = PornDoeScraper()
|
|
with session_scope() as session:
|
|
from app.connectors.direct_scrapers import SCRAPER_SOURCE_NAME
|
|
src = get_or_create_source(session, kind=SourceKind.scraper, name=SCRAPER_SOURCE_NAME)
|
|
source_id = src.id
|
|
|
|
counters = {"seen": 0, "new": 0, "updated": 0, "skipped": 0, "errors": 0}
|
|
t0 = time.time()
|
|
for page in range(args.start, args.end + 1):
|
|
try:
|
|
res = browser_get(s._listing_url(page), timeout=30)
|
|
html = res.text if hasattr(res, "text") else res
|
|
except Exception as e:
|
|
log.warning("listing page %d failed: %s", page, e)
|
|
continue
|
|
urls = s._extract_scene_urls(html)
|
|
if not urls:
|
|
print(f"empty listing page {page}, stop")
|
|
break
|
|
for u in urls:
|
|
try:
|
|
r = browser_get(u, timeout=30)
|
|
dh = r.text if hasattr(r, "text") else r
|
|
raw = s._parse_detail(u, dh)
|
|
except Exception:
|
|
counters["errors"] += 1
|
|
continue
|
|
if raw is None:
|
|
continue
|
|
counters["seen"] += 1
|
|
try:
|
|
# deep-crawl pilot = backfill katalogu (stary content z fałszywą datą
|
|
# tube), więc nie licz jako "nowe" (spójne z app/scheduler/deep_crawl.py).
|
|
_process_scene(
|
|
source_id=source_id, raw_scene=raw, counters=counters, backfill=True
|
|
)
|
|
except Exception:
|
|
counters["errors"] += 1
|
|
print(f"page {page}: {counters} ({time.time() - t0:.0f}s)", flush=True)
|
|
|
|
print(f"PILOT DONE pages {args.start}-{args.end}: {counters} elapsed={time.time() - t0:.0f}s")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|