37f8eb1762
Some checks are pending
Backend tests / test (push) Waiting to run
Mangoporn, streamporn.li i pandamovies przeszly w pazdzierniku na nowy motyw: obsada tylko w JSON-LD, studio, rok i gatunki w chipach bez rel="tag" (czasem bez koncowego ukosnika). Parser po cichu przestal czytac obsade i studio, a mangoporn dostawal jedynie tag "Porn Movies". Pandamovies dziala znowu pod pandamovies.org, wiec wraca do ingestu. Skrypty: pelne zaciagniecie zrodla po przerwie i uzupelnienie metadanych filmow z pazdziernika. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
88 lines
3.3 KiB
Python
88 lines
3.3 KiB
Python
"""Uzupelnia obsade, studio i tagi filmow z serwisow dooplay po zmianie motywu (2026-10).
|
|
|
|
Mangoporn, streamporn.li i pandamovies przeszly na motyw "vid": obsada tylko w JSON-LD,
|
|
studio i rok w chipach bez rel="tag". Parser tego nie czytal, wiec filmy dodane od ok.
|
|
5.10 przyszly bez obsady i czesto bez studia (cichy fail parsera, ingest konczyl sie
|
|
"errors: 0"). Skrypt pobiera ich strony na nowo i puszcza przez resolve_movie, ktory przy
|
|
znanym refie tylko uzupelnia puste pola i dopina obsade/tagi. Strony przez proxy, bo
|
|
mangoporn z VPS bywa niedostepny (timeout, blad TLS).
|
|
|
|
python3 /srv/scripts/backfill_dooplay_metadata.py # dry-run: tylko liczy
|
|
python3 /srv/scripts/backfill_dooplay_metadata.py --apply [--since 2026-10-01]
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import functools
|
|
import logging
|
|
|
|
import app.connectors.dooplay as dp
|
|
from app.config import get_settings
|
|
|
|
dp.browser_get = functools.partial(dp.browser_get, proxy=get_settings().brightdata_proxy_url)
|
|
|
|
from sqlalchemy import text # noqa: E402
|
|
|
|
from app.db import session_scope # noqa: E402
|
|
from app.normalize.movies import normalize_movie # noqa: E402
|
|
from app.resolve.movie_resolver import resolve_movie # noqa: E402
|
|
|
|
_CONNECTORS = {
|
|
"mangoporn": dp.MangopornConnector,
|
|
"streampornvip": dp.StreampornVipConnector,
|
|
"pandamovies": dp.PandamoviesConnector,
|
|
}
|
|
|
|
_REFS_SQL = """
|
|
SELECT r.movie_id, r.external_id, r.url, so.id AS source_id, so.name
|
|
FROM movie_external_refs r
|
|
JOIN sources so ON so.id = r.source_id
|
|
JOIN movies m ON m.id = r.movie_id
|
|
WHERE so.name = ANY(:names) AND m.created_at >= :since AND r.url IS NOT NULL
|
|
AND (m.studio_id IS NULL
|
|
OR NOT EXISTS (SELECT 1 FROM movie_performers p WHERE p.movie_id = m.id))
|
|
ORDER BY m.created_at DESC
|
|
"""
|
|
|
|
|
|
def main() -> None:
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--apply", action="store_true")
|
|
ap.add_argument("--since", default="2026-10-01")
|
|
args = ap.parse_args()
|
|
logging.basicConfig(level=logging.WARNING)
|
|
|
|
with session_scope() as s:
|
|
refs = s.execute(text(_REFS_SQL), {"names": list(_CONNECTORS), "since": args.since}).all()
|
|
print(f"refow do uzupelnienia: {len(refs)}", flush=True)
|
|
if not args.apply:
|
|
return
|
|
|
|
conns = {name: cls() for name, cls in _CONNECTORS.items()}
|
|
done = failed = with_cast = 0
|
|
for n, ref in enumerate(refs, 1):
|
|
try:
|
|
raw = conns[ref.name]._fetch_detail(ref.url)
|
|
except Exception as e: # noqa: BLE001
|
|
failed += 1
|
|
print(f" blad {ref.name} {ref.url}: {str(e)[:100]}", flush=True)
|
|
continue
|
|
if raw is None:
|
|
failed += 1
|
|
continue
|
|
norm = normalize_movie(raw)
|
|
if norm.external_id != ref.external_id:
|
|
failed += 1
|
|
print(f" inny id {ref.external_id} -> {norm.external_id}, pomijam", flush=True)
|
|
continue
|
|
with session_scope() as s:
|
|
resolve_movie(s, norm=norm, source_id=ref.source_id)
|
|
done += 1
|
|
with_cast += bool(raw.performers)
|
|
if n % 25 == 0:
|
|
print(f"... {n}/{len(refs)}, uzupelnione {done}, z obsada {with_cast}, bledy {failed}", flush=True)
|
|
print(f"uzupelnione: {done}, z obsada: {with_cast}, bledy: {failed}", flush=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|