fix(movies): yourdailypornmovies gubil nowe filmy przez nowy format okladki
Od ~15.09 nowe karty maja okladke imgs4cdn.adultempire.com/movie/<id>?w=... (czesc i.ibb.co albo bez okladki), a regex karty wymagal starego products/NN/<id>h.jpg. 33 z 40 filmow strony 1 odpadalo po cichu, ingest przez tydzien: 0 nowych. Okladka teraz opcjonalna i w dowolnym formacie, id Adult Empire z obu wariantow. Nadrobione: 26 nowych filmow, 6 dopietych do istniejacych. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
45ee7a81e1
commit
ed7c1cd3ed
2 changed files with 51 additions and 5 deletions
|
|
@ -51,14 +51,19 @@ log = logging.getLogger(__name__)
|
||||||
|
|
||||||
_BASE = "https://yourdailypornmovies.ws"
|
_BASE = "https://yourdailypornmovies.ws"
|
||||||
|
|
||||||
# Karta listingu: id produktu Adult Empire w miniaturce + URL + tytuł + rok.
|
# Karta listingu: okładka + URL + tytuł + rok. Okładka jest opcjonalna i w dowolnym
|
||||||
|
# formacie: od ~2026-09-15 nowe filmy mają `imgs4cdn.adultempire.com/movie/<id>?w=...`
|
||||||
|
# zamiast `.../products/NN/<id>h.jpg`, część ma okładkę z i.ibb.co. Regex wymagający
|
||||||
|
# starego formatu po cichu gubił 33 z 40 filmów strony 1 (ingest: 0 nowych przez tydzień).
|
||||||
_CARD_RE = re.compile(
|
_CARD_RE = re.compile(
|
||||||
r'<div class="movie"><div class="imagen">\s*'
|
r'<div class="movie"><div class="imagen">\s*'
|
||||||
r'<img src="(?P<poster>https://imgs\d*cdn\.adultempire\.com/products/\d+/(?P<ae>\d+)h\.jpg)"'
|
r'(?:<img src="(?P<poster>[^"]*)"[^>]*>\s*)?'
|
||||||
r'[^>]*>\s*<a href="(?P<url>https://yourdailypornmovies\.ws/[^"]+)">.*?'
|
r'<a href="(?P<url>https://yourdailypornmovies\.ws/[^"]+)">.*?'
|
||||||
r'</div><h2>(?P<title>[^<]+)</h2>\s*<span class="year">(?P<year>\d{4})</span>',
|
r'</div><h2>(?P<title>[^<]+)</h2>\s*<span class="year">(?P<year>\d{4})</span>',
|
||||||
re.DOTALL,
|
re.DOTALL,
|
||||||
)
|
)
|
||||||
|
# Id produktu Adult Empire z adresu okładki, w obu formatach (to ta sama numeracja).
|
||||||
|
_AE_ID_RE = re.compile(r"adultempire\.com/(?:products/\d+/(\d+)h\.jpg|movie/(\d+))")
|
||||||
|
|
||||||
# Blok metadanych filmu. WSZYSTKO czytamy TYLKO stąd, patrz pułapka w docstringu.
|
# Blok metadanych filmu. WSZYSTKO czytamy TYLKO stąd, patrz pułapka w docstringu.
|
||||||
_XMLL_RE = re.compile(r'<div class="xmll">.*?</div>', re.DOTALL)
|
_XMLL_RE = re.compile(r'<div class="xmll">.*?</div>', re.DOTALL)
|
||||||
|
|
@ -208,18 +213,21 @@ class YourDailyPornMoviesConnector(BaseMovieConnector):
|
||||||
desc = html_mod.unescape(om.group(1)).strip() or None
|
desc = html_mod.unescape(om.group(1)).strip() or None
|
||||||
|
|
||||||
slug = urlparse(url).path.strip("/").split("/")[-1]
|
slug = urlparse(url).path.strip("/").split("/")[-1]
|
||||||
|
poster = html_mod.unescape(card.group("poster") or "").strip() or None
|
||||||
|
aem = _AE_ID_RE.search(poster or "")
|
||||||
|
ae_id = (aem.group(1) or aem.group(2)) if aem else None
|
||||||
return RawMovie(
|
return RawMovie(
|
||||||
external_id=slug,
|
external_id=slug,
|
||||||
title=title,
|
title=title,
|
||||||
description=desc,
|
description=desc,
|
||||||
release_year=year,
|
release_year=year,
|
||||||
url=url,
|
url=url,
|
||||||
poster_url=card.group("poster"),
|
poster_url=poster,
|
||||||
studio=studio,
|
studio=studio,
|
||||||
performers=performers,
|
performers=performers,
|
||||||
tags=tags,
|
tags=tags,
|
||||||
playback_sources=playback,
|
playback_sources=playback,
|
||||||
# id produktu Adult Empire, mocniejsza kotwica do dedupu niż tytuł
|
# id produktu Adult Empire, mocniejsza kotwica do dedupu niż tytuł
|
||||||
# (przy ocenie źródła złapało 9 filmów, których tytuł się nie zgadzał).
|
# (przy ocenie źródła złapało 9 filmów, których tytuł się nie zgadzał).
|
||||||
raw={"source": self.name, "url": url, "adultempire_id": card.group("ae")},
|
raw={"source": self.name, "url": url, "adultempire_id": ae_id},
|
||||||
)
|
)
|
||||||
|
|
|
||||||
38
tests/test_yourdailypornmovies_cards.py
Normal file
38
tests/test_yourdailypornmovies_cards.py
Normal file
|
|
@ -0,0 +1,38 @@
|
||||||
|
from app.connectors.yourdailypornmovies import _AE_ID_RE, _CARD_RE
|
||||||
|
|
||||||
|
_OLD = (
|
||||||
|
'<div class="movie"><div class="imagen"> <img src="https://imgs1cdn.adultempire.com/products/05/5128705h.jpg" '
|
||||||
|
'alt="Ramming Into Her" width="100%" height="100%" /> <a href="https://yourdailypornmovies.ws/ramming-into-her/">'
|
||||||
|
'<span class="player"></span></a></div><h2>Ramming Into Her</h2> <span class="year">2026</span></div>'
|
||||||
|
)
|
||||||
|
_NEW = (
|
||||||
|
'<div class="movie"><div class="imagen"> <img src="https://imgs4cdn.adultempire.com/movie/5136742?w=500&h=700&fit=cover" '
|
||||||
|
'alt="Big Butts Drive Me Nuts 8" width="100%" height="100%" /> <a href="https://yourdailypornmovies.ws/big-butts-drive-me-nuts-8/">'
|
||||||
|
'<span class="player"></span></a></div><h2>Big Butts Drive Me Nuts 8</h2> <span class="year">2026</span></div>'
|
||||||
|
)
|
||||||
|
_IBB = (
|
||||||
|
'<div class="movie"><div class="imagen"> <img src="https://i.ibb.co/DHDd9pm9/Private-POV.jpg" alt="Private POV" '
|
||||||
|
'width="100%" height="100%" /> <a href="https://yourdailypornmovies.ws/private-pov/"><span class="player"></span></a>'
|
||||||
|
'</div><h2>Private POV</h2> <span class="year">2026</span></div>'
|
||||||
|
)
|
||||||
|
_NO_IMG = (
|
||||||
|
'<div class="movie"><div class="imagen"> <a href="https://yourdailypornmovies.ws/no-cover/"><span class="player">'
|
||||||
|
'</span></a></div><h2>No Cover</h2> <span class="year">2025</span></div>'
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_all_card_variants_parse():
|
||||||
|
cards = list(_CARD_RE.finditer(_OLD + _NEW + _IBB + _NO_IMG))
|
||||||
|
assert [c.group("title") for c in cards] == [
|
||||||
|
"Ramming Into Her", "Big Butts Drive Me Nuts 8", "Private POV", "No Cover"
|
||||||
|
]
|
||||||
|
assert cards[3].group("poster") is None
|
||||||
|
assert cards[3].group("url") == "https://yourdailypornmovies.ws/no-cover/"
|
||||||
|
|
||||||
|
|
||||||
|
def test_adultempire_id_from_both_poster_formats():
|
||||||
|
old = _AE_ID_RE.search("https://imgs1cdn.adultempire.com/products/05/5128705h.jpg")
|
||||||
|
new = _AE_ID_RE.search("https://imgs4cdn.adultempire.com/movie/5136742?w=500&h=700&fit=cover")
|
||||||
|
assert (old.group(1) or old.group(2)) == "5128705"
|
||||||
|
assert (new.group(1) or new.group(2)) == "5136742"
|
||||||
|
assert _AE_ID_RE.search("https://i.ibb.co/DHDd9pm9/Private-POV.jpg") is None
|
||||||
Loading…
Add table
Reference in a new issue