Compare commits
No commits in common. "0da36d5c1e0db76fc9213f16a47dd16ec9cfdfa1" and "45ee7a81e11f34bf4a177b348ebab556585ff1ec" have entirely different histories.
0da36d5c1e
...
45ee7a81e1
6 changed files with 39 additions and 113 deletions
|
|
@ -1851,7 +1851,7 @@ def enrich_tags_from_tube(
|
||||||
from app.models.tag import Tag
|
from app.models.tag import Tag
|
||||||
from app.normalize.scenes import NormalizedTag
|
from app.normalize.scenes import NormalizedTag
|
||||||
from app.normalize.text import slugify
|
from app.normalize.text import slugify
|
||||||
from app.resolve.tag_resolver import resolve_tag, resolve_tags_by_altkey
|
from app.resolve.tag_resolver import resolve_tag
|
||||||
|
|
||||||
scene = session.get(Scene, scene_id)
|
scene = session.get(Scene, scene_id)
|
||||||
if scene is None:
|
if scene is None:
|
||||||
|
|
@ -1923,17 +1923,10 @@ def enrich_tags_from_tube(
|
||||||
{t.slug: t for t in session.execute(select(Tag).where(Tag.slug.in_(direct))).scalars()}
|
{t.slug: t for t in session.execute(select(Tag).where(Tag.slug.in_(direct))).scalars()}
|
||||||
if direct else {}
|
if direct else {}
|
||||||
)
|
)
|
||||||
# Nazwy spoza slugow: wariant pisowni istniejacego tagu (alnum-klucz, liczba mnoga)
|
|
||||||
# tez jednym zapytaniem. resolve_tag zostaje tylko dla naprawde nowych nazw.
|
|
||||||
direct_set = set(direct)
|
|
||||||
by_alt = resolve_tags_by_altkey(
|
|
||||||
session,
|
|
||||||
[n for n in tag_names if slugify(n) in direct_set and slugify(n) not in by_slug],
|
|
||||||
)
|
|
||||||
tag_ids: list = []
|
tag_ids: list = []
|
||||||
for name in tag_names:
|
for name in tag_names:
|
||||||
slug = slugify(name)
|
slug = slugify(name)
|
||||||
tag = by_slug.get(slug) or by_alt.get(name)
|
tag = by_slug.get(slug)
|
||||||
if tag is None:
|
if tag is None:
|
||||||
tag = resolve_tag(session, norm=NormalizedTag(name=name, slug=slug, external_id=None))
|
tag = resolve_tag(session, norm=NormalizedTag(name=name, slug=slug, external_id=None))
|
||||||
if tag is not None and tag.id not in tag_ids:
|
if tag is not None and tag.id not in tag_ids:
|
||||||
|
|
|
||||||
|
|
@ -110,7 +110,7 @@ class SourceListOut(BaseModel):
|
||||||
# tubey mają nietypowe TLDs / brakujące kropki w sitetag.
|
# tubey mają nietypowe TLDs / brakujące kropki w sitetag.
|
||||||
_DISPLAY_OVERRIDES: dict[str, str] = {
|
_DISPLAY_OVERRIDES: dict[str, str] = {
|
||||||
"fpoxxx": "fpo.xxx",
|
"fpoxxx": "fpo.xxx",
|
||||||
"siskavideo": "siska.tv",
|
"siskavideo": "siska.video",
|
||||||
"porn4dayspw": "porn4days.pw",
|
"porn4dayspw": "porn4days.pw",
|
||||||
"porn00org": "porn00.org",
|
"porn00org": "porn00.org",
|
||||||
"freshpornoorg": "freshporno.org",
|
"freshpornoorg": "freshporno.org",
|
||||||
|
|
|
||||||
|
|
@ -1,8 +1,8 @@
|
||||||
"""siska.tv (do 2026-09-22 siska.video), latest-vids browse scraper.
|
"""siska.video, latest-vids browse scraper.
|
||||||
|
|
||||||
Historia: dawniej performer-driven search scraper (`?s=<q>`), ale siska zepsuła
|
Historia: dawniej performer-driven search scraper (`?s=<q>`), ale siska zepsuła
|
||||||
wyszukiwarkę (zwraca latest niezależnie od query). Przerobione na BROWSE (latest
|
wyszukiwarkę (zwraca latest niezależnie od query). Przerobione na BROWSE (latest
|
||||||
chronologicznie z listingu, patrz `_listing_url`), re-enabled 2026-06-20 (user fa4083a2).
|
chronologicznie z `/page/<n>/`), re-enabled 2026-06-20 (user fa4083a2).
|
||||||
|
|
||||||
Cały blok kafelka listingu ma komplet metadanych (zero detail-fetchy):
|
Cały blok kafelka listingu ma komplet metadanych (zero detail-fetchy):
|
||||||
<a title='<Tytuł>' href='https://siska.video/video.php?videoID=<n>'>
|
<a title='<Tytuł>' href='https://siska.video/video.php?videoID=<n>'>
|
||||||
|
|
@ -36,18 +36,11 @@ from app.extractors import browser_get
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
# Domena siska.video wygasla 2026-09-22 (NXDOMAIN), ten sam serwis dziala dalej pod
|
_BASE = "https://siska.video"
|
||||||
# siska.tv: te same numery videoID i sciezki miniatur. Adresy w bazie przepisane na
|
|
||||||
# siska.tv tego samego dnia. Kafelki przyjmujemy z obu domen i z www, ale zapisujemy
|
|
||||||
# zawsze jeden kanoniczny adres (_canonical), bo external_id sceny i unikalny klucz
|
|
||||||
# zrodla to sam adres; www.siska.video i siska.video dawaly 103 zdublowane zrodla.
|
|
||||||
_BASE = "https://siska.tv"
|
|
||||||
_HOST_RE = re.compile(r"^https://(?:www\.)?siska\.(?:tv|video)/", re.IGNORECASE)
|
|
||||||
|
|
||||||
# Kafelek: <a title='..' href='..videoID=N'>. Reszta pól w oknie po tym matchu.
|
# Kafelek: <a title='..' href='..videoID=N'>. Reszta pól w oknie po tym matchu.
|
||||||
_A_RE = re.compile(
|
_A_RE = re.compile(
|
||||||
r"<a\s+title='(?P<title>[^']*)'\s+"
|
r"<a\s+title='(?P<title>[^']*)'\s+href='(?P<url>https://siska\.video/video\.php\?videoID=\d+)'",
|
||||||
r"href='(?P<url>https://(?:www\.)?siska\.(?:tv|video)/video\.php\?videoID=\d+)'",
|
|
||||||
re.IGNORECASE,
|
re.IGNORECASE,
|
||||||
)
|
)
|
||||||
_DUR_RE = re.compile(r"th_video_duration'>\s*([\d]{1,2}(?:\s*:\s*[\d]{2}){1,2})\s*<")
|
_DUR_RE = re.compile(r"th_video_duration'>\s*([\d]{1,2}(?:\s*:\s*[\d]{2}){1,2})\s*<")
|
||||||
|
|
@ -56,10 +49,6 @@ _ALT_RE = re.compile(r"alt='([^']*)'")
|
||||||
_CAT_RE = re.compile(r"/category/([^/]+)/", re.IGNORECASE)
|
_CAT_RE = re.compile(r"/category/([^/]+)/", re.IGNORECASE)
|
||||||
|
|
||||||
|
|
||||||
def _canonical(url: str) -> str:
|
|
||||||
return _HOST_RE.sub(_BASE + "/", url)
|
|
||||||
|
|
||||||
|
|
||||||
def _parse_duration(text: str | None) -> int | None:
|
def _parse_duration(text: str | None) -> int | None:
|
||||||
"""`40 : 27` → 2427 (MM:SS); `1 : 05 : 30` → H:MM:SS. None gdy brak."""
|
"""`40 : 27` → 2427 (MM:SS); `1 : 05 : 30` → H:MM:SS. None gdy brak."""
|
||||||
if not text:
|
if not text:
|
||||||
|
|
@ -84,13 +73,11 @@ class SiskaScraper(BaseBrowseScraper):
|
||||||
sitetag = "siskavideo"
|
sitetag = "siskavideo"
|
||||||
|
|
||||||
def _listing_url(self, page: int) -> str:
|
def _listing_url(self, page: int) -> str:
|
||||||
# Na siska.tv `/page/<n>/` oddaje zawsze strone 1. Najnowsze po kolei (18 na
|
return f"{_BASE}/page/{page}/"
|
||||||
# strone, videoID malejaco) sa pod `best_xvideos.php`, mimo nazwy.
|
|
||||||
return f"{_BASE}/best_xvideos.php?page={page}"
|
|
||||||
|
|
||||||
# crawl_page nadpisany → poniższe abstrakcje nieużywane, ale wymagane do instancji.
|
# crawl_page nadpisany → poniższe abstrakcje nieużywane, ale wymagane do instancji.
|
||||||
def _extract_scene_urls(self, listing_html: str) -> list[str]:
|
def _extract_scene_urls(self, listing_html: str) -> list[str]:
|
||||||
return [_canonical(m.group("url")) for m in _A_RE.finditer(listing_html)]
|
return [m.group("url") for m in _A_RE.finditer(listing_html)]
|
||||||
|
|
||||||
def _parse_detail(self, scene_url: str, detail_html: str) -> RawScene | None:
|
def _parse_detail(self, scene_url: str, detail_html: str) -> RawScene | None:
|
||||||
return None
|
return None
|
||||||
|
|
@ -107,7 +94,7 @@ class SiskaScraper(BaseBrowseScraper):
|
||||||
out: list[RawScene] = []
|
out: list[RawScene] = []
|
||||||
seen: set[str] = set()
|
seen: set[str] = set()
|
||||||
for m in _A_RE.finditer(html):
|
for m in _A_RE.finditer(html):
|
||||||
scene_url = _canonical(m.group("url"))
|
scene_url = m.group("url")
|
||||||
if scene_url in seen:
|
if scene_url in seen:
|
||||||
continue
|
continue
|
||||||
seen.add(scene_url)
|
seen.add(scene_url)
|
||||||
|
|
@ -119,7 +106,7 @@ class SiskaScraper(BaseBrowseScraper):
|
||||||
dm = _DUR_RE.search(window)
|
dm = _DUR_RE.search(window)
|
||||||
duration_sec = _parse_duration(dm.group(1) if dm else None)
|
duration_sec = _parse_duration(dm.group(1) if dm else None)
|
||||||
tm = _THUMB_RE.search(window)
|
tm = _THUMB_RE.search(window)
|
||||||
thumbnail_url = _canonical(tm.group(1)) if tm else None
|
thumbnail_url = tm.group(1) if tm else None
|
||||||
|
|
||||||
# alt='Performer - Tytuł - Studio' → performer (pierwszy), studio (ostatni).
|
# alt='Performer - Tytuł - Studio' → performer (pierwszy), studio (ostatni).
|
||||||
performers: list[RawPerformer] = []
|
performers: list[RawPerformer] = []
|
||||||
|
|
|
||||||
|
|
@ -51,19 +51,14 @@ log = logging.getLogger(__name__)
|
||||||
|
|
||||||
_BASE = "https://yourdailypornmovies.ws"
|
_BASE = "https://yourdailypornmovies.ws"
|
||||||
|
|
||||||
# Karta listingu: okładka + URL + tytuł + rok. Okładka jest opcjonalna i w dowolnym
|
# Karta listingu: id produktu Adult Empire w miniaturce + URL + tytuł + rok.
|
||||||
# formacie: od ~2026-09-15 nowe filmy mają `imgs4cdn.adultempire.com/movie/<id>?w=...`
|
|
||||||
# zamiast `.../products/NN/<id>h.jpg`, część ma okładkę z i.ibb.co. Regex wymagający
|
|
||||||
# starego formatu po cichu gubił 33 z 40 filmów strony 1 (ingest: 0 nowych przez tydzień).
|
|
||||||
_CARD_RE = re.compile(
|
_CARD_RE = re.compile(
|
||||||
r'<div class="movie"><div class="imagen">\s*'
|
r'<div class="movie"><div class="imagen">\s*'
|
||||||
r'(?:<img src="(?P<poster>[^"]*)"[^>]*>\s*)?'
|
r'<img src="(?P<poster>https://imgs\d*cdn\.adultempire\.com/products/\d+/(?P<ae>\d+)h\.jpg)"'
|
||||||
r'<a href="(?P<url>https://yourdailypornmovies\.ws/[^"]+)">.*?'
|
r'[^>]*>\s*<a href="(?P<url>https://yourdailypornmovies\.ws/[^"]+)">.*?'
|
||||||
r'</div><h2>(?P<title>[^<]+)</h2>\s*<span class="year">(?P<year>\d{4})</span>',
|
r'</div><h2>(?P<title>[^<]+)</h2>\s*<span class="year">(?P<year>\d{4})</span>',
|
||||||
re.DOTALL,
|
re.DOTALL,
|
||||||
)
|
)
|
||||||
# Id produktu Adult Empire z adresu okładki, w obu formatach (to ta sama numeracja).
|
|
||||||
_AE_ID_RE = re.compile(r"adultempire\.com/(?:products/\d+/(\d+)h\.jpg|movie/(\d+))")
|
|
||||||
|
|
||||||
# Blok metadanych filmu. WSZYSTKO czytamy TYLKO stąd, patrz pułapka w docstringu.
|
# Blok metadanych filmu. WSZYSTKO czytamy TYLKO stąd, patrz pułapka w docstringu.
|
||||||
_XMLL_RE = re.compile(r'<div class="xmll">.*?</div>', re.DOTALL)
|
_XMLL_RE = re.compile(r'<div class="xmll">.*?</div>', re.DOTALL)
|
||||||
|
|
@ -213,21 +208,18 @@ class YourDailyPornMoviesConnector(BaseMovieConnector):
|
||||||
desc = html_mod.unescape(om.group(1)).strip() or None
|
desc = html_mod.unescape(om.group(1)).strip() or None
|
||||||
|
|
||||||
slug = urlparse(url).path.strip("/").split("/")[-1]
|
slug = urlparse(url).path.strip("/").split("/")[-1]
|
||||||
poster = html_mod.unescape(card.group("poster") or "").strip() or None
|
|
||||||
aem = _AE_ID_RE.search(poster or "")
|
|
||||||
ae_id = (aem.group(1) or aem.group(2)) if aem else None
|
|
||||||
return RawMovie(
|
return RawMovie(
|
||||||
external_id=slug,
|
external_id=slug,
|
||||||
title=title,
|
title=title,
|
||||||
description=desc,
|
description=desc,
|
||||||
release_year=year,
|
release_year=year,
|
||||||
url=url,
|
url=url,
|
||||||
poster_url=poster,
|
poster_url=card.group("poster"),
|
||||||
studio=studio,
|
studio=studio,
|
||||||
performers=performers,
|
performers=performers,
|
||||||
tags=tags,
|
tags=tags,
|
||||||
playback_sources=playback,
|
playback_sources=playback,
|
||||||
# id produktu Adult Empire, mocniejsza kotwica do dedupu niż tytuł
|
# id produktu Adult Empire, mocniejsza kotwica do dedupu niż tytuł
|
||||||
# (przy ocenie źródła złapało 9 filmów, których tytuł się nie zgadzał).
|
# (przy ocenie źródła złapało 9 filmów, których tytuł się nie zgadzał).
|
||||||
raw={"source": self.name, "url": url, "adultempire_id": ae_id},
|
raw={"source": self.name, "url": url, "adultempire_id": card.group("ae")},
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -29,43 +29,35 @@ def _resolve_by_altkey(session: Session, name: str) -> Tag | None:
|
||||||
scene_count wygrywa). Zapobiega regeneracji duplikatów: nowy wariant "Deep Throat"
|
scene_count wygrywa). Zapobiega regeneracji duplikatów: nowy wariant "Deep Throat"
|
||||||
trafia do istniejącego "deepthroat" zamiast tworzyć osobny tag. Używa indeksu
|
trafia do istniejącego "deepthroat" zamiast tworzyć osobny tag. Używa indeksu
|
||||||
funkcyjnego ix_tags_name_altkey."""
|
funkcyjnego ix_tags_name_altkey."""
|
||||||
return resolve_tags_by_altkey(session, [name]).get(name)
|
key = _altkey(name)
|
||||||
|
if not key:
|
||||||
|
return None
|
||||||
def resolve_tags_by_altkey(session: Session, names) -> dict[str, Tag]:
|
expr = func.regexp_replace(func.lower(func.btrim(Tag.name)), "[^a-z0-9]", "", "g")
|
||||||
"""`_resolve_by_altkey` dla wielu nazw jednym zapytaniem: {nazwa: tag} dla trafień.
|
hit = (
|
||||||
|
session.execute(select(Tag).where(expr == key).order_by(Tag.scene_count.desc()))
|
||||||
Po zapytaniu na nazwę enrich-tags robił N+1 przy kilku nowych nazwach ze strony
|
.scalars()
|
||||||
tuba (Sentry GOON-5F).
|
.first()
|
||||||
"""
|
)
|
||||||
|
if hit is not None:
|
||||||
|
return hit
|
||||||
# Liczba mnoga. Alnum-klucz z natury jej NIE łapie ("blowjob" != "blowjobs"), więc
|
# Liczba mnoga. Alnum-klucz z natury jej NIE łapie ("blowjob" != "blowjobs"), więc
|
||||||
# duplikaty odrastały mimo tamtej prewencji (audyt 2026-08-02: 1944 pary, m.in.
|
# duplikaty odrastały mimo tamtej prewencji (audyt 2026-08-02: 1944 pary, m.in.
|
||||||
# Blowjob/Blowjobs, Teen/Teens, Cumshot/Cumshots). Próbujemy obu kierunków: nowe
|
# Blowjob/Blowjobs, Teen/Teens, Cumshot/Cumshots). Próbujemy obu kierunków: nowe
|
||||||
# "Blowjobs" trafia w istniejące "Blowjob", a nowe "Blowjob" w istniejące "Blowjobs".
|
# "Blowjobs" trafia w istniejące "Blowjob", a nowe "Blowjob" w istniejące "Blowjobs".
|
||||||
# Tylko proste `+s`, form nieregularnych (panty/panties) świadomie nie ruszamy, bo
|
# Tylko proste `+s`, form nieregularnych (panty/panties) świadomie nie ruszamy, bo
|
||||||
# zbyt łatwo o fałszywy zlew (glass/glasses, bra/bras). Dokładny klucz ma pierwszeństwo.
|
# zbyt łatwo o fałszywy zlew (glass/glasses, bra/bras).
|
||||||
keys: dict[str, tuple[str, str | None]] = {}
|
alts = [key + "s"] if not key.endswith("s") else [key[:-1]]
|
||||||
for name in names:
|
for alt in alts:
|
||||||
key = _altkey(name)
|
if len(alt) < 3:
|
||||||
if not key:
|
|
||||||
continue
|
continue
|
||||||
alt = key + "s" if not key.endswith("s") else key[:-1]
|
hit = (
|
||||||
keys[name] = (key, alt if len(alt) >= 3 else None)
|
session.execute(select(Tag).where(expr == alt).order_by(Tag.scene_count.desc()))
|
||||||
wanted = {k for k, _ in keys.values()} | {a for _, a in keys.values() if a}
|
.scalars()
|
||||||
if not wanted:
|
.first()
|
||||||
return {}
|
)
|
||||||
expr = func.regexp_replace(func.lower(func.btrim(Tag.name)), "[^a-z0-9]", "", "g")
|
if hit is not None:
|
||||||
best: dict[str, Tag] = {}
|
return hit
|
||||||
for key, tag in session.execute(
|
return None
|
||||||
select(expr, Tag).where(expr.in_(wanted)).order_by(Tag.scene_count.desc())
|
|
||||||
).all():
|
|
||||||
best.setdefault(key, tag)
|
|
||||||
out: dict[str, Tag] = {}
|
|
||||||
for name, (key, alt) in keys.items():
|
|
||||||
tag = best.get(key) or (best.get(alt) if alt else None)
|
|
||||||
if tag is not None:
|
|
||||||
out[name] = tag
|
|
||||||
return out
|
|
||||||
|
|
||||||
|
|
||||||
def _canonical_dup2_slug(session: Session, slug: str) -> str:
|
def _canonical_dup2_slug(session: Session, slug: str) -> str:
|
||||||
|
|
|
||||||
|
|
@ -1,38 +0,0 @@
|
||||||
from app.connectors.yourdailypornmovies import _AE_ID_RE, _CARD_RE
|
|
||||||
|
|
||||||
_OLD = (
|
|
||||||
'<div class="movie"><div class="imagen"> <img src="https://imgs1cdn.adultempire.com/products/05/5128705h.jpg" '
|
|
||||||
'alt="Ramming Into Her" width="100%" height="100%" /> <a href="https://yourdailypornmovies.ws/ramming-into-her/">'
|
|
||||||
'<span class="player"></span></a></div><h2>Ramming Into Her</h2> <span class="year">2026</span></div>'
|
|
||||||
)
|
|
||||||
_NEW = (
|
|
||||||
'<div class="movie"><div class="imagen"> <img src="https://imgs4cdn.adultempire.com/movie/5136742?w=500&h=700&fit=cover" '
|
|
||||||
'alt="Big Butts Drive Me Nuts 8" width="100%" height="100%" /> <a href="https://yourdailypornmovies.ws/big-butts-drive-me-nuts-8/">'
|
|
||||||
'<span class="player"></span></a></div><h2>Big Butts Drive Me Nuts 8</h2> <span class="year">2026</span></div>'
|
|
||||||
)
|
|
||||||
_IBB = (
|
|
||||||
'<div class="movie"><div class="imagen"> <img src="https://i.ibb.co/DHDd9pm9/Private-POV.jpg" alt="Private POV" '
|
|
||||||
'width="100%" height="100%" /> <a href="https://yourdailypornmovies.ws/private-pov/"><span class="player"></span></a>'
|
|
||||||
'</div><h2>Private POV</h2> <span class="year">2026</span></div>'
|
|
||||||
)
|
|
||||||
_NO_IMG = (
|
|
||||||
'<div class="movie"><div class="imagen"> <a href="https://yourdailypornmovies.ws/no-cover/"><span class="player">'
|
|
||||||
'</span></a></div><h2>No Cover</h2> <span class="year">2025</span></div>'
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def test_all_card_variants_parse():
|
|
||||||
cards = list(_CARD_RE.finditer(_OLD + _NEW + _IBB + _NO_IMG))
|
|
||||||
assert [c.group("title") for c in cards] == [
|
|
||||||
"Ramming Into Her", "Big Butts Drive Me Nuts 8", "Private POV", "No Cover"
|
|
||||||
]
|
|
||||||
assert cards[3].group("poster") is None
|
|
||||||
assert cards[3].group("url") == "https://yourdailypornmovies.ws/no-cover/"
|
|
||||||
|
|
||||||
|
|
||||||
def test_adultempire_id_from_both_poster_formats():
|
|
||||||
old = _AE_ID_RE.search("https://imgs1cdn.adultempire.com/products/05/5128705h.jpg")
|
|
||||||
new = _AE_ID_RE.search("https://imgs4cdn.adultempire.com/movie/5136742?w=500&h=700&fit=cover")
|
|
||||||
assert (old.group(1) or old.group(2)) == "5128705"
|
|
||||||
assert (new.group(1) or new.group(2)) == "5136742"
|
|
||||||
assert _AE_ID_RE.search("https://i.ibb.co/DHDd9pm9/Private-POV.jpg") is None
|
|
||||||
Loading…
Add table
Reference in a new issue