Compare commits
No commits in common. "426b5e99a23d04dab52b60d0fd019abc87d6010f" and "617dda0bbe3cfd4f7d0893169b3ab6fc49689587" have entirely different histories.
426b5e99a2
...
617dda0bbe
24 changed files with 45 additions and 959 deletions
|
|
@ -1,55 +0,0 @@
|
|||
"""Liczniki klipu z feedu Edge: wyswietlenia, ucieczki (< 2 s) i lajki w samym Goonie.
|
||||
|
||||
Odtworzenie uczenia feedu na 189 tys. zdarzen (2026-10-02) pokazalo, ze najmocniejszym
|
||||
sygnalem tego, czy klip sie spodoba, jest to, jak przyjeli go inni u nas: odsetek lajkow
|
||||
(AUC w obrebie urzadzenia 0,600) bije wagi tagow (0,571), a popularnosc z RedGifs prawie
|
||||
nic nie mowi (0,532). Punktacja (app/clips_feed.py) potrzebuje tych liczb w chwili
|
||||
ukladania strony, stad kolumny przy klipie zamiast agregatu po clip_feed_events.
|
||||
|
||||
ADD COLUMN ze stalym DEFAULT nie przepisuje tabeli (Postgres 11+). Wypelnienie z
|
||||
dotychczasowych zdarzen jednym UPDATE (~70 tys. klipow).
|
||||
|
||||
Revision ID: 0052_clips_feed_counters
|
||||
Revises: 0051_movies_tpdb_checked_at
|
||||
Create Date: 2026-10-02
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision = "0052_clips_feed_counters"
|
||||
down_revision: str | None = "0051_movies_tpdb_checked_at"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.execute("SET LOCAL lock_timeout = '5s'")
|
||||
op.execute("""
|
||||
ALTER TABLE clips
|
||||
ADD COLUMN IF NOT EXISTS feed_views integer NOT NULL DEFAULT 0,
|
||||
ADD COLUMN IF NOT EXISTS feed_skips integer NOT NULL DEFAULT 0,
|
||||
ADD COLUMN IF NOT EXISTS feed_likes integer NOT NULL DEFAULT 0
|
||||
""")
|
||||
op.execute("""
|
||||
UPDATE clips c SET feed_views = a.v, feed_skips = a.s, feed_likes = greatest(a.l, 0)
|
||||
FROM (
|
||||
SELECT clip_id,
|
||||
count(*) FILTER (WHERE kind IN ('complete', 'skip', 'impression', 'autoswipe')) AS v,
|
||||
count(*) FILTER (WHERE kind = 'skip') AS s,
|
||||
count(*) FILTER (WHERE kind = 'like') - count(*) FILTER (WHERE kind = 'unlike') AS l
|
||||
FROM clip_feed_events WHERE clip_id IS NOT NULL
|
||||
GROUP BY clip_id
|
||||
) a
|
||||
WHERE a.clip_id = c.id
|
||||
""")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.execute("SET LOCAL lock_timeout = '5s'")
|
||||
op.execute("""
|
||||
ALTER TABLE clips
|
||||
DROP COLUMN IF EXISTS feed_views,
|
||||
DROP COLUMN IF EXISTS feed_skips,
|
||||
DROP COLUMN IF EXISTS feed_likes
|
||||
""")
|
||||
|
|
@ -17,7 +17,6 @@ bez image proxy: RedGifs nie sprawdza Referera, a proxy tylko dodałoby VPS do t
|
|||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import math
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
|
|
@ -45,8 +44,6 @@ _AFFINITY_TAGS, _AFFINITY_NICHES, _AFFINITY_CREATORS = 6, 3, 3
|
|||
_PER_TAG, _PER_NICHE, _PER_CREATOR = 60, 60, 40
|
||||
_UUID_ZERO = uuid.UUID(int=0)
|
||||
_BLACKLIST_KINDS = frozenset({"tag", "niche", "creator"})
|
||||
#: Zdarzenia oznaczajace, ze klip zostal obejrzany (zejscie ze strony), do clips.feed_views.
|
||||
_VIEW_KINDS = frozenset({"complete", "skip", "impression", "autoswipe"})
|
||||
|
||||
|
||||
# ---- schematy ---------------------------------------------------------------------------
|
||||
|
|
@ -254,10 +251,7 @@ def _visible_clauses(device_id: str, bl: dict[str, set[str]]) -> list:
|
|||
return clauses
|
||||
|
||||
|
||||
_CAND_COLS = (
|
||||
Clip.id, Clip.creator, Clip.likes, Clip.published_at, Clip.niches,
|
||||
Clip.feed_views, Clip.feed_skips, Clip.feed_likes,
|
||||
)
|
||||
_CAND_COLS = (Clip.id, Clip.creator, Clip.likes, Clip.published_at, Clip.niches)
|
||||
|
||||
|
||||
def _rows_to_candidates(session: Session, rows, *, with_tags: bool) -> list[cf.Candidate]:
|
||||
|
|
@ -265,10 +259,7 @@ def _rows_to_candidates(session: Session, rows, *, with_tags: bool) -> list[cf.C
|
|||
wyłącznie lajkami, więc ciągnięcie tagów dla 300 kandydatów było czystym kosztem."""
|
||||
tags = _tags_for(session, [r[0] for r in rows]) if with_tags else {}
|
||||
return [
|
||||
cf.Candidate(
|
||||
id=r[0], creator=r[1], likes=r[2], published_at=r[3], tags=tags.get(r[0], []), niches=list(r[4] or []),
|
||||
feed_views=r[5] or 0, feed_skips=r[6] or 0, feed_likes=r[7] or 0,
|
||||
)
|
||||
cf.Candidate(id=r[0], creator=r[1], likes=r[2], published_at=r[3], tags=tags.get(r[0], []), niches=list(r[4] or []))
|
||||
for r in rows
|
||||
]
|
||||
|
||||
|
|
@ -303,29 +294,6 @@ def _explore_candidates(
|
|||
return _rows_to_candidates(session, rows, with_tags=False), (end_id if not last else _UUID_ZERO), last
|
||||
|
||||
|
||||
# Rzadkosc tagow (IDF) do punktacji: ln(liczba klipow / klipy z tagiem). Liczenie to jedno
|
||||
# GROUP BY po ~1,7 mln clip_tags (~1 s), slownik zmienia sie wolno, wiec cache 6 h w procesie.
|
||||
_IDF_TTL_SEC = 6 * 3600
|
||||
_idf_cache: tuple[float, dict[str, float]] | None = None
|
||||
_idf_lock = threading.Lock()
|
||||
|
||||
|
||||
def _tag_idf(session: Session) -> dict[str, float]:
|
||||
global _idf_cache
|
||||
hit = _idf_cache
|
||||
if hit is not None and time.monotonic() - hit[0] < _IDF_TTL_SEC:
|
||||
return hit[1]
|
||||
with _idf_lock:
|
||||
hit = _idf_cache
|
||||
if hit is not None and time.monotonic() - hit[0] < _IDF_TTL_SEC:
|
||||
return hit[1]
|
||||
n_all = session.execute(select(func.count()).select_from(Clip)).scalar_one()
|
||||
rows = session.execute(text("SELECT tag, count(*) FROM clip_tags GROUP BY tag")).all()
|
||||
idf = {t: math.log((n_all + 1) / (n + 1)) for t, n in rows}
|
||||
_idf_cache = (time.monotonic(), idf)
|
||||
return idf
|
||||
|
||||
|
||||
def _affinity_candidate_ids(session: Session, aff: cf.Affinity, rng) -> set[uuid.UUID]:
|
||||
ids: set[uuid.UUID] = set()
|
||||
for tag in aff.top("tags", _AFFINITY_TAGS):
|
||||
|
|
@ -384,13 +352,7 @@ def clip_feed(
|
|||
picked_aff: list[cf.Candidate] = []
|
||||
if n_aff:
|
||||
cand_ids = _affinity_candidate_ids(session, aff, rng)
|
||||
picked_aff = cf.pick_affinity(
|
||||
_candidates(session, list(cand_ids), clauses),
|
||||
aff,
|
||||
n_aff,
|
||||
idf=_tag_idf(session),
|
||||
v1=cf.uses_v1_scoring(device_id),
|
||||
)
|
||||
picked_aff = cf.pick_affinity(_candidates(session, list(cand_ids), clauses), aff, n_aff)
|
||||
taken = {c.id for c in picked_aff}
|
||||
|
||||
# Explore: okno po PK; gdy w oknie brak widocznych, przesuwamy sie dalej (max 3 okna).
|
||||
|
|
@ -470,27 +432,6 @@ def clip_events(
|
|||
}
|
||||
row, aff = _load_affinity(session, device_id)
|
||||
changed = 0
|
||||
# Liczniki przyjecia klipu u nas (clips.feed_*), z nich korzysta punktacja feedu.
|
||||
counters: dict[uuid.UUID, list[int]] = {}
|
||||
for e in events:
|
||||
d = counters.setdefault(e.clip_id, [0, 0, 0])
|
||||
if e.kind in _VIEW_KINDS:
|
||||
d[0] += 1
|
||||
if e.kind == "skip":
|
||||
d[1] += 1
|
||||
if e.kind == "like":
|
||||
d[2] += 1
|
||||
elif e.kind == "unlike":
|
||||
d[2] -= 1
|
||||
for cid, (dv, ds, dl) in counters.items():
|
||||
if dv or ds or dl:
|
||||
session.execute(
|
||||
text(
|
||||
"UPDATE clips SET feed_views = feed_views + :v, feed_skips = feed_skips + :s, "
|
||||
"feed_likes = greatest(feed_likes + :l, 0) WHERE id = :id"
|
||||
),
|
||||
{"v": dv, "s": ds, "l": dl, "id": cid},
|
||||
)
|
||||
for e in events:
|
||||
session.add(
|
||||
ClipFeedEvent(
|
||||
|
|
|
|||
|
|
@ -17,7 +17,6 @@ from app.api.device import get_device_id
|
|||
from app.auth import require_api_key
|
||||
from app.db import get_session
|
||||
from app.models.playback_event import PlaybackEvent
|
||||
from app.playback_gone import is_gone_report, verify_gone_async
|
||||
|
||||
router = APIRouter(prefix="/playback-events", tags=["telemetry"], dependencies=[Depends(require_api_key)])
|
||||
|
||||
|
|
@ -52,7 +51,3 @@ def post_playback_event(
|
|||
)
|
||||
)
|
||||
session.commit()
|
||||
# Plik zniknal (404/410 w odtwarzaczu): serwer sprawdza zrodlo sam i oznacza martwe
|
||||
# dopiero po wlasnym potwierdzeniu (app/playback_gone.py).
|
||||
if is_gone_report(body.status, body.error_kind, body.error_detail):
|
||||
verify_gone_async(body.scene_id, body.origin)
|
||||
|
|
|
|||
|
|
@ -585,40 +585,6 @@ async def sxyprn_thumb(post_id: str) -> Response:
|
|||
)
|
||||
|
||||
|
||||
#: Po tych statusach proxy pobiera strone tube na nowo (gdy token ma rf + rh "tube:...").
|
||||
#: 503 dopiero po nieudanym ponowieniu: nowy token bywa kierowany na inny wezel CDN.
|
||||
_TUBE_REFRESH_STATUSES = frozenset({403, 404, 410, 503})
|
||||
_QUALITY_IN_URL_RE = re.compile(r"_(\d{3,4})[a-z]?\.mp4", re.IGNORECASE)
|
||||
|
||||
|
||||
async def _fresh_tube_link(sitetag: str, page_url: str, old_target: str) -> str | None:
|
||||
"""Swiezy adres pliku z ekstraktora tube, w tej samej jakosci co `old_target`.
|
||||
|
||||
hdporn.gg / fullmovies.xxx: get_file jest przypiety do IP ladujacego strone i bywa
|
||||
jednorazowy. Telefon, ktoremu nie wyszedl wlasny resolver, dostawal od proxy ten sam
|
||||
zuzyty adres i konczyl na 410/503 (14 urzadzen w tydzien z samymi porazkami, 2026-10-02).
|
||||
"""
|
||||
import anyio
|
||||
|
||||
from app.extractors import try_extract
|
||||
|
||||
try:
|
||||
sources = await anyio.to_thread.run_sync(lambda: try_extract(sitetag, page_url))
|
||||
except Exception as e: # noqa: BLE001 (HosterDead/TubePageError: zostaje stary blad)
|
||||
log.info("proxy: odswiezenie %s nie wyszlo: %s", page_url, e)
|
||||
return None
|
||||
links = [s.link for s in (sources or []) if s.link]
|
||||
if not links:
|
||||
return None
|
||||
want = _QUALITY_IN_URL_RE.search(old_target)
|
||||
if want:
|
||||
for link in links:
|
||||
m = _QUALITY_IN_URL_RE.search(link)
|
||||
if m and m.group(1) == want.group(1):
|
||||
return link
|
||||
return links[0]
|
||||
|
||||
|
||||
async def _refetch_mixdrop_url(session: "AsyncSession", embed_url: str) -> str | None:
|
||||
"""Re-fetch mixdrop embed, decode P.A.C.K.E.R., extract fresh MDCore.wurl.
|
||||
Cookies persist w session, użytkowane potem do mp4 GET (same-session bind).
|
||||
|
|
@ -904,36 +870,6 @@ async def proxy_stream(
|
|||
stream=True,
|
||||
follow_redirects=True,
|
||||
)
|
||||
# okcdn (OK.ru, porndupe) odrzuca obcy naglowek Origin kodem 400, a _build_headers
|
||||
# dokleja go zawsze, gdy znamy Referer. Przegladarka przy zwyklym <video> Origin
|
||||
# nie wysyla. Bez tego kazdy zapasowy proxy porndupe konczyl sie 502 (2026-10-02:
|
||||
# ten sam adres z Refererem 206, z Refererem i Originem 400).
|
||||
if upstream.status_code == 400 and "Origin" in headers and method.upper() in ("GET", "HEAD"):
|
||||
await upstream.aclose()
|
||||
headers.pop("Origin", None)
|
||||
log.info("proxy: upstream 400 z Origin, ponawiam bez niego %s", target[:70])
|
||||
upstream = await client.send(
|
||||
client.build_request(method, target, headers=headers),
|
||||
stream=True,
|
||||
follow_redirects=True,
|
||||
)
|
||||
if (
|
||||
upstream.status_code in _TUBE_REFRESH_STATUSES
|
||||
and refetch_url
|
||||
and (refetch_hoster or "").startswith("tube:")
|
||||
and method.upper() in ("GET", "HEAD")
|
||||
):
|
||||
fresh = await _fresh_tube_link(refetch_hoster[5:], refetch_url, target)
|
||||
if fresh:
|
||||
log.info("proxy: upstream %d, swiezy adres z %s", upstream.status_code, refetch_url[:70])
|
||||
await upstream.aclose()
|
||||
target = fresh
|
||||
cached_target = None # nowy cel: po sukcesie cache dostanie jego koncowy adres
|
||||
upstream = await client.send(
|
||||
client.build_request(method, target, headers=headers),
|
||||
stream=True,
|
||||
follow_redirects=True,
|
||||
)
|
||||
if upstream.status_code >= 400:
|
||||
status = upstream.status_code
|
||||
ups_headers = dict(upstream.headers)
|
||||
|
|
|
|||
|
|
@ -14,10 +14,8 @@ zapomina stare gusta, cap trzyma słowniki małe (60/40/40). Wszystko celowo pro
|
|||
przejrzyste: da się wytłumaczyć użytkownikowi, dlaczego coś widzi, i skasować jednym
|
||||
wywołaniem.
|
||||
|
||||
**Punktacja kandydata** (od 2026-10-02, uzasadnienie przy `score_candidate`) = log(Σ waga
|
||||
tagu × rzadkość tagu) + 0,25·log(nisze) + 2,5·log(twórca) + 0,8·ln(1+lajki RedGifs)
|
||||
+ 120·(odsetek lajków u nas − 3 %) − 15·(odsetek ucieczek u nas − 50 %) + 0,3 za świeżość.
|
||||
25 % urządzeń zostaje na starej punktacji jako grupa kontrolna. Strona: 80 % z afiniczności (max 2 klipy jednego twórcy),
|
||||
**Punktacja kandydata** = Σ wag tagów + 2·waga niszy + 3·waga twórcy + 0,3·ln(1+lajki)
|
||||
+ 0,5 gdy klip z ostatnich 7 dni. Strona: 80 % z afiniczności (max 2 klipy jednego twórcy),
|
||||
20 % explore, przemieszane deterministycznie z seeda.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
|
@ -188,60 +186,9 @@ class Candidate:
|
|||
published_at: datetime | None
|
||||
tags: list[str]
|
||||
niches: list[str]
|
||||
#: Przyjecie klipu u nas (kolumny clips.feed_*, migracja 0052).
|
||||
feed_views: int = 0
|
||||
feed_skips: int = 0
|
||||
feed_likes: int = 0
|
||||
|
||||
|
||||
# Punktacja od 2026-10-02, dobrana na odtworzeniu 189 tys. zdarzen (wagi urzadzenia
|
||||
# liczone zdarzenie po zdarzeniu, bez zagladania w przyszlosc; trening na 70 % najstarszych,
|
||||
# test na reszcie). Miara: czy wyzszy wynik u TEGO urzadzenia znaczy lajk albo >= 10 s
|
||||
# ogladania, a nie ucieczke < 2 s (AUC w obrebie urzadzenia). Stara punktacja: 0,576,
|
||||
# ta: 0,652, regresja logistyczna na tych samych cechach: 0,654. Wnioski:
|
||||
# * suma wag tagow niewiele mowi, bo tagi RedGifs sa liczne i ogolne, a lajk dokladal
|
||||
# +5 kazdemu z ~10 tagow klipu; waga razy rzadkosc tagu (IDF) i log daje wiecej,
|
||||
# * nisze prawie nic (0,54), stad 0,25 zamiast 2,
|
||||
# * tworca to najlepszy sygnal z wag (0,58),
|
||||
# * najmocniejszy jest odsetek lajkow klipu u nas (0,60), wygladzony, zeby klip bez
|
||||
# historii nie dostawal skrajnych ocen; popularnosc z RedGifs slaba, ale nie zerowa.
|
||||
W_NICHE = 0.25
|
||||
W_CREATOR = 2.5
|
||||
W_POP = 0.8
|
||||
W_FEED_LIKE = 120.0
|
||||
W_FEED_SKIP = 15.0
|
||||
W_FRESH = 0.3
|
||||
#: Wygladzanie odsetkow: klip zaczyna od 10 „wirtualnych" wyswietlen ze srednimi wartosciami.
|
||||
PRIOR_VIEWS = 10.0
|
||||
PRIOR_LIKE_RATE = 0.03
|
||||
PRIOR_SKIP_RATE = 0.5
|
||||
|
||||
|
||||
def _slog(x: float) -> float:
|
||||
return math.copysign(math.log1p(abs(x)), x)
|
||||
|
||||
|
||||
def score_candidate(
|
||||
c: Candidate, aff: Affinity, now: datetime | None = None, idf: dict[str, float] | None = None
|
||||
) -> float:
|
||||
"""`idf`: tag -> ln(N/df) po calym katalogu klipow; bez niego kazdy tag waży 1."""
|
||||
now = now or datetime.now(UTC)
|
||||
default_idf = max(idf.values()) if idf else 1.0
|
||||
tag = sum(aff.tags.get(t, 0.0) * (idf.get(t, default_idf) if idf else 1.0) for t in c.tags)
|
||||
s = _slog(tag)
|
||||
s += W_NICHE * _slog(sum(aff.niches.get(n, 0.0) for n in c.niches))
|
||||
s += W_CREATOR * _slog(aff.creators.get(c.creator or "", 0.0))
|
||||
s += W_POP * math.log1p(max(c.likes, 0))
|
||||
like_rate = (c.feed_likes + PRIOR_VIEWS * PRIOR_LIKE_RATE) / (c.feed_views + PRIOR_VIEWS)
|
||||
skip_rate = (c.feed_skips + PRIOR_VIEWS * PRIOR_SKIP_RATE) / (c.feed_views + PRIOR_VIEWS)
|
||||
s += W_FEED_LIKE * (like_rate - PRIOR_LIKE_RATE) - W_FEED_SKIP * (skip_rate - PRIOR_SKIP_RATE)
|
||||
if c.published_at and c.published_at >= now - timedelta(days=FRESH_DAYS):
|
||||
s += W_FRESH
|
||||
return s
|
||||
|
||||
|
||||
def score_candidate_v1(c: Candidate, aff: Affinity, now: datetime | None = None) -> float:
|
||||
"""Punktacja sprzed 2026-10-02, zostaje dla grupy kontrolnej (patrz `uses_v1_scoring`)."""
|
||||
def score_candidate(c: Candidate, aff: Affinity, now: datetime | None = None) -> float:
|
||||
now = now or datetime.now(UTC)
|
||||
s = sum(aff.tags.get(t, 0.0) for t in c.tags)
|
||||
s += 2.0 * sum(aff.niches.get(n, 0.0) for n in c.niches)
|
||||
|
|
@ -252,30 +199,9 @@ def score_candidate_v1(c: Candidate, aff: Affinity, now: datetime | None = None)
|
|||
return s
|
||||
|
||||
|
||||
#: Odsetek urzadzen na starej punktacji (grupa kontrolna). Przydzial staly po hashu id
|
||||
#: urzadzenia, wiec do porownania wystarczy ten sam hash przy analizie zdarzen.
|
||||
V1_CONTROL_PERCENT = 25
|
||||
|
||||
|
||||
def uses_v1_scoring(device_id: str) -> bool:
|
||||
h = int(hashlib.sha256(device_id.encode("utf-8")).hexdigest()[:8], 16)
|
||||
return h % 100 < V1_CONTROL_PERCENT
|
||||
|
||||
|
||||
def pick_affinity(
|
||||
cands: list[Candidate],
|
||||
aff: Affinity,
|
||||
n: int,
|
||||
*,
|
||||
max_per_creator: int = MAX_PER_CREATOR,
|
||||
idf: dict[str, float] | None = None,
|
||||
v1: bool = False,
|
||||
) -> list[Candidate]:
|
||||
def pick_affinity(cands: list[Candidate], aff: Affinity, n: int, *, max_per_creator: int = MAX_PER_CREATOR) -> list[Candidate]:
|
||||
"""Najlepsze `n` po punktach, max `max_per_creator` klipów jednego twórcy na stronę."""
|
||||
if v1:
|
||||
ranked = sorted(cands, key=lambda c: -score_candidate_v1(c, aff))
|
||||
else:
|
||||
ranked = sorted(cands, key=lambda c: -score_candidate(c, aff, idf=idf))
|
||||
ranked = sorted(cands, key=lambda c: -score_candidate(c, aff))
|
||||
out: list[Candidate] = []
|
||||
per: dict[str, int] = {}
|
||||
for c in ranked:
|
||||
|
|
|
|||
|
|
@ -56,13 +56,10 @@ USER_AGENT = (
|
|||
# 2. streamporn/pandamovies: <a class="ml-mask jt" href="<base>/<slug>/">
|
||||
# (slug bez /movies/ prefix, np. /watch-xxx-...-adult-movie-online-free/)
|
||||
# Łapiemy oba przez alternatywę.
|
||||
# 2026-10-02 mangoporn przeszedl na `<a class="thumb" href=...>` (klasa PRZED href) i
|
||||
# wzorzec z href na poczatku nie lapal juz nic: 0 filmow od 2026-09-23. Teraz href moze
|
||||
# stac w dowolnym miejscu znacznika; linki menu do /movies/category/ odpadaja.
|
||||
_LIST_ITEM_RE = re.compile(
|
||||
r'<a\s+href="(?P<url>https?://[^"]+)"[^>]*\bclass="ml-mask\b[^"]*"'
|
||||
r"|"
|
||||
r'<a\s+(?:[^>]*?\s)?href="(?P<url2>https?://[^"]+/movies/(?!category/)[a-z0-9-]+/)"',
|
||||
r'<a\s+href="(?P<url2>https?://[^"]+/movies/[a-z0-9-]+/)"',
|
||||
re.IGNORECASE,
|
||||
)
|
||||
# Tolerantny title, mangoporn (dooplay) używa <h1> w class="data", streamporn/pandamovies
|
||||
|
|
@ -72,9 +69,6 @@ _TITLE_RE = re.compile(
|
|||
r'|class="data"[^>]*>\s*<h[1-6][^>]*>([^<]+)</h[1-6]>',
|
||||
re.IGNORECASE | re.DOTALL,
|
||||
)
|
||||
# Zapas: mangoporn od 2026-10 ma tytul w golym <h1>, bez itemprop i bez class="data".
|
||||
# Tylko gdy dwa wzorce wyzej nic nie znajda, zeby inne motywy nie braly logo z <h1>.
|
||||
_H1_TITLE_RE = re.compile(r"<h1[^>]*>\s*([^<]{2,300}?)\s*</h1>", re.IGNORECASE)
|
||||
# dooplay uses /year/, raw PsyPlay uses /release-year/. Same dla pozostałych slugów,
|
||||
# różne thema dziedziczą podstawowy markup ale customizują URL słowniki.
|
||||
_YEAR_RE = re.compile(
|
||||
|
|
@ -278,11 +272,11 @@ class DooplayConnector(BaseMovieConnector):
|
|||
def _parse_dooplay_detail(
|
||||
*, slug: str, html: str, source_name: str, base_url: str, page_url: str | None = None
|
||||
) -> RawMovie | None:
|
||||
m_title = _TITLE_RE.search(html) or _H1_TITLE_RE.search(html)
|
||||
m_title = _TITLE_RE.search(html)
|
||||
if not m_title:
|
||||
log.warning("%s: no title in %s", source_name, slug)
|
||||
return None
|
||||
title = _decode_html(next(g for g in m_title.groups() if g).strip())
|
||||
title = _decode_html((m_title.group(1) or m_title.group(2)).strip())
|
||||
|
||||
m_year = _YEAR_RE.search(html)
|
||||
release_year = int(m_year.group(1)) if m_year else None
|
||||
|
|
@ -458,10 +452,7 @@ class StreampornVipConnector(DooplayConnector):
|
|||
# samym PsyPlay theme i świeższym katalogiem (ocena 2026-07-02). Osobny source_name
|
||||
# → nowe origins `streampornvip:<host>`, nie miesza z martwym .nl.
|
||||
name = "streampornvip"
|
||||
# 2026-10-02 streamporn.vip -> 302 streamporn.li. Konektor odrzuca linki spoza swojej
|
||||
# domeny, wiec po przekierowaniu lista byla pusta (0 filmow od 2026-09-23). Id filmu to
|
||||
# slug, bez domeny, wiec przeprowadzka nie robi duplikatow.
|
||||
base_url = "https://streamporn.li"
|
||||
base_url = "https://streamporn.vip"
|
||||
|
||||
def _listing_path(self, page: int) -> str:
|
||||
"""Strona główna, NIE `/movies/`. Diagnoza 2026-08-03: źródło stało od
|
||||
|
|
@ -489,15 +480,6 @@ class MangopornConnector(DooplayConnector):
|
|||
name = "mangoporn"
|
||||
base_url = "https://mangoporn.net"
|
||||
|
||||
def __init__(self, *, timeout: float = 75.0):
|
||||
# Strona odpowiada z VPS po 30-60 s (2026-10-02), przy 30 s ingest konczyl sie
|
||||
# na timeoutcie listy.
|
||||
super().__init__(timeout=timeout)
|
||||
|
||||
def _listing_path(self, page: int) -> str:
|
||||
"""Od 2026-10 `/movies/` daje 404, najnowsze sa na stronie glownej (`/page/N/`)."""
|
||||
return "/" if page == 1 else f"/page/{page}/"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers (zduplikowane z paradisehill.py, celowo, żeby connectory były niezależne)
|
||||
|
|
|
|||
|
|
@ -55,13 +55,11 @@ _BASE = "https://yourdailypornmovies.ws"
|
|||
# formacie: od ~2026-09-15 nowe filmy mają `imgs4cdn.adultempire.com/movie/<id>?w=...`
|
||||
# zamiast `.../products/NN/<id>h.jpg`, część ma okładkę z i.ibb.co. Regex wymagający
|
||||
# starego formatu po cichu gubił 33 z 40 filmów strony 1 (ingest: 0 nowych przez tydzień).
|
||||
# 2026-10 strona wstawila nowe linie miedzy znacznikami karty i wzorzec bez \s* nie
|
||||
# lapal juz nic (0 filmow od 2026-09-29, przy dzialajacej stronie).
|
||||
_CARD_RE = re.compile(
|
||||
r'<div class="movie">\s*<div class="imagen">\s*'
|
||||
r'<div class="movie"><div class="imagen">\s*'
|
||||
r'(?:<img src="(?P<poster>[^"]*)"[^>]*>\s*)?'
|
||||
r'<a href="(?P<url>https://yourdailypornmovies\.ws/[^"]+)">.*?'
|
||||
r'</div>\s*<h2>(?P<title>[^<]+)</h2>\s*<span class="year">(?P<year>\d{4})</span>',
|
||||
r'</div><h2>(?P<title>[^<]+)</h2>\s*<span class="year">(?P<year>\d{4})</span>',
|
||||
re.DOTALL,
|
||||
)
|
||||
# Id produktu Adult Empire z adresu okładki, w obu formatach (to ta sama numeracja).
|
||||
|
|
|
|||
|
|
@ -31,9 +31,7 @@ _SOURCE_RE = re.compile(
|
|||
_SKIP_QUALITY_RE = re.compile(r"2160|1440|4k", re.IGNORECASE)
|
||||
|
||||
|
||||
def resolve(
|
||||
page_url: str, base_url: str, *, timeout: float = 30.0, sitetag: str | None = None
|
||||
) -> list[StreamSource] | None:
|
||||
def resolve(page_url: str, base_url: str, *, timeout: float = 30.0) -> list[StreamSource] | None:
|
||||
if not _HAS_CURL_CFFI:
|
||||
log.info("source_getfile: curl_cffi unavailable — %s", page_url)
|
||||
return None
|
||||
|
|
@ -61,16 +59,9 @@ def resolve(
|
|||
if _SKIP_QUALITY_RE.search(quality):
|
||||
log.info("source_getfile: skip broken-CDN quality %r on %s", quality, page_url)
|
||||
continue
|
||||
raw: dict = {"mobile_direct_ok": True}
|
||||
if sitetag:
|
||||
# get_file jest przypiety do IP tego, kto zaladowal strone, i bywa jednorazowy,
|
||||
# wiec proxy (zapas, gdy resolver na telefonie zawiedzie) po 403/410 pobiera
|
||||
# strone na nowo zamiast oddac blad (stream_proxy._fresh_tube_link, 2026-10-02).
|
||||
raw["refetch_url"] = page_url
|
||||
raw["refetch_hoster"] = f"tube:{sitetag}"
|
||||
out.append(StreamSource(
|
||||
link=url, type="mp4", quality=quality or None,
|
||||
referer=base_url + "/", raw=raw,
|
||||
referer=base_url + "/", raw={"mobile_direct_ok": True},
|
||||
))
|
||||
|
||||
if not out:
|
||||
|
|
|
|||
|
|
@ -16,4 +16,4 @@ _BASE = "https://www.fullmovies.xxx"
|
|||
|
||||
|
||||
def extract(page_url: str, *, timeout: float = 30.0) -> list[StreamSource] | None:
|
||||
return _source_getfile.resolve(page_url, _BASE, timeout=timeout, sitetag="fullmoviesxxx")
|
||||
return _source_getfile.resolve(page_url, _BASE, timeout=timeout)
|
||||
|
|
|
|||
|
|
@ -14,4 +14,4 @@ _BASE = "https://www.hdporn.gg"
|
|||
|
||||
|
||||
def extract(page_url: str, *, timeout: float = 30.0) -> list[StreamSource] | None:
|
||||
return _source_getfile.resolve(page_url, _BASE, timeout=timeout, sitetag="hdporngg")
|
||||
return _source_getfile.resolve(page_url, _BASE, timeout=timeout)
|
||||
|
|
|
|||
|
|
@ -68,11 +68,6 @@ class Clip(UUIDPKMixin, TimestampMixin, Base):
|
|||
hls: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
||||
likes: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
views: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
#: Przyjecie klipu w samym Goonie (feed Edge): wyswietlenia, ucieczki < 2 s, lajki.
|
||||
#: Licznik bumpuje POST /clips/events, punktacja feedu z tego korzysta (migracja 0052).
|
||||
feed_views: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0")
|
||||
feed_skips: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0")
|
||||
feed_likes: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0")
|
||||
#: Data publikacji u źródła (RedGifs `createDate`), do odznaki „nowe" i sortu.
|
||||
published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
dead_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
|
|
|
|||
|
|
@ -1,136 +0,0 @@
|
|||
"""Zgloszenie „plik zniknal" z telefonu -> sprawdzenie z serwera -> zrodlo martwe.
|
||||
|
||||
Skad: telemetria 7 dni (2026-10-02) miala ~230 bledow „Response code: 410" z fullmovies.xxx
|
||||
i hdporn.gg, a zrodlo znikalo tylko po dlugim przytrzymaniu przez uzytkownika. Kolejne osoby
|
||||
trafialy w ten sam skasowany plik.
|
||||
|
||||
Dwie kontrole, jak przy kazdym przemiale zrodel (patrz notatka o reaperach): telefon dostal
|
||||
404/410 na pliku, a serwer rozwiazuje strone tube na nowo i sam dostaje 404/410 na pliku albo
|
||||
ekstraktor mowi wprost, ze film usuniety (HosterDead, TubePageError 404/410). 503, bledy sieci,
|
||||
pusty ekstraktor i hostery rozwiazywane na telefonie niczego nie oznaczaja.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
|
||||
import httpx
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.db import session_scope
|
||||
from app.models.playback_source import PlaybackSource
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
#: Komunikat ExoPlayera o pliku, ktorego nie ma (expo-video: "... Source error Response code: 410").
|
||||
_GONE_RE = re.compile(r"response code: (404|410)\b", re.IGNORECASE)
|
||||
_UA = "Mozilla/5.0 (Linux; Android 14) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Mobile Safari/537.36"
|
||||
#: Ta sama para (scena, origin) sprawdzana najwyzej raz na tyle sekund (wiele telefonow naraz).
|
||||
_RECHECK_SEC = 3600
|
||||
_checked: dict[tuple, float] = {}
|
||||
_lock = threading.Lock()
|
||||
|
||||
|
||||
def is_gone_report(status: str, error_kind: str | None, error_detail: str | None) -> bool:
|
||||
return status == "error" and bool(_GONE_RE.search(error_detail or ""))
|
||||
|
||||
|
||||
def _file_status(url: str, referer: str | None) -> int | None:
|
||||
headers = {"User-Agent": _UA, "Range": "bytes=0-1023"}
|
||||
if referer:
|
||||
headers["Referer"] = referer
|
||||
try:
|
||||
with httpx.Client(timeout=20, follow_redirects=True) as http:
|
||||
return http.get(url, headers=headers).status_code
|
||||
except httpx.HTTPError:
|
||||
return None
|
||||
|
||||
|
||||
def _mark_dead(pb_id: uuid.UUID, reason: str) -> None:
|
||||
with session_scope() as s:
|
||||
pb = s.get(PlaybackSource, pb_id)
|
||||
if pb is not None and pb.dead_at is None:
|
||||
pb.dead_at = datetime.now(UTC)
|
||||
pb.dead_reason = reason[:512]
|
||||
s.commit()
|
||||
log.info("gone: playback %s (%s) martwy: %s", pb_id, pb.origin, reason)
|
||||
|
||||
|
||||
def verify_gone(scene_id: uuid.UUID, origin: str, *, dry_run: bool = False) -> dict[str, int]:
|
||||
"""Sprawdza zywe zrodla (scena, origin) zgloszone jako 404/410. Zwraca liczniki.
|
||||
`dry_run`: tylko liczy, nic nie oznacza (do sprawdzenia na historii przed backfillem)."""
|
||||
mark = (lambda *_a: None) if dry_run else _mark_dead
|
||||
from app.extractors import HosterDead, TubePageError, is_vps_blocked_fallback, try_extract
|
||||
|
||||
out = {"checked": 0, "dead": 0, "alive": 0, "unknown": 0}
|
||||
if not origin.startswith("tube:"):
|
||||
return out
|
||||
sitetag = origin.split(":", 1)[1]
|
||||
if is_vps_blocked_fallback(sitetag):
|
||||
return out # rozwiazywane tylko na telefonie, serwer nie ma jak sprawdzic
|
||||
with session_scope() as s:
|
||||
rows = s.execute(
|
||||
select(PlaybackSource.id, PlaybackSource.page_url).where(
|
||||
PlaybackSource.scene_id == scene_id,
|
||||
PlaybackSource.origin == origin,
|
||||
PlaybackSource.dead_at.is_(None),
|
||||
)
|
||||
).all()
|
||||
for pb_id, page_url in rows:
|
||||
out["checked"] += 1
|
||||
try:
|
||||
sources = try_extract(sitetag, page_url)
|
||||
except HosterDead as e:
|
||||
mark(pb_id, f"{e} (zgloszenie 410 z telefonu)")
|
||||
out["dead"] += 1
|
||||
continue
|
||||
except TubePageError as e:
|
||||
if e.status_code in (404, 410):
|
||||
mark(pb_id, f"tube page {e.status_code} (zgloszenie 410 z telefonu)")
|
||||
out["dead"] += 1
|
||||
else:
|
||||
out["unknown"] += 1
|
||||
continue
|
||||
except Exception: # noqa: BLE001
|
||||
out["unknown"] += 1
|
||||
continue
|
||||
direct = [x for x in (sources or []) if (x.type or "mp4") in ("mp4", "m3u8")]
|
||||
if not direct:
|
||||
out["unknown"] += 1
|
||||
continue
|
||||
codes = [_file_status(x.link, x.referer or page_url) for x in direct[:3]]
|
||||
if codes and all(c in (404, 410) for c in codes):
|
||||
mark(pb_id, f"plik {codes[0]} z telefonu i z serwera")
|
||||
out["dead"] += 1
|
||||
elif any(c in (200, 206) for c in codes):
|
||||
out["alive"] += 1
|
||||
else:
|
||||
out["unknown"] += 1
|
||||
return out
|
||||
|
||||
|
||||
def verify_gone_async(scene_id: uuid.UUID | None, origin: str) -> None:
|
||||
"""Fire-and-forget z endpointu telemetrii; ta sama para najwyzej raz na godzine."""
|
||||
if scene_id is None:
|
||||
return
|
||||
key = (scene_id, origin)
|
||||
now = time.monotonic()
|
||||
with _lock:
|
||||
if now - _checked.get(key, -1e9) < _RECHECK_SEC:
|
||||
return
|
||||
_checked[key] = now
|
||||
if len(_checked) > 20000:
|
||||
for k in [k for k, t in _checked.items() if now - t > _RECHECK_SEC]:
|
||||
_checked.pop(k, None)
|
||||
|
||||
def run() -> None:
|
||||
try:
|
||||
verify_gone(scene_id, origin)
|
||||
except Exception: # noqa: BLE001
|
||||
log.exception("gone: sprawdzenie %s %s nie wyszlo", scene_id, origin)
|
||||
|
||||
threading.Thread(target=run, name="gone-check", daemon=True).start()
|
||||
|
|
@ -35,27 +35,6 @@ export type ChangelogEntry = {
|
|||
};
|
||||
|
||||
export const CHANGELOG: ChangelogEntry[] = [
|
||||
{
|
||||
id: '2026-10-02d',
|
||||
date: 'October 2026',
|
||||
items: [
|
||||
'Latestpornvideo: newer videos sit on a new lulu domain the app did not know, so they opened the host page and failed. They now play in the app player like the older ones.',
|
||||
],
|
||||
},
|
||||
{
|
||||
id: '2026-10-02c',
|
||||
date: 'October 2026',
|
||||
items: [
|
||||
'Porndoe: many plays ended with "Response code: 403". The video link is tied to the address that asked for it, and the page and the player could reach the site through different addresses. The app now asks for the link itself, the same way the player connects, and skips the web page.',
|
||||
],
|
||||
},
|
||||
{
|
||||
id: '2026-10-02b',
|
||||
date: 'October 2026',
|
||||
items: [
|
||||
'Galaxyporn: about half of the plays ended with "Playback failed". The app grabbed the address of an ad on the vikingfile page and tried to play it as the video. It now waits for the real file.',
|
||||
],
|
||||
},
|
||||
{
|
||||
id: '2026-10-02a',
|
||||
date: 'October 2026',
|
||||
|
|
|
|||
|
|
@ -18,10 +18,6 @@
|
|||
|
||||
const PACKER_HOSTS = new Set([
|
||||
'luluvid.com', 'luluvdo.com', 'lulustream.com', 'lulu.st',
|
||||
// lulust.com: nowa domena tej samej rodziny (latestpornvideo 2026-10-02). Bez niej szla
|
||||
// przez WebView, a w probce wszystkie sceny na lulust konczyly sie 403, na luluvdo gra.
|
||||
// Rodzina rotuje domeny, dlatego nizej jest tez regula na przedrostek lulu.
|
||||
'lulust.com',
|
||||
// luluvid/streamwish forki rotują TLD, `luluvids.top` + cdnstream/cdnvids.top
|
||||
// potwierdzone P.A.C.K.E.R.-JWPlayer (mypornerleak embeds, bug-report 2026-06-07:
|
||||
// bez tego `isPackerHoster` zwracał false → WebView na luluvids.top → "disable
|
||||
|
|
@ -49,9 +45,6 @@ export function isPackerHoster(url: string): boolean {
|
|||
for (const h of PACKER_HOSTS) {
|
||||
if (host.endsWith('.' + h)) return true;
|
||||
}
|
||||
// Kolejne domeny rodziny lulu (luluvid, luluvdo, lulustream, lulust): osadzenie ma
|
||||
// zawsze sciezke /e/<id>. Gdy resolver i tak nie znajdzie skryptu, Player spada na WebView.
|
||||
if (/^lulu[a-z]*\.[a-z]{2,}$/.test(host) && /^\/e\//.test(new URL(url).pathname)) return true;
|
||||
} catch {}
|
||||
return false;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,60 +0,0 @@
|
|||
/**
|
||||
* Mobile-side porndoe.com resolver.
|
||||
*
|
||||
* Linki do pliku (mp4 `?ip=<adres>&token=` i HLS z tokenem w sciezce) sa podpisane pod
|
||||
* adres, z ktorego przyszlo zapytanie `/service/index.json`. Sprawdzone 2026-10-02: token
|
||||
* pobrany przez IPv4 gra tylko przez IPv4, przez IPv6 tylko przez IPv6, z VPS 403.
|
||||
*
|
||||
* Wczesniej porndoe szedl przez WebView: token pobieral Chromium, a plik ExoPlayer
|
||||
* (OkHttp). Na sieciach z IPv4 i IPv6 albo za CGNAT z pula adresow te dwa stosy potrafia
|
||||
* wyjsc innymi adresami i plik dostawal 403 (164 bledy na 278 udanych w tydzien).
|
||||
* Tu token pobiera zwykly `fetch` (tez OkHttp), wiec idzie ta sama droga co odtwarzacz.
|
||||
*
|
||||
* Jedno zapytanie wystarcza: serwis rozpoznaje film po Refererze, ciasteczka niepotrzebne.
|
||||
*/
|
||||
import type { StreamLink } from '../types';
|
||||
|
||||
const UA =
|
||||
'Mozilla/5.0 (Linux; Android 14) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Mobile Safari/537.36';
|
||||
|
||||
type PorndoeSource = {
|
||||
name?: string;
|
||||
height?: string | number;
|
||||
link?: string;
|
||||
type?: string;
|
||||
};
|
||||
|
||||
export async function resolvePorndoePage(pageUrl: string): Promise<StreamLink[]> {
|
||||
let origin: string;
|
||||
try {
|
||||
origin = new URL(pageUrl).origin;
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
let data: { payload?: { player?: { sources?: { mp4?: unknown; hls?: unknown } } } } | null;
|
||||
try {
|
||||
const r = await fetch(`${origin}/service/index.json?device=mobile`, {
|
||||
headers: { 'User-Agent': UA, Referer: pageUrl, Accept: 'application/json' },
|
||||
});
|
||||
if (!r.ok) return [];
|
||||
data = await r.json();
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
const sources = data?.payload?.player?.sources;
|
||||
// mp4 przed HLS: jeden plik z przewijaniem. 1080p ma `type: 'link'` (reklama konta
|
||||
// premium zamiast adresu), dlatego bierzemy tylko `type: 'video'`.
|
||||
const pick = (list: unknown, kind: 'mp4' | 'm3u8'): StreamLink[] =>
|
||||
(Array.isArray(list) ? (list as PorndoeSource[]) : [])
|
||||
.filter((s) => s.type === 'video' && /^https?:\/\//.test(s.link || ''))
|
||||
.sort((a, b) => Number(b.height || 0) - Number(a.height || 0))
|
||||
.map((s) => ({
|
||||
stream_url: s.link!,
|
||||
direct_url: s.link!,
|
||||
headers: { Referer: `${origin}/`, 'User-Agent': UA },
|
||||
quality: s.name || (s.height ? `${s.height}p` : null),
|
||||
type: kind,
|
||||
}));
|
||||
const mp4 = pick(sources?.mp4, 'mp4');
|
||||
return mp4.length > 0 ? mp4 : pick(sources?.hls, 'm3u8');
|
||||
}
|
||||
|
|
@ -505,8 +505,6 @@ function NativeVideoPlayer({ params }: { params: RouteParams }) {
|
|||
links = await (await import('../lib/epornerResolver')).resolveEpornerPage(resolvePageUrl);
|
||||
} else if (resolvePageUrl && playOrigin === 'tube:fpoxxx') {
|
||||
links = await (await import('../lib/fpoxxxResolver')).resolveFpoxxxPage(resolvePageUrl);
|
||||
} else if (resolvePageUrl && playOrigin === 'tube:porndoecom') {
|
||||
links = await (await import('../lib/porndoeResolver')).resolvePorndoePage(resolvePageUrl);
|
||||
} else if (canBackendReresolve && sceneId && params.playbackId) {
|
||||
// Świeży resolve przez backend (VPS re-runuje extractor KVS → nowy nod + token).
|
||||
const res = await client.resolvePlayback(sceneId, params.playbackId);
|
||||
|
|
@ -1283,18 +1281,6 @@ const INJECTED_JS = `
|
|||
// podsluchujemy POST zwracajacy bezposredni link. NICZEGO nie chowamy: pierwsza
|
||||
// wersja chowala oprawe strony i przy nierenderujacym sie widgecie zostawal
|
||||
// czarny ekran, gorzej niz przed zmiana.
|
||||
//
|
||||
// Host sprawdzamy NA POCZATKU i wychodzimy POZA blokiem try. Wczesniej wyjscie
|
||||
// siedzialo w tym samym try co oswajanie strony, a skrypt rusza, zanim powstanie
|
||||
// <html>: appendChild na null rzucal, catch polykal blad i omijal return. Na
|
||||
// vikingfile szedl wiec caly skrypt ponizej, a podsluch fetch bral zapytanie
|
||||
// reklamy Adcash (adexchangerapid.com/...&cbtitle=<nazwa pliku>.mp4) za film.
|
||||
// Telemetria: ~155 bledow "None of the available extractors" i ~50 "adexchangerapid"
|
||||
// tygodniowo, polowa odtworzen galaxyporn. Dowod z emulatora 2026-10-02.
|
||||
var goonTurnstileHost = false;
|
||||
try {
|
||||
goonTurnstileHost = /vik[i1]ngfile\\.|playmogo\\.|dood|d0000d|dooood|do0od|do7go|ds2play|doply\\.|myvidplay\\./i.test(location.hostname);
|
||||
} catch (e) {}
|
||||
try {
|
||||
if (/vik[i1]ngfile\\./i.test(location.hostname)) {
|
||||
// UWAGA: NIE chowamy tu reszty strony. Pierwsza wersja robila dokladnie to i user
|
||||
|
|
@ -1302,12 +1288,6 @@ const INJECTED_JS = `
|
|||
// wyrenderuje, na czarnym tle nie zostaje juz nic. Gorsze niz stan wyjsciowy. Teraz
|
||||
// tylko podswietlamy wlasciwe miejsce i przewijamy do niego, wiec najgorszy przypadek
|
||||
// to normalna strona hostera, czyli to, co user widzial przed ta zmiana.
|
||||
var vkSent = false;
|
||||
var vkSendUrl = function(u) {
|
||||
if (vkSent || !/^https?:/i.test(u || '') || !window.ReactNativeWebView) return;
|
||||
vkSent = true;
|
||||
window.ReactNativeWebView.postMessage(JSON.stringify({ type: 'video_url', url: u }));
|
||||
};
|
||||
var vkCss = document.createElement('style');
|
||||
vkCss.textContent = [
|
||||
'#captcha-download{outline:2px solid #e05a6d!important;border-radius:10px!important;',
|
||||
|
|
@ -1315,17 +1295,9 @@ const INJECTED_JS = `
|
|||
'#goon-vk-hint{color:#e05a6d;font:600 15px system-ui,sans-serif;text-align:center;',
|
||||
'padding:8px 0}',
|
||||
].join('');
|
||||
(document.head || document.documentElement).appendChild(vkCss);
|
||||
var vkScrolled = false;
|
||||
var vkReady = function() {
|
||||
// Styl dokladamy dopiero tutaj: na starcie skryptu nie ma jeszcze ani <head>,
|
||||
// ani <html> (patrz komentarz przy goonTurnstileHost).
|
||||
if (!vkCss.parentNode) {
|
||||
var vkRoot = document.head || document.documentElement;
|
||||
if (vkRoot) vkRoot.appendChild(vkCss);
|
||||
}
|
||||
// Zapas dla podsluchu nizej: strona sama wpisuje link do <video>, gdy to plik wideo.
|
||||
var vkVideo = document.querySelector('video');
|
||||
if (vkVideo && /^https?:/i.test(vkVideo.src || '')) vkSendUrl(vkVideo.src);
|
||||
var box = document.getElementById('captcha-download');
|
||||
if (box && !document.getElementById('goon-vk-hint')) {
|
||||
var hint = document.createElement('div');
|
||||
|
|
@ -1366,50 +1338,41 @@ const INJECTED_JS = `
|
|||
// gdy tylko link sie pojawi, oddajemy go aplikacji, ktora przechodzi na natywny
|
||||
// odtwarzacz. Oswojony wyglad strony zostaje jako zabezpieczenie na wypadek, gdyby
|
||||
// zabezpieczenie jednak zazadalo kliniecia.
|
||||
//
|
||||
// Bierzemy tylko odpowiedz na POST do wlasnej domeny strony: skrypty reklam tez
|
||||
// robia zapytania i ich JSON-y bywaja podobne.
|
||||
var vkOpen = XMLHttpRequest.prototype.open;
|
||||
var vkSend = XMLHttpRequest.prototype.send;
|
||||
XMLHttpRequest.prototype.open = function(method, reqUrl) {
|
||||
try {
|
||||
this.__goonVkOwn = String(method).toUpperCase() === 'POST' &&
|
||||
new URL(String(reqUrl), location.href).origin === location.origin;
|
||||
} catch (e) {
|
||||
this.__goonVkOwn = false;
|
||||
}
|
||||
return vkOpen.apply(this, arguments);
|
||||
};
|
||||
XMLHttpRequest.prototype.open = function() { return vkOpen.apply(this, arguments); };
|
||||
XMLHttpRequest.prototype.send = function() {
|
||||
var self = this;
|
||||
try {
|
||||
if (self.__goonVkOwn) {
|
||||
self.addEventListener('load', function() {
|
||||
try {
|
||||
var j = JSON.parse(self.responseText);
|
||||
if (j && typeof j.link === 'string') vkSendUrl(j.link);
|
||||
} catch (e) {}
|
||||
});
|
||||
}
|
||||
self.addEventListener('load', function() {
|
||||
try {
|
||||
var j = JSON.parse(self.responseText);
|
||||
if (j && j.link && window.ReactNativeWebView) {
|
||||
window.ReactNativeWebView.postMessage(
|
||||
JSON.stringify({ type: 'video_url', url: j.link })
|
||||
);
|
||||
}
|
||||
} catch (e) {}
|
||||
});
|
||||
} catch (e) {}
|
||||
return vkSend.apply(this, arguments);
|
||||
};
|
||||
}
|
||||
// Na KAZDYM hosterze z brama Cloudflare KONCZYMY tutaj. Ponizej patchujemy prototypy
|
||||
// HTMLMediaElement, klikamy w plakat i usuwamy elementy uznane za reklamy, a Turnstile
|
||||
// potrafi uznac tak zmodyfikowane srodowisko za podejrzane i odrzucic weryfikacje
|
||||
// (zgloszenie: "wywolanie cloudflare dziala, ale weryfikacja jest negatywna").
|
||||
//
|
||||
// Rodzina dooda doszla 2026-08-26. Playmogo (tam laduje myvidplay i doply) renderuje
|
||||
// widget przez turnstile.render z size flexible, DOPIERO po kliknieciu w play,
|
||||
// i trzyma go w kontenerze o klasie captcha-player z z-index 100000, czyli w tym,
|
||||
// co nasz usuwacz nakladek traktuje jak reklame. Do tego nasze auto-klikniecie w plakat
|
||||
// odpalalo ta brame, zanim widget zdazyl sie wyrenderowac.
|
||||
//
|
||||
// Na stronie pliku i tak nie potrzebujemy ani wyciszania autoplay, ani ad-blocka,
|
||||
// a bezposredni link przechwytujemy podsluchem powyzej.
|
||||
if (/vik[i1]ngfile\\.|playmogo\\.|dood|d0000d|dooood|do0od|do7go|ds2play|doply\\.|myvidplay\\./i.test(location.hostname)) return;
|
||||
} catch (e) {}
|
||||
// Na KAZDYM hosterze z brama Cloudflare KONCZYMY tutaj. Ponizej patchujemy prototypy
|
||||
// HTMLMediaElement, klikamy w plakat i usuwamy elementy uznane za reklamy, a Turnstile
|
||||
// potrafi uznac tak zmodyfikowane srodowisko za podejrzane i odrzucic weryfikacje
|
||||
// (zgloszenie: "wywolanie cloudflare dziala, ale weryfikacja jest negatywna").
|
||||
//
|
||||
// Rodzina dooda doszla 2026-08-26. Playmogo (tam laduje myvidplay i doply) renderuje
|
||||
// widget przez turnstile.render z size flexible, DOPIERO po kliknieciu w play,
|
||||
// i trzyma go w kontenerze o klasie captcha-player z z-index 100000, czyli w tym,
|
||||
// co nasz usuwacz nakladek traktuje jak reklame. Do tego nasze auto-klikniecie w plakat
|
||||
// odpalalo ta brame, zanim widget zdazyl sie wyrenderowac.
|
||||
//
|
||||
// Na stronie pliku i tak nie potrzebujemy ani wyciszania autoplay, ani ad-blocka,
|
||||
// a bezposredni link przechwytujemy podsluchem powyzej.
|
||||
if (goonTurnstileHost) return;
|
||||
|
||||
// -- 0a. Mute autoplay until the FIRST user gesture (kills the unmuted-audio flash
|
||||
// gdy WebView otwiera stronę żeby przechwycić stream, report dccc05e4).
|
||||
|
|
@ -1532,10 +1495,6 @@ const INJECTED_JS = `
|
|||
// ("zacina się na pierwszych sekundach i robi loop", bug-reports 40ec88cc / 92e9133e).
|
||||
// isLikelyAd odsiewał je już przy auto-extrakcji, ale sama strona i tak je ładowała.
|
||||
'opencdn.b-cdn.net',
|
||||
// Adcash: zapytanie o reklame niesie tytul strony w parametrze (cbtitle=<plik>.mp4),
|
||||
// a VIDEO_RE widzi ".mp4" takze w parametrach. Tak reklama trafiala do odtwarzacza
|
||||
// na vikingfile (2026-10-02).
|
||||
'adexchangerapid.com','usrpubtrk.com',
|
||||
];
|
||||
const isAdHost = function(url) {
|
||||
if (!url) return false;
|
||||
|
|
|
|||
|
|
@ -25,7 +25,6 @@ import { resolvePornxpPage } from '../lib/pornxpResolver';
|
|||
import { resolveSxyprnPage } from '../lib/sxyprnResolver';
|
||||
import { resolveEpornerPage } from '../lib/epornerResolver';
|
||||
import { resolveFpoxxxPage } from '../lib/fpoxxxResolver';
|
||||
import { resolvePorndoePage } from '../lib/porndoeResolver';
|
||||
import type { RootStackParamList } from '../navigation';
|
||||
import { fonts, theme } from '../theme';
|
||||
import { HeaderIconButton } from '../ui';
|
||||
|
|
@ -644,7 +643,7 @@ function PlaybackButton({
|
|||
|
||||
// IP-bound tuby resolwowane phone-side: przekaż page_url, by Player mógł
|
||||
// re-resolve'ować świeży token gdy native padnie na initial-load (zmiana IP / TTL).
|
||||
const PHONE_RESOLVE_ORIGINS = ['tube:sxyprncom', 'tube:epornercom', 'tube:fpoxxx', 'tube:porndoecom'];
|
||||
const PHONE_RESOLVE_ORIGINS = ['tube:sxyprncom', 'tube:epornercom', 'tube:fpoxxx'];
|
||||
nav.navigate('Player', {
|
||||
url: initialUrl,
|
||||
sceneId,
|
||||
|
|
@ -697,8 +696,6 @@ function PlaybackButton({
|
|||
? resolveFpoxxxPage
|
||||
: source.origin === 'tube:pornxpph'
|
||||
? resolvePornxpPage
|
||||
: source.origin === 'tube:porndoecom'
|
||||
? resolvePorndoePage
|
||||
: // hqfap zdjęty 2026-07-28: strona wdrożyła Cloudflare z JS-challenge na
|
||||
// /watch/ (403 dla każdego zwykłego fetcha, także z pełnym kompletem
|
||||
// nagłówków przeglądarki), więc phone-resolve tylko padał i zrzucał usera
|
||||
|
|
|
|||
|
|
@ -1,62 +0,0 @@
|
|||
"""Porownanie punktacji feedu Edge: grupa nowa vs kontrolna (stara punktacja).
|
||||
|
||||
Przydzial do grupy jest staly po hashu id urzadzenia (`app.clips_feed.uses_v1_scoring`),
|
||||
wiec wystarczy pogrupowac zdarzenia z okresu po wdrozeniu. Liczymy tylko slot 'affinity',
|
||||
bo tylko tam punktacja dziala (explore losuje tak samo w obu grupach). Srednie sa po
|
||||
urzadzeniach (kazde urzadzenie z >= MIN_VIEWS obejrzeniami waży tyle samo), zeby kilku
|
||||
najaktywniejszych nie przesadzalo o wyniku.
|
||||
|
||||
docker compose exec -T worker python3 /srv/scripts/edge_ab_report.py [od] [do]
|
||||
Domyslnie od wdrozenia (2026-10-02 10:00 UTC) do teraz. Okno sprzed wdrozenia to test A/A:
|
||||
obie grupy mialy wtedy te sama punktacje, wiec roznice pokazuja sam szum.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import statistics
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
|
||||
from sqlalchemy import text
|
||||
|
||||
from app.clips_feed import uses_v1_scoring
|
||||
from app.db import session_scope
|
||||
|
||||
SINCE = sys.argv[1] if len(sys.argv) > 1 else "2026-10-02T10:00:00+00:00"
|
||||
UNTIL = sys.argv[2] if len(sys.argv) > 2 else "2100-01-01T00:00:00+00:00"
|
||||
MIN_VIEWS = 20
|
||||
VIEW_KINDS = ("complete", "skip", "impression", "autoswipe")
|
||||
|
||||
with session_scope() as s:
|
||||
rows = s.execute(text("""
|
||||
SELECT device_id, kind, dwell_ms FROM clip_feed_events
|
||||
WHERE created_at >= CAST(:since AS timestamptz) AND created_at < CAST(:until AS timestamptz)
|
||||
AND slot = 'affinity'
|
||||
"""), {"since": SINCE, "until": UNTIL}).all()
|
||||
|
||||
per = defaultdict(lambda: {"views": 0, "complete": 0, "skip": 0, "like": 0, "not_interested": 0, "dwell": []})
|
||||
for dev, kind, dwell in rows:
|
||||
d = per[dev]
|
||||
if kind in VIEW_KINDS:
|
||||
d["views"] += 1
|
||||
if dwell:
|
||||
d["dwell"].append(dwell)
|
||||
if kind in d and kind != "views":
|
||||
d[kind] += 1
|
||||
|
||||
groups: dict[str, list[dict]] = {"nowa": [], "kontrolna": []}
|
||||
for dev, d in per.items():
|
||||
if d["views"] >= MIN_VIEWS:
|
||||
groups["kontrolna" if uses_v1_scoring(dev) else "nowa"].append(d)
|
||||
|
||||
print(f"od {SINCE} do {UNTIL}, zdarzen {len(rows)}, urzadzen z >= {MIN_VIEWS} obejrzeniami: "
|
||||
+ ", ".join(f"{g} {len(v)}" for g, v in groups.items()))
|
||||
for g, devs in groups.items():
|
||||
if not devs:
|
||||
continue
|
||||
def avg(f):
|
||||
return 100 * statistics.mean(f(d) for d in devs)
|
||||
print(f" {g:9s} dokonczone {avg(lambda d: d['complete'] / d['views']):5.1f}%"
|
||||
f" lajki {avg(lambda d: d['like'] / d['views']):4.1f}%"
|
||||
f" ucieczki {avg(lambda d: d['skip'] / d['views']):5.1f}%"
|
||||
f" nie-interesuje {avg(lambda d: d['not_interested'] / d['views']):4.1f}%"
|
||||
f" mediana dwell {statistics.median(statistics.median(d['dwell']) for d in devs if d['dwell']) / 1000:.1f} s")
|
||||
|
|
@ -1,95 +0,0 @@
|
|||
"""Odtworzenie uczenia feedu Edge zdarzenie po zdarzeniu (bez zagladania w przyszlosc).
|
||||
|
||||
Dla kazdego wyswietlenia (complete/skip/impression/autoswipe) zapisuje cechy znane W TEJ
|
||||
CHWILI: skladniki obecnej punktacji z wag urzadzenia sprzed zdarzenia oraz liczniki klipu
|
||||
z wczesniejszych zdarzen wszystkich urzadzen. Wynik CSV na stdout.
|
||||
|
||||
Uzyte do przestrojenia punktacji 2026-10-02 (patrz app/clips_feed.py, score_candidate).
|
||||
Wybor tagow na starcie (onboarding) nie jest zdarzeniem, wiec tu go brakuje.
|
||||
|
||||
docker compose exec -T worker sh -c "cd /srv && python3 scripts/edge_replay_features.py | gzip -c" > edge.csv.gz
|
||||
Analiza lokalnie: python scripts/edge_tuning_analysis.py edge.csv (pandas + scikit-learn).
|
||||
"""
|
||||
import csv
|
||||
import math
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from datetime import timedelta
|
||||
|
||||
from sqlalchemy import text
|
||||
|
||||
from app import clips_feed as cf
|
||||
from app.db import session_scope
|
||||
|
||||
with session_scope() as s:
|
||||
ev = s.execute(text("""
|
||||
select device_id, clip_id, kind, dwell_ms, reason, slot, created_at
|
||||
from clip_feed_events where clip_id is not null order by created_at""")).all()
|
||||
ids = list({e.clip_id for e in ev})
|
||||
clips = {}
|
||||
for i in range(0, len(ids), 5000):
|
||||
for r in s.execute(text("""select id, creator, likes, published_at, niches, duration_sec, width, height,
|
||||
has_audio, creator_verified
|
||||
from clips where id = any(:ids)"""), {"ids": ids[i:i + 5000]}).all():
|
||||
clips[r.id] = r
|
||||
tags = defaultdict(list)
|
||||
for i in range(0, len(ids), 5000):
|
||||
for cid, t in s.execute(text("select clip_id, tag from clip_tags where clip_id = any(:ids)"), {"ids": ids[i:i + 5000]}).all():
|
||||
tags[cid].append(t)
|
||||
n_all = s.execute(text("select count(*) from clips")).scalar()
|
||||
tag_df = dict(s.execute(text("select tag, count(*) from clip_tags group by tag")).all())
|
||||
idf = {t: math.log((n_all + 1) / (n + 1)) for t, n in tag_df.items()}
|
||||
print(f"zdarzen {len(ev)}, klipow {len(clips)}, tagow {len(idf)}", file=sys.stderr)
|
||||
|
||||
aff = defaultdict(cf.Affinity)
|
||||
last_day = {}
|
||||
dev_n = defaultdict(int)
|
||||
# liczniki klipu z wczesniejszych zdarzen (wszystkie urzadzenia)
|
||||
c_seen = defaultdict(int)
|
||||
c_comp = defaultdict(int)
|
||||
c_skip = defaultdict(int)
|
||||
c_like = defaultdict(int)
|
||||
liked = set((e.device_id, e.clip_id) for e in ev if e.kind == "like")
|
||||
|
||||
w = csv.writer(sys.stdout)
|
||||
w.writerow(["dev", "ts", "slot", "kind", "dwell", "liked", "tag_sum", "tag_pos", "tag_n", "niche_sum", "creator_w",
|
||||
"tag_idf_sum", "tag_idf_pos", "pop", "fresh", "dur", "vertical", "audio", "verified", "c_seen", "c_comp", "c_skip", "c_like", "dev_n"])
|
||||
OUTCOME = {"complete", "skip", "impression", "autoswipe"}
|
||||
for e in ev:
|
||||
c = clips.get(e.clip_id)
|
||||
if c is None:
|
||||
continue
|
||||
a = aff[e.device_id]
|
||||
day = e.created_at.date()
|
||||
prev = last_day.get(e.device_id)
|
||||
if prev is not None and day > prev:
|
||||
for _ in range((day - prev).days):
|
||||
a.tags = cf.decay_weights(a.tags)
|
||||
a.niches = cf.decay_weights(a.niches)
|
||||
a.creators = cf.decay_weights(a.creators)
|
||||
last_day[e.device_id] = day
|
||||
t = tags.get(e.clip_id, [])
|
||||
niches = list(c.niches or [])
|
||||
if e.kind in OUTCOME:
|
||||
ts_vals = [a.tags.get(x, 0.0) for x in t]
|
||||
fresh = int(bool(c.published_at and c.published_at >= e.created_at - timedelta(days=cf.FRESH_DAYS)))
|
||||
vert = int(bool(c.width and c.height and c.height > c.width))
|
||||
w.writerow([e.device_id[:12], e.created_at.isoformat(), e.slot or "", e.kind, e.dwell_ms or 0,
|
||||
int((e.device_id, e.clip_id) in liked),
|
||||
round(sum(ts_vals), 3), sum(1 for v in ts_vals if v > 0), len(t),
|
||||
round(sum(a.niches.get(n, 0.0) for n in niches), 3),
|
||||
round(a.creators.get(c.creator or "", 0.0), 3),
|
||||
round(sum(a.tags.get(x, 0.0) * idf.get(x, 0.0) for x in t), 3),
|
||||
round(sum(idf.get(x, 0.0) for x in t if a.tags.get(x, 0.0) > 0), 3),
|
||||
round(math.log1p(max(c.likes or 0, 0)), 3), fresh, round(c.duration_sec or 0, 1), vert,
|
||||
int(bool(c.has_audio)), int(bool(c.creator_verified)),
|
||||
c_seen[e.clip_id], c_comp[e.clip_id], c_skip[e.clip_id], c_like[e.clip_id], dev_n[e.device_id]])
|
||||
c_seen[e.clip_id] += 1
|
||||
if e.kind == "complete":
|
||||
c_comp[e.clip_id] += 1
|
||||
elif e.kind == "skip":
|
||||
c_skip[e.clip_id] += 1
|
||||
dev_n[e.device_id] += 1
|
||||
elif e.kind == "like":
|
||||
c_like[e.clip_id] += 1
|
||||
cf.apply_event(a, kind=e.kind, dwell_ms=e.dwell_ms, reason=e.reason, tags=t, niches=niches, creator=c.creator)
|
||||
|
|
@ -1,82 +0,0 @@
|
|||
"""Ocena punktacji feedu Edge na cechach z scripts/edge_replay_features.py.
|
||||
|
||||
Trening na 70% najstarszych wyswietlen, test na reszcie. Glowna miara: AUC W OBREBIE
|
||||
URZADZENIA (czy wyzszy wynik u tego samego urzadzenia znaczy lepsza reakcje), bo globalne
|
||||
AUC miesza aktywnych z nowymi. Dwie etykiety: lajk albo >= 10 s ogladania (odporna na
|
||||
dlugosc klipu) oraz dokonczenie albo lajk. F5 to formula wdrozona 2026-10-02.
|
||||
Wymaga pandas i scikit-learn (lokalnie, nie w kontenerze).
|
||||
"""
|
||||
import sys
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from sklearn.linear_model import LogisticRegression
|
||||
from sklearn.metrics import roc_auc_score
|
||||
|
||||
df = pd.read_csv(sys.argv[1], parse_dates=["ts"])
|
||||
A = 10.0
|
||||
P_LIKE, P_SKIP = 0.03, 0.5
|
||||
df["like_rate"] = (df.c_like + A * P_LIKE) / (df.c_seen + A)
|
||||
df["skip_rate"] = (df.c_skip + A * P_SKIP) / (df.c_seen + A)
|
||||
df["popul"] = df["pop"]
|
||||
slog = lambda x: np.sign(x) * np.log1p(np.abs(x)) # noqa: E731
|
||||
for c in ["tag_sum", "tag_idf_sum", "creator_w", "niche_sum"]:
|
||||
df["s_" + c] = slog(df[c])
|
||||
|
||||
|
||||
def within(d, col):
|
||||
num = den = 0.0
|
||||
for _, g in d.groupby("dev"):
|
||||
if len(g) < 30 or g.y.nunique() < 2 or g[col].nunique() < 2:
|
||||
continue
|
||||
num += roc_auc_score(g.y, g[col]) * len(g)
|
||||
den += len(g)
|
||||
return num / den if den else float("nan")
|
||||
|
||||
|
||||
def labelled(kind):
|
||||
if kind == "dwell":
|
||||
good = (df.liked == 1) | (df.dwell >= 10000)
|
||||
else:
|
||||
good = (df.kind == "complete") | (df.liked == 1)
|
||||
bad = df.kind == "skip"
|
||||
d = df[good | bad].copy()
|
||||
d["y"] = good[good | bad].astype(int)
|
||||
return d.sort_values("ts")
|
||||
|
||||
|
||||
def cur(d):
|
||||
return d.tag_sum + 2 * d.niche_sum + 3 * d.creator_w + 0.3 * d.popul + 0.5 * d.fresh
|
||||
|
||||
|
||||
def formula(d, w):
|
||||
return (w["tag"] * d.s_tag_idf_sum + w["creator"] * d.s_creator_w + w["pop"] * d.popul
|
||||
+ w["like"] * (d.like_rate - P_LIKE) - w["skip"] * (d.skip_rate - P_SKIP))
|
||||
|
||||
|
||||
for kind in ("dwell", "complete"):
|
||||
d = labelled(kind)
|
||||
cut = int(len(d) * 0.7)
|
||||
tr, te = d.iloc[:cut].copy(), d.iloc[cut:].copy()
|
||||
print(f"\n===== etykieta {kind}: trening {len(tr)}, test {len(te)}")
|
||||
print(" AUC w obrebie urzadzenia (test):")
|
||||
for f in ["s_tag_sum", "s_tag_idf_sum", "tag_idf_pos", "s_creator_w", "s_niche_sum", "popul", "like_rate"]:
|
||||
print(f" {f:14s} {within(te, f):.3f}")
|
||||
cols = ["s_tag_idf_sum", "s_creator_w", "s_niche_sum", "popul", "like_rate", "skip_rate"]
|
||||
m = LogisticRegression(max_iter=3000).fit(tr[cols], tr.y)
|
||||
coef = dict(zip(cols, m.coef_[0]))
|
||||
base = coef["s_tag_idf_sum"]
|
||||
print(" wspolczynniki w jednostkach tagu:", {c: round(v / base, 2) for c, v in coef.items()})
|
||||
te["cur"] = cur(te)
|
||||
te["model"] = m.decision_function(te[cols])
|
||||
for name, w in {
|
||||
"F1": {"tag": 1.0, "creator": 1.0, "pop": 0.3, "like": 20.0, "skip": 2.0},
|
||||
"F2": {"tag": 1.0, "creator": 1.5, "pop": 0.5, "like": 30.0, "skip": 3.0},
|
||||
"F3": {"tag": 1.0, "creator": 1.0, "pop": 0.5, "like": 40.0, "skip": 4.0},
|
||||
"F4": {"tag": 1.0, "creator": 2.5, "pop": 0.8, "like": 120.0, "skip": 15.0},
|
||||
}.items():
|
||||
te[name] = formula(te, w)
|
||||
print(" porownanie (test): AUC globalnie / w obrebie urzadzenia")
|
||||
te["F5"] = formula(te, {"tag": 1.0, "creator": 2.5, "pop": 0.8, "like": 120.0, "skip": 15.0}) + 0.25 * te.s_niche_sum + 0.3 * te.fresh
|
||||
for f in ["cur", "model", "F1", "F2", "F3", "F4", "F5"]:
|
||||
print(f" {f:6s} {roc_auc_score(te.y, te[f]):.3f} / {within(te, f):.3f}")
|
||||
|
|
@ -1,38 +0,0 @@
|
|||
"""Jednorazowo: zrodla, ktore telefony zglosily jako 404/410, sprawdzone z serwera.
|
||||
|
||||
Ta sama logika co na zywo (app/playback_gone.py), dla zgloszen sprzed wdrozenia.
|
||||
docker compose exec -T worker python3 /srv/scripts/verify_gone_backfill.py [dni] [--commit] [--limit N]
|
||||
Bez --commit tylko liczy.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from collections import Counter
|
||||
|
||||
from sqlalchemy import text
|
||||
|
||||
from app.db import session_scope
|
||||
from app.playback_gone import verify_gone
|
||||
|
||||
days = int(next((a for a in sys.argv[1:] if a.isdigit()), "14"))
|
||||
commit = "--commit" in sys.argv
|
||||
limit = int(sys.argv[sys.argv.index("--limit") + 1]) if "--limit" in sys.argv else 100000
|
||||
|
||||
with session_scope() as s:
|
||||
pairs = s.execute(text("""
|
||||
SELECT DISTINCT e.scene_id, e.origin FROM playback_events e
|
||||
WHERE e.created_at > now() - make_interval(days => :d) AND e.status = 'error'
|
||||
AND e.error_detail ~* 'response code: (404|410)' AND e.scene_id IS NOT NULL
|
||||
AND EXISTS (SELECT 1 FROM playback_sources p WHERE p.scene_id = e.scene_id
|
||||
AND p.origin = e.origin AND p.dead_at IS NULL)
|
||||
LIMIT :lim"""), {"d": days, "lim": limit}).all()
|
||||
print(f"par (scena, serwis) do sprawdzenia: {len(pairs)}, {'NA SERIO' if commit else 'na sucho'}", flush=True)
|
||||
total: Counter = Counter()
|
||||
per_origin: dict[str, Counter] = {}
|
||||
for scene_id, origin in pairs:
|
||||
r = verify_gone(scene_id, origin, dry_run=not commit)
|
||||
total.update(r)
|
||||
per_origin.setdefault(origin, Counter()).update(r)
|
||||
for o, c in sorted(per_origin.items(), key=lambda kv: -kv[1]["checked"]):
|
||||
print(f" {o:24s} {dict(c)}")
|
||||
print("razem:", dict(total))
|
||||
|
|
@ -15,9 +15,7 @@ from app.clips_feed import (
|
|||
pick_affinity,
|
||||
pick_explore,
|
||||
score_candidate,
|
||||
score_candidate_v1,
|
||||
split_counts,
|
||||
uses_v1_scoring,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -97,41 +95,3 @@ def test_pick_explore_weighted_but_complete():
|
|||
cands = [_cand("a", likes=0), _cand("b", likes=10**6)]
|
||||
out = pick_explore(cands, 2, random.Random(1))
|
||||
assert {c.creator for c in out} == {"a", "b"}
|
||||
|
||||
|
||||
def test_feed_history_dominates_ranking():
|
||||
# Klip, ktory u innych zbiera lajki, idzie przed takim samym z ucieczkami (2026-10-02).
|
||||
aff = Affinity(tags={"x": 5.0})
|
||||
liked = Candidate(id=uuid.uuid4(), creator="a", likes=0, published_at=None, tags=["x"], niches=[],
|
||||
feed_views=50, feed_skips=10, feed_likes=8)
|
||||
skipped = Candidate(id=uuid.uuid4(), creator="a", likes=0, published_at=None, tags=["x"], niches=[],
|
||||
feed_views=50, feed_skips=45, feed_likes=0)
|
||||
fresh_clip = Candidate(id=uuid.uuid4(), creator="a", likes=0, published_at=None, tags=["x"], niches=[])
|
||||
assert score_candidate(liked, aff) > score_candidate(fresh_clip, aff) > score_candidate(skipped, aff)
|
||||
|
||||
|
||||
def test_rare_tag_counts_more_than_common():
|
||||
aff = Affinity(tags={"rare": 5.0, "common": 5.0})
|
||||
idf = {"rare": 8.0, "common": 0.5}
|
||||
rare = _cand("a", tags=["rare"])
|
||||
common = _cand("a", tags=["common"])
|
||||
assert score_candidate(rare, aff, idf=idf) > score_candidate(common, aff, idf=idf)
|
||||
# stara punktacja ich nie rozroznia
|
||||
assert score_candidate_v1(rare, aff) == score_candidate_v1(common, aff)
|
||||
|
||||
|
||||
def test_many_generic_tags_do_not_swamp_creator():
|
||||
# Stara punktacja: 12 ogolnych tagow po 5 pkt bilo ulubionego tworce (60 > 3*10).
|
||||
aff = Affinity(tags={f"t{i}": 5.0 for i in range(12)}, creators={"fav": 10.0})
|
||||
idf = {f"t{i}": 0.3 for i in range(12)}
|
||||
generic = _cand("other", tags=[f"t{i}" for i in range(12)])
|
||||
favourite = _cand("fav")
|
||||
assert score_candidate(favourite, aff, idf=idf) > score_candidate(generic, aff, idf=idf)
|
||||
assert score_candidate_v1(generic, aff) > score_candidate_v1(favourite, aff)
|
||||
|
||||
|
||||
def test_v1_control_group_is_stable_and_about_a_quarter():
|
||||
ids = [uuid.uuid4().hex for _ in range(4000)]
|
||||
share = sum(uses_v1_scoring(d) for d in ids) / len(ids)
|
||||
assert 0.21 < share < 0.29
|
||||
assert all(uses_v1_scoring(d) == uses_v1_scoring(d) for d in ids[:50])
|
||||
|
|
|
|||
|
|
@ -1,23 +0,0 @@
|
|||
from app.connectors.dooplay import _LIST_ITEM_RE
|
||||
|
||||
|
||||
def _urls(html):
|
||||
return [m.group("url") or m.group("url2") for m in _LIST_ITEM_RE.finditer(html)]
|
||||
|
||||
|
||||
def test_mangoporn_class_before_href():
|
||||
# mangoporn od 2026-10: klasa przed href, wczesniej wzorzec nie lapal nic.
|
||||
html = (
|
||||
'<a class="thumb" href="https://mangoporn.net/movies/major-creampie/"><div></div></a>'
|
||||
'<a class="infos" href="https://mangoporn.net/movies/major-creampie/" title="Major Creampie">'
|
||||
'<a title="Porn Movies" href="https://mangoporn.net/movies/category/porn-movies-xxx/" class="nav-link">'
|
||||
)
|
||||
assert _urls(html) == [
|
||||
"https://mangoporn.net/movies/major-creampie/",
|
||||
"https://mangoporn.net/movies/major-creampie/",
|
||||
]
|
||||
|
||||
|
||||
def test_old_layouts_still_match():
|
||||
assert _urls('<a href="https://mangoporn.net/movies/old-one/">x</a>') == ["https://mangoporn.net/movies/old-one/"]
|
||||
assert _urls('<a href="https://streamporn.li/watch-xxx-a/" class="ml-mask jt">') == ["https://streamporn.li/watch-xxx-a/"]
|
||||
|
|
@ -36,18 +36,3 @@ def test_adultempire_id_from_both_poster_formats():
|
|||
assert (old.group(1) or old.group(2)) == "5128705"
|
||||
assert (new.group(1) or new.group(2)) == "5136742"
|
||||
assert _AE_ID_RE.search("https://i.ibb.co/DHDd9pm9/Private-POV.jpg") is None
|
||||
|
||||
|
||||
_MULTILINE = (
|
||||
'<div class="movie">\n<div class="imagen">\n'
|
||||
'<img src="https://imgs4cdn.adultempire.com/movie/2755934?w=500&h=700&fit=cover" alt="Busty Treats Vol. 2" '
|
||||
'width="100%" height="100%" />\n<a href="https://yourdailypornmovies.ws/busty-treats-vol-2/"><span class="player"></span></a>\n'
|
||||
'</div>\n<h2>Busty Treats Vol. 2</h2>\n<span class="year">2026</span></div>'
|
||||
)
|
||||
|
||||
|
||||
def test_card_with_newlines_between_tags():
|
||||
# Uklad strony od 2026-10: nowe linie miedzy znacznikami (0 filmow do poprawki).
|
||||
cards = list(_CARD_RE.finditer(_MULTILINE + _MULTILINE.replace("busty-treats-vol-2", "b2")))
|
||||
assert [c.group("title") for c in cards] == ["Busty Treats Vol. 2", "Busty Treats Vol. 2"]
|
||||
assert cards[0].group("url").endswith("/busty-treats-vol-2/")
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue