From 2ef54ad6eac724098fc1a80cb52727fad517f2a4 Mon Sep 17 00:00:00 2001 From: goon-foss Date: Mon, 27 Jul 2026 09:37:51 +0200 Subject: [PATCH] fix(triage): theporndude candidate audit had two false-negative bugs The domain fallback built its guess from the REVIEW slug: when pdude.link resolved to the porndudecams interstitial, "youperv-website-review" became youperv-website-review.com, which of course refused the connection and the candidate was filed as dead. That hit 15 of 144 entries, and among them were youperv and pornbusy, which we later found independently and shipped at 4.5/5. Now the -website-review / -review suffix is stripped, and 13 of those 15 answer 200. The audit also fetched with plain httpx, whose TLS fingerprint most Cloudflare-fronted tubes reject, so timeouts and 403s were being read as "dead" for sites that work fine in a browser. It now goes through browser_get (curl_cffi), the same path our scrapers use, so the triage finally measures what production sees. Worth knowing before trusting the output: the heuristic score does NOT predict a candidate's value to us. It matches cosmetic homepage markers, and on sites we have since evaluated properly it scored youperv 0 and pornbusy -1.5 (a false auth_wall) while giving 2.5 to hqfap, which we removed as an orphan factory. Use the refreshed file as a list of live, unexplored domains (95 of them) and judge each one on canonical match and metadata quality, not on this number. Co-Authored-By: Claude Opus 4.5 --- scripts/theporndude_curl_triage.py | 43 +- theporndude_triage.json | 827 +++++++++++++++++------------ 2 files changed, 521 insertions(+), 349 deletions(-) diff --git a/scripts/theporndude_curl_triage.py b/scripts/theporndude_curl_triage.py index a19af91..98bfab0 100644 --- a/scripts/theporndude_curl_triage.py +++ b/scripts/theporndude_curl_triage.py @@ -41,16 +41,43 @@ META_MARKERS = [ ] +# Slug w feedzie to slug RECENZJI ("youperv-website-review"), nie domena. Bez odcięcia +# sufiksu fallback budował "youperv-website-review.com" → conn_refused → wpis lądował +# jako "dead/skip". Tak wypadło 15 ze 144 kandydatów, w tym youperv i pornbusy, które +# potem niezależnie oceniliśmy na 4.5-5/5 i dodaliśmy. Fix 2026-07-27. +_REVIEW_SUFFIX_RE = re.compile(r"-(?:website-)?review$", re.IGNORECASE) + + +def _slug_to_domain(slug: str) -> str: + return _REVIEW_SUFFIX_RE.sub("", slug.lower().strip()) + + async def fetch_one(cli: httpx.AsyncClient, url: str) -> tuple[int, str]: + """Fetch z Chrome TLS (curl_cffi) — NIE gołym httpx. + + FIX 2026-07-27: pierwotna wersja szła czystym httpx, którego JA3 fingerprint + większość tubów za Cloudflare odrzuca. Efekt: 8 timeoutów + 4× 403 w wynikach, + czytane potem jako "dead/skip", choć strony działają w przeglądarce. `browser_get` + (curl_cffi, chrome120) przechodzi — to ta sama ścieżka, której używają nasze + scrapery, więc triage mierzy wreszcie to samo co produkcja. + """ + import asyncio as _asyncio + + from app.extractors._fetch import browser_get + + def _sync(): + return browser_get(url, timeout=20.0, headers={"User-Agent": UA}) + try: - r = await cli.get(url, headers={"User-Agent": UA}, follow_redirects=True) - return r.status_code, r.text[:200_000] # cap response - except httpx.ConnectError: - return -1, "conn_refused" - except httpx.TimeoutException: - return -2, "timeout" + r = await _asyncio.to_thread(_sync) + return r.status_code, (r.text or "")[:200_000] except Exception as e: - return -9, str(e)[:120] + name = type(e).__name__ + if "Timeout" in name: + return -2, "timeout" + if "Connect" in name or "Resolve" in name or "DNS" in name: + return -1, "conn_refused" + return -9, f"{name}: {e}"[:120] def analyze_html(html: str) -> dict: @@ -152,7 +179,7 @@ async def main(): domain = r.get("domain") or "" # Jeśli pdude.link daje porndudecams.com (interstitial), użyj .com if not domain or "porndudecams" in domain: - domain = f"{r['slug'].lower()}.com" + domain = f"{_slug_to_domain(r['slug'])}.com" return {**r, **(await audit_one(cli, r["slug"], domain))} results = await asyncio.gather(*[worker(r) for r in new_candidates]) diff --git a/theporndude_triage.json b/theporndude_triage.json index 631446a..9e8c8c9 100644 --- a/theporndude_triage.json +++ b/theporndude_triage.json @@ -16,8 +16,8 @@ "domain": "yourporn.com", "score": null, "final_url": "https://porndudecams.com/?affiliateId=200526k4oihgsf9ibisr5ye1n2t6z2oyy37ofwl867dhq5sspo2mp3yxkpxj0v1v&realDomain=go.rmhfrtnd.com&sourceId=im&userId=c637fd2a4a50870636df00bfc878c986", - "root_status": -1, - "root_error": "conn_refused" + "root_status": -9, + "root_error": "CertificateVerifyError: Failed to perform, curl: (60) SSL certificate problem: s" }, { "rank": 9, @@ -42,9 +42,9 @@ "hls_marker": true, "auth_wall": true, "scene_link_samples": [ - "/videos/pdwshwkovtxikwk32040-121459031.html", - "/videos/stepsister-likes-to-be-nakedkoty-121867771.html", - "/videos/my-wife%26%23039%3bs-best-friend-~lustful-strike-sexual-intercourse-after-rubbing-her-pleasant-body-and-fucking-" + "/videos/nicole-aniston-gets-fizzed-after-sucking-a-cock-and-fucking-125288411.html", + "/videos/stepmom-fucks-son-before-she-goes-on-a-trip-swedish-casting-snapchat-american-82994921.html", + "/videos/step-family-share-a-bed-in-hotel-room%21-step-%27s-first-3some-97824081.html" ], "scene_path_prefixes": [ "/videos" @@ -64,8 +64,8 @@ "domain": "vporn.com", "score": null, "final_url": "https://porndudecams.com/?affiliateId=200526kp8vl9o30evfq17l9qkg9f0hojn17kwl8ujfipeqdzlttye0p9rsc2haj1&realDomain=go.rmhfrtnd.com&sourceId=im&userId=c637fd2a4a50870636df00bfc878c986", - "root_status": -2, - "root_error": "timeout" + "root_status": 522, + "root_error": "\n