feat(workers): drive the crawler browser with patchright

Playwright always enables the CDP Runtime domain, which bot-protection
scripts detect. patchright is a maintained drop-in Playwright fork that
avoids it. The stale puppeteer stealth plugin is dropped: its evasions
contradicted each other (Windows UA, Linux platform, Mac GPU) and differed
between pages and workers.

patchright's page.evaluate defaults to an isolated world, so autoconsent's
lifecycle is now read through waitForFunction, which runs in the page's
main world under both Playwright and patchright.

Requires the new karakeep-chrome image: on the old headless-shell image,
dropping the stealth plugin regresses PerimeterX-protected sites.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

Author: MohamedBassem

apps/workers/package.json

@@ -48,13 +48,11 @@
     "metascraper-youtube": "^5.58.1",
     "node-cron": "^3.0.3",
     "node-fetch": "^3.3.2",
+    "patchright": "1.59.4",
     "pdf2json": "^3.1.5",
     "pdf2pic": "^3.1.3",
     "pdfjs-dist": "^4.2.67",
-    "playwright": "^1.59.1",
-    "playwright-extra": "^4.3.6",
     "prom-client": "^15.1.3",
-    "puppeteer-extra-plugin-stealth": "^2.11.2",
     "rss-parser": "^3.13.0",
     "tesseract.js": "^7.0.0",
     "tsx": "^4.23.13",

apps/workers/tsconfig.json

@@ -5,6 +5,11 @@
   "exclude": ["node_modules"],
   "compilerOptions": {
     "baseUrl": ".",
-    "tsBuildInfoFile": "node_modules/.cache/tsbuildinfo.json"
+    "tsBuildInfoFile": "node_modules/.cache/tsbuildinfo.json",
+    // @ghostery/adblocker-playwright only imports "playwright" for its type
+    // declarations; point them at patchright (same API) so it accepts our pages.
+    "paths": {
+      "playwright": ["../../node_modules/patchright"]
+    }
   }
 }

apps/workers/workers/crawler/autoconsent.ts

@@ -12,7 +12,7 @@ import { readFileSync } from "node:fs";
 import { createRequire } from "node:module";
 import * as path from "node:path";
 import { setTimeout as sleep } from "node:timers/promises";
-import type { Page } from "playwright";
+import type { Page } from "patchright";
 import { abortRaceResolve, raceWith } from "utils";
 
 import serverConfig from "@karakeep/shared/config";
@@ -108,24 +108,29 @@ export async function installAutoconsent(
 
 /**
  * Waits (capped at AUTOCONSENT_WAIT_MS, abort-aware) until the main frame's
- * autoconsent lifecycle leaves `pendingStates`.
+ * autoconsent lifecycle leaves `pendingStates`, and returns that lifecycle
+ * (undefined on timeout/abort). Reads it through waitForFunction, which runs in
+ * the page's main world (where the bundle lives) in both Playwright and
+ * patchright; patchright's page.evaluate defaults to an isolated world.
  */
 async function waitWhileIn(
   page: Page,
   pendingStates: string[],
   abortSignal: AbortSignal,
-): Promise<void> {
-  await raceWith<unknown>(
+): Promise<string | undefined> {
+  return await raceWith<string | undefined>(
     page
       .waitForFunction(
-        (pending) =>
-          !pending.includes(
+        (pending) => {
+          const lifecycle =
             globalThis.autoconsentStandalone?.instance.state.lifecycle ??
-              "loading",
-          ),
+            "loading";
+          return pending.includes(lifecycle) ? false : lifecycle;
+        },
         pendingStates,
         { timeout: AUTOCONSENT_WAIT_MS, polling: 100 },
       )
+      .then(async (handle) => (await handle.jsonValue()) || undefined)
       .catch(() => undefined),
     abortRaceResolve(abortSignal, undefined),
   );
@@ -152,11 +157,7 @@ export async function waitForPageLoadAndAutoconsent(
     return;
   }
   await Promise.all([pageLoad, waitWhileIn(page, PENDING_STATES, abortSignal)]);
-  await waitWhileIn(page, OPTING_OUT_STATES, abortSignal);
-
-  const lifecycle = await page
-    .evaluate(() => globalThis.autoconsentStandalone?.instance.state.lifecycle)
-    .catch(() => undefined);
+  const lifecycle = await waitWhileIn(page, OPTING_OUT_STATES, abortSignal);
   if (lifecycle && OPTED_OUT_STATES.includes(lifecycle)) {
     await sleep(DISMISS_SETTLE_MS, undefined, { signal: abortSignal }).catch(
       () => undefined,

apps/workers/workers/crawler/browser.ts

@@ -11,9 +11,14 @@ import { Mutex } from "async-mutex";
 import { exitAbortController } from "exit";
 import { fetchWithProxy } from "network";
 import type { RunProxyConfig } from "network";
-import { Browser, BrowserContext, BrowserContextOptions } from "playwright";
-import { chromium } from "playwright-extra";
-import StealthPlugin from "puppeteer-extra-plugin-stealth";
+// patchright is a drop-in Playwright fork that drives Chrome without enabling
+// the CDP Runtime domain, the most widely checked automation signal.
+import {
+  Browser,
+  BrowserContext,
+  BrowserContextOptions,
+  chromium,
+} from "patchright";
 import { raceWith, timeoutRace } from "utils";
 import { z } from "zod";
 
@@ -297,12 +302,11 @@ async function loadCookiesFromFile(): Promise<void> {
 }
 
 /**
- * One-time setup of the crawler's browser environment: stealth plugin,
- * adblocker, the shared browser connection (unless connecting on demand),
- * cookies, and the stale-context reaper.
+ * One-time setup of the crawler's browser environment: adblocker, the shared
+ * browser connection (unless connecting on demand), cookies, and the
+ * stale-context reaper.
  */
 export async function initializeBrowserEnvironment(): Promise<void> {
-  chromium.use(StealthPlugin());
   await loadAdblocker();
   loadAutoconsent();
   if (!serverConfig.crawler.browserConnectOnDemand) {

apps/workers/workers/crawler/crawlPage.ts

@@ -17,7 +17,7 @@ import {
   BrowserContextOptions,
   CDPSession,
   Page,
-} from "playwright";
+} from "patchright";
 import { abortRace, abortRaceResolve, raceWith, timeoutRace } from "utils";
 
 import { db } from "@karakeep/db";