feat(workers): derive the crawler user agent from the connected browser

The crawler hardcoded a Chrome 124 macOS user agent regardless of the
browser it drives, which contradicts the browser's real version, platform
and client hints. Read the browser's own UA over CDP instead, dropping the
"Headless" marker and applying Chrome's UA reduction.

Author: MohamedBassem

apps/workers/workers/crawler/browser.ts

@@ -23,7 +23,7 @@ import { setUrlHostnameFromResolvedAddress } from "@karakeep/shared/utils/url";
 import { tryCatch } from "@karakeep/shared/tryCatch";
 
 import { loadAutoconsent } from "./autoconsent";
-import { redactUrlCredentials } from "./utils";
+import { normalizeBrowserUserAgent, redactUrlCredentials } from "./utils";
 
 interface Cookie {
   name: string;
@@ -104,6 +104,36 @@ export function getPlaywrightProxyConfig(
   };
 }
 
+const browserUserAgents = new WeakMap<Browser, Promise<string | undefined>>();
+
+/**
+ * Returns the user agent to use for crawl contexts, derived from the connected
+ * browser so it always matches the browser's real version, platform, and
+ * client hints. Resolves to undefined (keep the browser's default) if the
+ * browser can't be queried.
+ */
+export function getBrowserUserAgent(
+  browser: Browser,
+): Promise<string | undefined> {
+  let userAgent = browserUserAgents.get(browser);
+  if (!userAgent) {
+    userAgent = (async () => {
+      const session = await browser.newBrowserCDPSession();
+      try {
+        const version = await session.send("Browser.getVersion");
+        return normalizeBrowserUserAgent(version.userAgent);
+      } finally {
+        await session.detach().catch(() => undefined);
+      }
+    })().catch((e: unknown) => {
+      logger.warn(`[Crawler] Failed to read the browser's user agent: ${e}`);
+      return undefined;
+    });
+    browserUserAgents.set(browser, userAgent);
+  }
+  return userAgent;
+}
+
 /**
  * Reaps browser contexts that have been open longer than the max job timeout.
  * This is a safety net for cases where context.close() hangs or is never called.

apps/workers/workers/crawler/crawlPage.ts

@@ -41,6 +41,7 @@ import {
   PAGE_CLOSE_TIMEOUT_MS,
   getGlobalBlocker,
   getGlobalBrowser,
+  getBrowserUserAgent,
   getGlobalCookies,
   getPlaywrightProxyConfig,
   startBrowserInstance,
@@ -634,6 +635,7 @@ export async function crawlPage(
       }
 
       const proxyConfig = getPlaywrightProxyConfig(runProxy);
+      const userAgent = await getBrowserUserAgent(browser);
       const isRunningInProxyContext =
         proxyConfig !== undefined &&
         !matchesNoProxy(url, proxyConfig.bypass?.split(",") ?? []);
@@ -648,8 +650,7 @@ export async function crawlPage(
         async () =>
           browser.newContext({
             viewport: { width: 1440, height: 900 },
-            userAgent:
-              "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+            userAgent,
             proxy: proxyConfig,
             serviceWorkers: "block",
           }),

apps/workers/workers/crawler/utils.test.ts

@@ -0,0 +1,31 @@
+import { describe, expect, it } from "vitest";
+
+import { normalizeBrowserUserAgent } from "./utils";
+
+describe("normalizeBrowserUserAgent", () => {
+  it("drops the headless marker and reduces the version", () => {
+    expect(
+      normalizeBrowserUserAgent(
+        "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/154.0.8037.57 Safari/537.36",
+      ),
+    ).toBe(
+      "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/154.0.0.0 Safari/537.36",
+    );
+  });
+
+  it("reduces the full version that headless-shell reports", () => {
+    expect(
+      normalizeBrowserUserAgent(
+        "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.7922.47 Safari/537.36",
+      ),
+    ).toBe(
+      "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.0.0 Safari/537.36",
+    );
+  });
+
+  it("keeps an already reduced user agent and its platform unchanged", () => {
+    const ua =
+      "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/154.0.0.0 Safari/537.36";
+    expect(normalizeBrowserUserAgent(ua)).toBe(ua);
+  });
+});

apps/workers/workers/crawler/utils.ts

@@ -40,3 +40,14 @@ export function shouldRetryCrawlStatusCode(statusCode: number | null): boolean {
   }
   return statusCode === 403 || statusCode === 429 || statusCode >= 500;
 }
+
+/**
+ * Turns the browser's own user agent into the one a regular (non-headless)
+ * Chrome of the same version and platform would send: drops the "Headless"
+ * marker and applies Chrome's user-agent reduction (minor version zeroed).
+ */
+export function normalizeBrowserUserAgent(userAgent: string): string {
+  return userAgent
+    .replace("HeadlessChrome/", "Chrome/")
+    .replace(/Chrome\/(\d+)\.[\d.]+/, "Chrome/$1.0.0.0");
+}