feat(workers): wait for self-clearing bot challenges before capturing

Cloudflare's "Just a moment..." page and DataDome's device check reload
into the real page on their own when the browser passes, but only after
several seconds; the crawler captured them right after load. When a
challenge is detected, wait up to CRAWLER_CHALLENGE_WAIT_SEC (default 15)
for it to clear, skipping DataDome captchas that need a human. Crawls now
report the final main-document status instead of the challenge's.

Author: MohamedBassem

apps/workers/workers/crawler/challenge.ts

@@ -0,0 +1,97 @@
+// Waits out non-interactive bot-protection challenges (e.g. Cloudflare's
+// "Just a moment..." page or DataDome's device check). These interstitials run
+// some JS and then reload into the real page on their own when the browser
+// passes. Capturing right after the initial load would store the challenge
+// page instead, so when one is detected we give it a bounded amount of time to
+// clear.
+import type { Page } from "patchright";
+import { abortRace, raceWith } from "utils";
+
+import { setSpanAttributes } from "@karakeep/shared-server";
+import serverConfig from "@karakeep/shared/config";
+import logger from "@karakeep/shared/logger";
+
+import {
+  isLikelyChallengePage,
+  isWaitableChallenge,
+} from "../utils/metadataResolver";
+
+// Statuses bot-protection vendors serve their interstitials with.
+const CHALLENGE_STATUS_CODES = new Set([401, 403, 429, 503]);
+
+const POLL_INTERVAL_MS = 500;
+const SETTLE_TIMEOUT_MS = 5000;
+
+async function pageIsWaitableChallenge(page: Page): Promise<boolean> {
+  try {
+    const [title, htmlContent] = await Promise.all([
+      page.title(),
+      page.content(),
+    ]);
+    return isWaitableChallenge(title, htmlContent);
+  } catch {
+    // The page is navigating (typically the challenge reloading into the real
+    // page); treat it as still pending and check again on the next poll.
+    return true;
+  }
+}
+
+/**
+ * If the page looks like a self-clearing bot challenge, waits (up to
+ * CRAWLER_CHALLENGE_WAIT_SEC) for it to go away and for the resulting page to
+ * settle. Returns without waiting for regular pages.
+ */
+export async function waitForChallengeToClear(
+  page: Page,
+  statusCode: number,
+  jobId: string,
+  abortSignal: AbortSignal,
+): Promise<void> {
+  const waitMs = serverConfig.crawler.challengeWaitSec * 1000;
+  if (waitMs <= 0) {
+    return;
+  }
+  // Cheap gate so regular pages don't pay for serializing their DOM.
+  const title = await page.title().catch(() => "");
+  if (
+    !CHALLENGE_STATUS_CODES.has(statusCode) &&
+    !isLikelyChallengePage({ title })
+  ) {
+    return;
+  }
+  if (!(await pageIsWaitableChallenge(page))) {
+    return;
+  }
+
+  logger.info(
+    `[Crawler][${jobId}] The page looks like a bot challenge, waiting up to ${serverConfig.crawler.challengeWaitSec}s for it to clear`,
+  );
+  const startedAt = Date.now();
+  const deadline = startedAt + waitMs;
+  let cleared = false;
+  while (Date.now() < deadline) {
+    await raceWith<unknown>(
+      new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS)),
+      abortRace(abortSignal),
+    );
+    if (!(await pageIsWaitableChallenge(page))) {
+      cleared = true;
+      break;
+    }
+  }
+
+  if (cleared) {
+    await page
+      .waitForLoadState("networkidle", { timeout: SETTLE_TIMEOUT_MS })
+      .catch(() => undefined);
+  }
+  abortSignal.throwIfAborted();
+  const waitedMs = Date.now() - startedAt;
+  setSpanAttributes({
+    "crawler.challenge.cleared": cleared,
+    "crawler.challenge.waitMs": waitedMs,
+  });
+  logger.info(
+    `[Crawler][${jobId}] Bot challenge ${cleared ? "cleared" : "did not clear"} after ${waitedMs}ms`,
+  );
+}

apps/workers/workers/crawler/crawlPage.ts

@@ -47,6 +47,7 @@ import {
   trackContext,
   untrackContext,
 } from "./browser";
+import { waitForChallengeToClear } from "./challenge";
 import { truncateUrl } from "./utils";
 
 const tracer = getTracer("@karakeep/workers");
@@ -695,6 +696,18 @@ export async function crawlPage(
           );
         }
         const targetUrl = navigationValidation.url.toString();
+        // Tracks the status of the latest main-frame document, which differs
+        // from the initial navigation's when a bot challenge clears and
+        // reloads into the real page.
+        let mainDocumentStatus: number | undefined;
+        activePage.on("response", (res) => {
+          if (
+            res.request().isNavigationRequest() &&
+            res.frame() === activePage.mainFrame()
+          ) {
+            mainDocumentStatus = res.status();
+          }
+        });
         logger.info(`[Crawler][${jobId}] Navigating to "${targetUrl}"`);
         const response = await withSpan(
           tracer,
@@ -751,9 +764,21 @@ export async function crawlPage(
           setup.autoconsentEnabled,
           abortSignal,
         );
-
         abortSignal.throwIfAborted();
 
+        await withSpan(
+          tracer,
+          "crawlerWorker.crawlPage.waitForChallenge",
+          { attributes: { "job.id": jobId } },
+          () =>
+            waitForChallengeToClear(
+              activePage,
+              mainDocumentStatus ?? response?.status() ?? 0,
+              jobId,
+              abortSignal,
+            ),
+        );
+
         logger.info(
           `[Crawler][${jobId}] Finished waiting for the page to load.`,
         );
@@ -767,7 +792,7 @@ export async function crawlPage(
 
         return {
           htmlContent,
-          statusCode: response?.status() ?? 0,
+          statusCode: mainDocumentStatus ?? response?.status() ?? 0,
           screenshot,
           pdf,
           url: activePage.url(),

apps/workers/workers/utils/metadataResolver.test.ts

@@ -1,6 +1,10 @@
 import { describe, expect, it } from "vitest";
 
-import { isLikelyChallengePage, resolveMetadata } from "./metadataResolver";
+import {
+  isLikelyChallengePage,
+  isWaitableChallenge,
+  resolveMetadata,
+} from "./metadataResolver";
 
 function meta(fields: Record<string, string | null | undefined>) {
   return {
@@ -139,3 +143,41 @@ describe("isLikelyChallengePage", () => {
     );
   });
 });
+
+describe("isWaitableChallenge", () => {
+  it("waits on Cloudflare's interstitial", () => {
+    expect(
+      isWaitableChallenge(
+        "Just a moment...",
+        "<html><head><title>Just a moment...</title></head><body><script>window._cf_chl_opt={}</script></body></html>",
+      ),
+    ).toBe(true);
+  });
+
+  it("waits on DataDome's device check", () => {
+    expect(
+      isWaitableChallenge(
+        "wsj.com",
+        "<html><body><script>var dd={'rt':'i','cid':'x','host':'geo.captcha-delivery.com'}</script></body></html>",
+      ),
+    ).toBe(true);
+  });
+
+  it("doesn't wait on DataDome's captcha, which needs a human", () => {
+    expect(
+      isWaitableChallenge(
+        "wsj.com",
+        "<html><body><script>var dd={'rt':'c','cid':'x','host':'geo.captcha-delivery.com'}</script></body></html>",
+      ),
+    ).toBe(false);
+  });
+
+  it("doesn't wait on regular pages", () => {
+    expect(
+      isWaitableChallenge(
+        "Example Domain",
+        "<html><head><title>Example Domain</title></head><body><p>Hello</p></body></html>",
+      ),
+    ).toBe(false);
+  });
+});

apps/workers/workers/utils/metadataResolver.ts

@@ -82,6 +82,24 @@ export function isLikelyChallengePage({
   return CHALLENGE_PAGE_BODY_MARKERS.some((marker) => lowered.includes(marker));
 }
 
+// DataDome's captcha page (as opposed to its self-clearing device check) needs
+// a human to solve it, so there's no point waiting for it to clear.
+const DATADOME_HARD_CAPTCHA = /['"]rt['"]\s*:\s*['"]c['"]/;
+
+/**
+ * Whether the page is a bot challenge that may clear by itself (and reload
+ * into the real page) if the browser keeps waiting.
+ */
+export function isWaitableChallenge(
+  title: string,
+  htmlContent: string,
+): boolean {
+  return (
+    isLikelyChallengePage({ title, htmlContent }) &&
+    !DATADOME_HARD_CAPTCHA.test(htmlContent)
+  );
+}
+
 /**
  * Merges the browser render's metadata with the preflight probe's, per field.
  * The render normally wins (it sees the post-JS DOM), with the probe filling

docs/docs/03-configuration/01-environment-variables.md

@@ -149,6 +149,7 @@ Either `OPENAI_API_KEY` or `OLLAMA_BASE_URL` need to be set for automatic taggin
 | CRAWLER_FULL_PAGE_ARCHIVE_MAX_SIZE_MB    | No        | 0          | Maximum size in MB for a stored full-page archive. Archives larger than this are discarded instead of saved, since media-rich pages can produce 1GB+ files that never render. 0 disables the limit. |
 | CRAWLER_JOB_TIMEOUT_SEC                  | No       | 60        | How long to wait for the crawler job to finish before timing out. If you have a slow internet connection or a low powered device, you might want to bump this up a bit                                                                                                                                                                                                        |
 | CRAWLER_NAVIGATE_TIMEOUT_SEC             | No       | 30        | How long to spend navigating to the page (along with its redirects). Increase this if you have a slow internet connection                                                                                                                                                                                                                                                     |
+| CRAWLER_CHALLENGE_WAIT_SEC               | No       | 15        | When a page turns out to be a bot-protection challenge that clears by itself (e.g. Cloudflare's "Just a moment..." page), how long to wait for it to clear before capturing the page. Set to `0` to disable.                                                                                                                                                                  |
 | CRAWLER_BROWSER_TIMEZONE                 | No       | Not set   | IANA timezone (e.g. `America/New_York`) that the crawler's browser reports to websites. Bot-protection services flag browsers whose timezone doesn't match their IP's location, so if you crawl through a proxy, set this to the proxy's location. When not set, the browser uses its own timezone (usually UTC in containers).                                               |
 | CRAWLER_PREFLIGHT_USER_AGENT             | No       | Not set   | The `User-Agent` header to send on the crawler's preflight request (the plain HTTP request that determines a link's content type and extracts early preview metadata before the full crawl). If not set, no `User-Agent` header is sent.                                                                                                                                      |
 | CRAWLER_PARSE_TIMEOUT_SEC                | No       | 60        | How long to wait for the HTML parsing subprocess (metadata extraction + readability) to finish before timing out.                                                                                                                                                                                                                                                             |

packages/shared-server/src/tracingTypes.ts

@@ -33,6 +33,8 @@ export type TracingAttributeKey =
   | "crawler.cleanup.hasPage"
   | "crawler.cleanup.pageClosed"
   | "crawler.cleanup.contextClosed"
+  | "crawler.challenge.cleared"
+  | "crawler.challenge.waitMs"
   // Database attributes
   | "db.system"
   | "db.statement"

packages/shared/config.ts

@@ -126,6 +126,7 @@ const allEnv = z.object({
   BROWSER_COOKIE_PATH: z.string().optional(),
   CRAWLER_JOB_TIMEOUT_SEC: z.coerce.number().default(60),
   CRAWLER_NAVIGATE_TIMEOUT_SEC: z.coerce.number().default(30),
+  CRAWLER_CHALLENGE_WAIT_SEC: z.coerce.number().default(15),
   CRAWLER_BROWSER_TIMEZONE: z
     .string()
     .refine(
@@ -420,6 +421,7 @@ const serverConfigSchema = allEnv.transform((val, ctx) => {
       browserCookiePath: val.BROWSER_COOKIE_PATH,
       jobTimeoutSec: val.CRAWLER_JOB_TIMEOUT_SEC,
       navigateTimeoutSec: val.CRAWLER_NAVIGATE_TIMEOUT_SEC,
+      challengeWaitSec: val.CRAWLER_CHALLENGE_WAIT_SEC,
       browserTimezone: val.CRAWLER_BROWSER_TIMEZONE,
       downloadBannerImage: val.CRAWLER_DOWNLOAD_BANNER_IMAGE,
       storeScreenshot: val.CRAWLER_STORE_SCREENSHOT,