perf(engine-js): reuse each regex's last search while tokenizing a line (#1314)

Co-authored-by: Anthony Fu <github@antfu.me>

Author: issacgerges

docs/references/engine-js-compat.md

@@ -2,9 +2,9 @@
 
 Compatibility reference of all built-in grammars with the [JavaScript RegExp engine](/guide/regex-engines#javascript-regexp-engine).
 
-> Generated on Friday, July 31, 2026
+> Generated on Friday, September 11, 2026
 >
-> Version `4.3.1`
+> Version `4.4.3`
 >
 > Runtime: Node.js v24.16.0
 
@@ -239,7 +239,7 @@ In some edge cases, it's not guaranteed that the highlighting will be 100% the s
 | twig               | ✅ OK           |              2445 |               - |      |
 | typescript         | ✅ OK           |               362 |               - |      |
 | typespec           | ✅ OK           |                73 |               - |      |
-| typst              | ✅ OK           |                78 |               - |      |
+| typst              | ✅ OK           |              5699 |               - |      |
 | v                  | ✅ OK           |                77 |               - |      |
 | vala               | ✅ OK           |                20 |               - |      |
 | vb                 | ✅ OK           |                34 |               - |      |

packages/engine-javascript/src/engine-compile.ts

@@ -3,6 +3,7 @@ import type { ToRegExpOptions } from 'oniguruma-to-es'
 import type { JavaScriptRegexScannerOptions } from './scanner'
 import { toRegExp } from 'oniguruma-to-es'
 import { JavaScriptScanner } from './scanner'
+import { createJavaScriptEngineString } from './string'
 
 export interface JavaScriptRegexEngineOptions extends JavaScriptRegexScannerOptions {
   /**
@@ -73,10 +74,6 @@ export function createJavaScriptRegexEngine(options: JavaScriptRegexEngineOption
     createScanner(patterns) {
       return new JavaScriptScanner(patterns, _options)
     },
-    createString(s: string) {
-      return {
-        content: s,
-      }
-    },
+    createString: createJavaScriptEngineString,
   }
 }

packages/engine-javascript/src/engine-raw.ts

@@ -1,6 +1,7 @@
 import type { RegexEngine } from '@shikijs/types'
 import type { JavaScriptRegexScannerOptions } from './scanner'
 import { JavaScriptScanner } from './scanner'
+import { createJavaScriptEngineString } from './string'
 
 /**
  * Raw JavaScript regex engine that only supports precompiled grammars.
@@ -21,10 +22,6 @@ export function createJavaScriptRawEngine(): RegexEngine {
     createScanner(patterns) {
       return new JavaScriptScanner(patterns, options)
     },
-    createString(s: string) {
-      return {
-        content: s,
-      }
-    },
+    createString: createJavaScriptEngineString,
   }
 }

packages/engine-javascript/src/scanner.ts

@@ -3,6 +3,7 @@ import type {
   RegexEngineString,
 } from '@shikijs/types'
 import type { IOnigMatch } from '@shikijs/vscode-textmate'
+import type { JavaScriptEngineString } from './string'
 
 const MAX = 4294967295
 
@@ -27,9 +28,48 @@ export interface JavaScriptRegexScannerOptions {
   regexConstructor?: (pattern: string) => RegExp
 }
 
+interface LastSearch {
+  strId: number
+  searchedFrom: number
+  matchIndex: number // -1 for no match
+  indices: RegExpIndicesArray | null
+}
+
+// Sticky regexps and `EmulatedRegExp`s with a search strategy (how `oniguruma-to-es` emulates `\G`)
+// depend on the exact start position, so their searches can't be reused
+function isSearchCacheable(regexp: RegExp): boolean {
+  return !regexp.sticky && !(regexp as { rawOptions?: { strategy?: string | null } }).rawOptions?.strategy
+}
+
+function toResult(index: number, indices: RegExpIndicesArray): IOnigMatch {
+  return {
+    index,
+    captureIndices: indices.map((indice) => {
+      if (indice == null) {
+        return {
+          start: MAX,
+          end: MAX,
+          length: 0,
+        }
+      }
+      return {
+        start: indice[0],
+        end: indice[1],
+        length: indice[1] - indice[0],
+      }
+    }),
+  }
+}
+
 export class JavaScriptScanner implements PatternScanner {
   regexps: (RegExp | null)[]
 
+  /**
+   * Last search of each regexp on the current string, reused the way `vscode-oniguruma` does.
+   * `null` for regexps whose searches can't be reused.
+   */
+  private lastSearches: (LastSearch | null)[]
+
   constructor(
     public patterns: (string | RegExp)[],
     public options: JavaScriptRegexScannerOptions = {},
@@ -71,51 +111,66 @@ export class JavaScriptScanner implements PatternScanner {
         throw e
       }
     })
+
+    this.lastSearches = this.regexps.map(regexp => regexp && isSearchCacheable(regexp)
+      ? { strId: 0, searchedFrom: 0, matchIndex: -1, indices: null }
+      : null,
+    )
   }
 
   findNextMatchSync(string: string | RegexEngineString, startPosition: number, _options: number): IOnigMatch | null {
     const str = typeof string === 'string'
       ? string
       : string.content
-    const pending: [index: number, match: RegExpExecArray, offset: number][] = []
+    // No id (plain string or a string from another engine): never reuse
+    const strId = typeof string === 'string'
+      ? 0
+      : (string as Partial<JavaScriptEngineString>).id || 0
 
-    function toResult(index: number, match: RegExpExecArray, offset = 0): IOnigMatch {
-      return {
-        index,
-        captureIndices: match.indices!.map((indice) => {
-          if (indice == null) {
-            return {
-              start: MAX,
-              end: MAX,
-              length: 0,
-            }
-          }
-          return {
-            start: indice[0] + offset,
-            end: indice[1] + offset,
-            length: indice[1] - indice[0],
-          }
-        }),
-      }
-    }
+    let bestIndex = -1
+    let bestMatchIndex = 0
+    let bestIndices: RegExpIndicesArray | null = null
 
     for (let i = 0; i < this.regexps.length; i++) {
       const regexp = this.regexps[i]
       if (!regexp)
         continue
       try {
-        regexp.lastIndex = startPosition
-        const match = regexp.exec(str)
+        const last = strId ? this.lastSearches[i] : null
+        let matchIndex: number
+        let indices: RegExpIndicesArray | null
+
+        // Still valid if it found nothing, or a match at or after the start position
+        if (last && last.strId === strId && last.searchedFrom <= startPosition && (last.matchIndex === -1 || last.matchIndex >= startPosition)) {
+          matchIndex = last.matchIndex
+          indices = last.indices
+        }
+        else {
+          regexp.lastIndex = startPosition
+          const match = regexp.exec(str)
+          matchIndex = match ? match.index : -1
+          indices = match ? match.indices! : null
+          if (last) {
+            last.strId = strId
+            last.searchedFrom = startPosition
+            last.matchIndex = matchIndex
+            last.indices = indices
+          }
+        }
 
-        if (!match)
+        if (matchIndex === -1)
           continue
 
         // If the match is at the start position, return it immediately
-        if (match.index === startPosition) {
-          return toResult(i, match, 0)
+        if (matchIndex === startPosition) {
+          return toResult(i, indices!)
+        }
+        // Otherwise, keep the closest one
+        if (bestIndex === -1 || matchIndex < bestMatchIndex) {
+          bestIndex = i
+          bestMatchIndex = matchIndex
+          bestIndices = indices
         }
-        // Otherwise, store it for later
-        pending.push([i, match, 0])
       }
       catch (e) {
         if (this.options.forgiving)
@@ -124,16 +179,8 @@ export class JavaScriptScanner implements PatternScanner {
       }
     }
 
-    // Find the closest match to the start position
-    if (pending.length) {
-      const minIndex = Math.min(...pending.map(m => m[1].index))
-      for (const [i, match, offset] of pending) {
-        if (match.index === minIndex) {
-          return toResult(i, match, offset)
-        }
-      }
-    }
-
-    return null
+    return bestIndex === -1
+      ? null
+      : toResult(bestIndex, bestIndices!)
   }
 }

packages/engine-javascript/src/string.ts

@@ -0,0 +1,17 @@
+import type { RegexEngineString } from '@shikijs/types'
+
+/**
+ * String with an `id`, so scanners can reuse searches on it (like `OnigString`).
+ */
+export interface JavaScriptEngineString extends RegexEngineString {
+  readonly id: number
+}
+
+let lastStringId = 0
+
+export function createJavaScriptEngineString(content: string): JavaScriptEngineString {
+  return {
+    content,
+    id: ++lastStringId,
+  }
+}