diff --git a/docs/guide/regex-engines.md b/docs/guide/regex-engines.md index 42612758..072b0039 100644 --- a/docs/guide/regex-engines.md +++ b/docs/guide/regex-engines.md @@ -56,8 +56,6 @@ const html = shiki.codeToHtml('const a = 1', { lang: 'javascript', theme: 'nord' The advantages of using the JavaScript engine are that it doesn't require loading a large WebAssembly file for Oniguruma and it is faster for some grammars (since the regular expressions run as native JavaScript). -Although the JavaScript engine's Oniguruma emulation is quite robust, it's not guaranteed that the highlighting will be 100% the same in some edge cases. Additionally, a few grammars are unsupported. - Please check the [compatibility table](/references/engine-js-compat) for the support status of languages you are using. The JavaScript engine is strict by default, and will throw an error if it encounters a pattern that it cannot convert. If mismatches are acceptable and you want best-effort results for unsupported grammars, you can enable the `forgiving` option to suppress any conversion errors: @@ -68,7 +66,7 @@ const jsEngine = createJavaScriptRegexEngine({ forgiving: true }) ``` ::: info -If you run Shiki on Node.js (or at build time) and bundle size or WebAssembly support is not a concern, we still recommend using the Oniguruma engine for best results. +If you run Shiki on Node.js (or at build time) and bundle size or WebAssembly support is not a concern, we still recommend using the Oniguruma engine. The JavaScript engine is best when running in the browser and in cases when you want to control the bundle size. ::: diff --git a/docs/references/engine-js-compat.md b/docs/references/engine-js-compat.md index 0f6d9a42..85c19bcb 100644 --- a/docs/references/engine-js-compat.md +++ b/docs/references/engine-js-compat.md @@ -2,19 +2,19 @@ Compatibility reference of all built-in grammars with the [JavaScript RegExp engine](/guide/regex-engines#javascript-regexp-engine). -> Generated on Monday, January 13, 2025 +> Generated on Tuesday, January 14, 2025 > -> Version `1.26.1` +> Version `1.27.0` > -> Runtime: Node.js v22.12.0 +> Runtime: Node.js v22.11.0 ## Report Summary | | Count | | :-------------- | --------------------------: | | Total Languages | 219 | -| Supported | [212](#supported-languages) | -| Mismatched | [1](#mismatched-languages) | +| Supported | [213](#supported-languages) | +| Mismatched | [0](#mismatched-languages) | | Unsupported | [6](#unsupported-languages) | ## Supported Languages @@ -147,6 +147,7 @@ In some edge cases, it's not guaranteed that the highlighting will be 100% the s | move | ✅ OK | 120 | - | | | narrat | ✅ OK | 34 | - | | | nextflow | ✅ OK | 32 | - | | +| nginx | ✅ OK | 378 | - | | | nim | ✅ OK | 1130 | - | | | nix | ✅ OK | 80 | - | | | nushell | ✅ OK | 81 | - | | @@ -204,7 +205,7 @@ In some edge cases, it's not guaranteed that the highlighting will be 100% the s | talonscript | ✅ OK | 44 | - | | | tasl | ✅ OK | 23 | - | | | tcl | ✅ OK | 34 | - | | -| templ | ✅ OK | 680 | - | | +| templ | ✅ OK | 682 | - | | | terraform | ✅ OK | 68 | - | | | tex | ✅ OK | 106 | - | | | toml | ✅ OK | 44 | - | | @@ -246,11 +247,10 @@ In some edge cases, it's not guaranteed that the highlighting will be 100% the s ## Mismatched Languages -Languages that do not throw with the JavaScript RegExp engine, but will produce different results than the WASM engine. Please use with caution. +Languages that do not throw with the JavaScript RegExp engine, but will produce different results than the WASM engine. -| Language | Highlight Match | Patterns Parsable | Patterns Failed | Diff | -| -------- | :------------------------------------------------------------------------- | ----------------: | --------------: | ---: | -| nginx | [🚧 Mismatch](https://textmate-grammars-themes.netlify.app/?grammar=nginx) | 378 | - | 122 | +| Language | Highlight Match | Patterns Parsable | Patterns Failed | Diff | +| -------- | :-------------- | ----------------: | --------------: | ---: | ## Unsupported Languages @@ -262,5 +262,5 @@ Languages that throw with the JavaScript RegExp engine, either because they cont | sass | ✅ OK | 67 | 2 | | | purescript | ❌ Error | 72 | 1 | | | csharp | ❌ Error | 310 | 3 | 137 | -| swift | ❌ Error | 326 | 3 | 8 | +| swift | ❌ Error | 326 | 3 | | | razor | ❌ Error | 957 | 5 | | diff --git a/packages/engine-javascript/src/engine-compile.ts b/packages/engine-javascript/src/engine-compile.ts index 366e82e1..5f46c9cd 100644 --- a/packages/engine-javascript/src/engine-compile.ts +++ b/packages/engine-javascript/src/engine-compile.ts @@ -38,9 +38,6 @@ export function defaultJavaScriptRegexConstructor(pattern: string, options?: Oni asciiWordBoundaries: true, // Follow `vscode-oniguruma` which enables this Oniguruma option by default captureGroup: true, - // Removing `\G` anchors in cases when they're not supported for emulation allows - // supporting more grammars, but also allows some mismatches - ignoreUnsupportedGAnchors: true, // Oniguruma uses depth limit `20`; lowered here to keep regexes shorter and maybe // sometimes faster, but can be increased if issues reported due to low limit recursionLimit: 5, diff --git a/packages/langs-precompiled/scripts/__snapshots__/precompile.test.ts.snap b/packages/langs-precompiled/scripts/__snapshots__/precompile.test.ts.snap index 001e2b8c..aafe5165 100644 --- a/packages/langs-precompiled/scripts/__snapshots__/precompile.test.ts.snap +++ b/packages/langs-precompiled/scripts/__snapshots__/precompile.test.ts.snap @@ -91,7 +91,9 @@ exports[`precompile 1`] = ` ], }, }, - end: /(?<=^|\\n(?!$))(?=\\P{space})|(?!)/dgv, + end: new EmulatedRegExp("(?<=^|\\\\n(?!$))(?=\\\\P{space})|(?!^)", "dgv", { + strategy: "search_start_clip", + }), patterns: [ { begin: /(?<=^|\\n(?!$))([ ]+)(?! )/dgv, @@ -110,7 +112,9 @@ exports[`precompile 1`] = ` beginCaptures: { "1": { name: "punctuation.whitespace.comment.leading.yaml" }, }, - end: new EmulatedRegExp("", "dgv", { strategy: "not_search_start" }), + end: new EmulatedRegExp("(?!^)", "dgv", { + strategy: "search_start_clip", + }), patterns: [ { begin: /#/dgv, @@ -373,7 +377,9 @@ exports[`precompile 1`] = ` node: { patterns: [{ include: "#block-node" }] }, property: { begin: /(?=!|&)/dgv, - end: new EmulatedRegExp("", "dgv", { strategy: "not_search_start" }), + end: new EmulatedRegExp("(?!^)", "dgv", { + strategy: "search_start_clip", + }), name: "meta.property.yaml", patterns: [ { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f3ad500a..e3c00fc3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -148,8 +148,8 @@ catalogs: specifier: ^1.1.4 version: 1.1.4 oniguruma-to-es: - specifier: ^1.0.0 - version: 1.0.0 + specifier: ^2.0.0 + version: 2.0.0 picocolors: specifier: ^1.1.1 version: 1.1.1 @@ -578,7 +578,7 @@ importers: version: 10.0.1 oniguruma-to-es: specifier: 'catalog:' - version: 1.0.0 + version: 2.0.0 packages/engine-oniguruma: dependencies: @@ -610,7 +610,7 @@ importers: version: link:../types oniguruma-to-es: specifier: 'catalog:' - version: 1.0.0 + version: 2.0.0 devDependencies: tm-grammars: specifier: 'catalog:' @@ -4517,8 +4517,8 @@ packages: resolution: {integrity: sha512-VXJjc87FScF88uafS3JllDgvAm+c/Slfz06lorj2uAY34rlUu0Nt+v8wreiImcrgAjjIHp1rXpTDlLOGw29WwQ==} engines: {node: '>=18'} - oniguruma-to-es@1.0.0: - resolution: {integrity: sha512-kihvp0O4lFwf5tZMkfanwQLIZ9ORe9OeOFgZonH0BQeThgwfJiaZFeOfvvJVnJIM9TiVmx0RDD35hUJDR0++rQ==} + oniguruma-to-es@2.0.0: + resolution: {integrity: sha512-pE7+9jQgomy10aK6BJKRNHj1Nth0YLOzb3iRuhlz4gRzNSBSd7hga6U8BE6o0SoSuSkqv+PPtt511Msd1Hkl0w==} optionator@0.9.4: resolution: {integrity: sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==} @@ -10418,7 +10418,7 @@ snapshots: dependencies: mimic-function: 5.0.1 - oniguruma-to-es@1.0.0: + oniguruma-to-es@2.0.0: dependencies: emoji-regex-xs: 1.0.0 regex: 5.1.1 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 356fc787..072ee886 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -56,7 +56,7 @@ catalog: monaco-editor-core: ^0.52.2 ofetch: ^1.4.1 ohash: ^1.1.4 - oniguruma-to-es: ^1.0.0 + oniguruma-to-es: ^2.0.0 picocolors: ^1.1.1 pinia: ^2.3.0 pnpm: ^9.15.4 diff --git a/scripts/report-engine-js-compat.ts b/scripts/report-engine-js-compat.ts index 6df91198..9fa7679d 100644 --- a/scripts/report-engine-js-compat.ts +++ b/scripts/report-engine-js-compat.ts @@ -237,7 +237,7 @@ async function run() { '', '## Mismatched Languages', '', - 'Languages that do not throw with the JavaScript RegExp engine, but will produce different results than the WASM engine. Please use with caution.', + 'Languages that do not throw with the JavaScript RegExp engine, but will produce different results than the WASM engine.', '', createTable(reportMismatch), '',