Don't merge consecutive equal tokens if the line contains RTL (#257)

Author: alexdima

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "vscode-textmate",
-  "version": "9.2.0",
+  "version": "9.2.1",
   "description": "VSCode TextMate grammar helpers",
   "author": {
     "name": "Microsoft Corporation"

package.json

@@ -10,7 +10,7 @@ import { disposeOnigString, IOnigLib, OnigScanner, OnigString } from '../onigLib
 import { IRawGrammar, IRawRepository, IRawRule } from '../rawGrammar';
 import { ruleIdFromNumber, IRuleFactoryHelper, IRuleRegistry, Rule, RuleFactory, RuleId, ruleIdToNumber } from '../rule';
 import { FontStyle, ScopeName, ScopePath, ScopeStack, StyleAttributes } from '../theme';
-import { clone } from '../utils';
+import { clone, containsRTL } from '../utils';
 import { BasicScopeAttributes, BasicScopeAttributesProvider } from './basicScopesAttributeProvider';
 import { _tokenizeString } from './tokenizeString';
 
@@ -960,6 +960,7 @@ export class LineTokens {
 	private _lastTokenEndIndex: number;
 
 	private readonly _tokenTypeOverrides: TokenTypeMatcher[];
+	private readonly _mergeConsecutiveTokensWithEqualMetadata: boolean;
 
 	constructor(
 		emitBinaryTokens: boolean,
@@ -974,6 +975,8 @@ export class LineTokens {
 		} else {
 			this._lineText = null;
 		}
+		// Don't merge tokens if the line contains RTL characters
+		this._mergeConsecutiveTokensWithEqualMetadata = !containsRTL(lineText);
 		this._tokens = [];
 		this._binaryTokens = [];
 		this._lastTokenEndIndex = 0;
@@ -1031,7 +1034,7 @@ export class LineTokens {
 				);
 			}
 
-			if (this._binaryTokens.length > 0 && this._binaryTokens[this._binaryTokens.length - 1] === metadata) {
+			if (this._mergeConsecutiveTokensWithEqualMetadata && this._binaryTokens.length > 0 && this._binaryTokens[this._binaryTokens.length - 1] === metadata) {
 				// no need to push a token with the same metadata
 				this._lastTokenEndIndex = endIndex;
 				return;

src/grammar/grammar.ts

@@ -184,4 +184,22 @@ export const performanceNow =
 		  }
 		: function () {
 				return performance!.now();
-		  };
+		  };
+
+let CONTAINS_RTL: RegExp | undefined = undefined;
+
+function makeContainsRtl() {
+	// Generated using https://github.com/alexdima/unicode-utils/blob/main/rtl-test.js
+	return /(?:[\u05BE\u05C0\u05C3\u05C6\u05D0-\u05F4\u0608\u060B\u060D\u061B-\u064A\u066D-\u066F\u0671-\u06D5\u06E5\u06E6\u06EE\u06EF\u06FA-\u0710\u0712-\u072F\u074D-\u07A5\u07B1-\u07EA\u07F4\u07F5\u07FA\u07FE-\u0815\u081A\u0824\u0828\u0830-\u0858\u085E-\u088E\u08A0-\u08C9\u200F\uFB1D\uFB1F-\uFB28\uFB2A-\uFD3D\uFD50-\uFDC7\uFDF0-\uFDFC\uFE70-\uFEFC]|\uD802[\uDC00-\uDD1B\uDD20-\uDE00\uDE10-\uDE35\uDE40-\uDEE4\uDEEB-\uDF35\uDF40-\uDFFF]|\uD803[\uDC00-\uDD23\uDE80-\uDEA9\uDEAD-\uDF45\uDF51-\uDF81\uDF86-\uDFF6]|\uD83A[\uDC00-\uDCCF\uDD00-\uDD43\uDD4B-\uDFFF]|\uD83B[\uDC00-\uDEBB])/;
+}
+
+/**
+ * Returns true if `str` contains any Unicode character that is classified as "R" or "AL".
+ */
+export function containsRTL(str: string): boolean {
+	if (!CONTAINS_RTL) {
+		CONTAINS_RTL = makeContainsRtl();
+	}
+
+	return CONTAINS_RTL.test(str);
+}