From 951f0913da8ce19a650218fe0a96c88784903a7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jefferson=20Gonz=C3=A1lez?= Date: Fri, 25 Mar 2022 11:25:32 -0400 Subject: [PATCH] syntax: add pattern to boost tokenizer performance (#896) --- data/core/syntax.lua | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/data/core/syntax.lua b/data/core/syntax.lua index adecd0cd..8ff5062d 100644 --- a/data/core/syntax.lua +++ b/data/core/syntax.lua @@ -7,6 +7,11 @@ local plain_text_syntax = { name = "Plain Text", patterns = {}, symbols = {} } function syntax.add(t) + -- this rule gives us a performance gain for the tokenizer in lines with + -- long amounts of consecutive spaces without affecting other patterns + if t.patterns then + table.insert(t.patterns, 1, { pattern = "%s+", type = "normal" }) + end table.insert(syntax.items, t) end