walkero
/
lite-xl
1
0
Fork 0
Code Issues 7 Pull Requests Projects Releases 14 Wiki Activity

Show error if language plugin pattern has mismatching number of groups 

The number of results from a pattern with groups must never be greater
than the number of token types for that pattern.

Also if a token type was undefined, it's now pushed as a `normal` one.
This commit is contained in:
Guldoman 2022-05-31 02:03:42 +02:00
parent 7ac776bef6
commit d8efb1ab53
No known key found for this signature in database
GPG Key ID: EA928C8BDA1A8825
1 changed files with 12 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
data/core/tokenizer.lua
View File

@ -1,9 +1,12 @@
local core = require "core"
local syntax = require "core.syntax" local syntax = require "core.syntax"
local common = require "core.common" local common = require "core.common"
local tokenizer = {} local tokenizer = {}
local bad_patterns = {}
local function push_token(t, type, text) local function push_token(t, type, text)
type = type or "normal"
local prev_type = t[#t-1] local prev_type = t[#t-1]
local prev_text = t[#t] local prev_text = t[#t]
if prev_type and (prev_type == type or prev_text:ufind("^%s*$")) then if prev_type and (prev_type == type or prev_text:ufind("^%s*$")) then
@ -256,6 +259,15 @@ function tokenizer.tokenize(incoming_syntax, text, state)
local matched = false local matched = false
for n, p in ipairs(current_syntax.patterns) do for n, p in ipairs(current_syntax.patterns) do
local find_results = { find_text(text, p, i, true, false) } local find_results = { find_text(text, p, i, true, false) }
if #find_results - 1 > #p.type then
if not bad_patterns[current_syntax] then
bad_patterns[current_syntax] = { }
end
if not bad_patterns[current_syntax][n] then
bad_patterns[current_syntax][n] = true
core.error("Malformed pattern #%d in %s language plugin", n, current_syntax.name or "unnamed")
end
end
if find_results[1] then if find_results[1] then
-- matched pattern; make and add tokens -- matched pattern; make and add tokens
push_tokens(res, current_syntax, p, text, find_results) push_tokens(res, current_syntax, p, text, find_results)