walkero
/
lite-xl
1
0
Fork 0
Code Issues 7 Pull Requests Projects Releases 14 Wiki Activity

Merge pull request #1040 from Guldoman/PR_tokenizer_errors_alert 

Add more tokenizer errors/warnings
This commit is contained in:
Jefferson González 2022-06-22 19:43:51 -04:00 committed by GitHub
parent 522d8a8094 d169619f69
commit 880e6e4f0f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 23 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
data/core/tokenizer.lua
View File

@ -95,6 +95,16 @@ local function retrieve_syntax_state(incoming_syntax, state)
return current_syntax, subsyntax_info, current_pattern_idx, current_level return current_syntax, subsyntax_info, current_pattern_idx, current_level
end end
local function report_bad_pattern(log_fn, syntax, pattern_idx, msg, ...)
if not bad_patterns[syntax] then
bad_patterns[syntax] = { }
end
if bad_patterns[syntax][pattern_idx] then return end
bad_patterns[syntax][pattern_idx] = true
log_fn("Malformed pattern #%d in %s language plugin. " .. msg,
pattern_idx, syntax.name or "unnamed", ...)
end
---@param incoming_syntax table ---@param incoming_syntax table
---@param text string ---@param text string
---@param state integer ---@param state integer
@ -267,16 +277,20 @@ function tokenizer.tokenize(incoming_syntax, text, state)
local matched = false local matched = false
for n, p in ipairs(current_syntax.patterns) do for n, p in ipairs(current_syntax.patterns) do
local find_results = { find_text(text, p, i, true, false) } local find_results = { find_text(text, p, i, true, false) }
if #find_results - 1 > #p.type then
if not bad_patterns[current_syntax] then
bad_patterns[current_syntax] = { }
end
if not bad_patterns[current_syntax][n] then
bad_patterns[current_syntax][n] = true
core.error("Malformed pattern #%d in %s language plugin", n, current_syntax.name or "unnamed")
end
end
if find_results[1] then if find_results[1] then
local type_is_table = type(p.type) == "table"
local n_types = type_is_table and #p.type or 1
if #find_results == 2 and type_is_table then
report_bad_pattern(core.warn, current_syntax, n,
"Token type is a table, but a string was expected.")
p.type = p.type[1]
elseif #find_results - 1 > n_types then
report_bad_pattern(core.error, current_syntax, n,
"Not enough token types: got %d needed %d.", n_types, #find_results - 1)
elseif #find_results - 1 < n_types then
report_bad_pattern(core.warn, current_syntax, n,
"Too many token types: got %d needed %d.", n_types, #find_results - 1)
end
-- matched pattern; make and add tokens -- matched pattern; make and add tokens
push_tokens(res, current_syntax, p, text, find_results) push_tokens(res, current_syntax, p, text, find_results)
-- update state if this was a start|end pattern pair -- update state if this was a start|end pattern pair