Return state when tokenizing plaintext syntaxes

This commit is contained in:
Guldoman 2023-07-09 18:59:25 +02:00 committed by George Sokianos
parent 74d9c15736
commit 0ebf3c0393
1 changed files with 4 additions and 4 deletions

View File

@ -133,12 +133,12 @@ function tokenizer.tokenize(incoming_syntax, text, state, resume)
local res
local i = 1
if #incoming_syntax.patterns == 0 then
return { "normal", text }
end
state = state or string.char(0)
if #incoming_syntax.patterns == 0 then
return { "normal", text }, state
end
if resume then
res = resume.res
-- Remove "incomplete" tokens