Return state when tokenizing plaintext syntaxes

This commit is contained in:
Guldoman 2023-07-09 18:59:25 +02:00
parent 99d72eaebd
commit 96d6d9acac
No known key found for this signature in database
GPG Key ID: EA928C8BDA1A8825
1 changed files with 4 additions and 4 deletions

View File

@ -133,12 +133,12 @@ function tokenizer.tokenize(incoming_syntax, text, state, resume)
local res
local i = 1
if #incoming_syntax.patterns == 0 then
return { "normal", text }
end
state = state or string.char(0)
if #incoming_syntax.patterns == 0 then
return { "normal", text }, state
end
if resume then
res = resume.res
-- Remove "incomplete" tokens