a small fix in case the text is nil

This commit is contained in:
George Sokianos 2023-01-08 20:42:19 +00:00
parent 789004ea2c
commit 0cb20ab7b2
1 changed files with 81 additions and 79 deletions

View File

@ -225,6 +225,7 @@ function tokenizer.tokenize(incoming_syntax, text, state)
end end
local text_len = text:ulen() local text_len = text:ulen()
if text_len ~= nil then
while i <= text_len do while i <= text_len do
-- continue trying to match the end pattern of a pair if we have a state set -- continue trying to match the end pattern of a pair if we have a state set
if current_pattern_idx > 0 then if current_pattern_idx > 0 then
@ -315,6 +316,7 @@ function tokenizer.tokenize(incoming_syntax, text, state)
i = i + 1 i = i + 1
end end
end end
end
return res, state return res, state
end end