Fix problem checking utf-8 cont at end of string
This commit is contained in:
parent
ffb66cefd7
commit
5cdd800910
|
@ -145,7 +145,7 @@ function tokenizer.tokenize(incoming_syntax, text, state)
|
||||||
repeat
|
repeat
|
||||||
local next = res[2] + 1
|
local next = res[2] + 1
|
||||||
-- go to the start of the next utf-8 character
|
-- go to the start of the next utf-8 character
|
||||||
while common.is_utf8_cont(text, next) do
|
while text:byte(next) and common.is_utf8_cont(text, next) do
|
||||||
next = next + 1
|
next = next + 1
|
||||||
end
|
end
|
||||||
res = p.pattern and { text:find(at_start and "^" .. code or code, next) }
|
res = p.pattern and { text:find(at_start and "^" .. code or code, next) }
|
||||||
|
|
Loading…
Reference in New Issue