Merge pull request #725 from Guldoman/fix_consume_no_match

Consume unmatched character correctly
This commit is contained in:
Adam 2021-12-11 15:35:22 -05:00 committed by GitHub
commit 29ece95962
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 7 additions and 2 deletions

View File

@ -237,8 +237,13 @@ function tokenizer.tokenize(incoming_syntax, text, state)
-- consume character if we didn't match -- consume character if we didn't match
if not matched then if not matched then
push_token(res, "normal", text:sub(i, i)) local n = 0
i = i + 1 -- reach the next character
while text:byte(i + n + 1) and common.is_utf8_cont(text, i + n + 1) do
n = n + 1
end
push_token(res, "normal", text:sub(i, i + n))
i = i + n + 1
end end
end end