data/core/tokenizer.lua

@@ -225,7 +225,6 @@ function tokenizer.tokenize(incoming_syntax, text, state)
   end
 
   local text_len = text:ulen()
-  if text_len ~= nil then
   while i <= text_len do
     -- continue trying to match the end pattern of a pair if we have a state set
     if current_pattern_idx > 0 then
@@ -316,7 +315,6 @@ function tokenizer.tokenize(incoming_syntax, text, state)
       i = i + 1
     end
   end
-  end
 
   return res, state
 end