From 523e62bdce675a167bc7d47393a160fe25c03ca7 Mon Sep 17 00:00:00 2001
From: Guldoman <giulio.lettieri@gmail.com>
Date: Sun, 9 Jul 2023 18:59:25 +0200
Subject: [PATCH] Return state when tokenizing plaintext syntaxes

---
 data/core/tokenizer.lua | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/data/core/tokenizer.lua b/data/core/tokenizer.lua
index d32d7d70..6db47ff6 100644
--- a/data/core/tokenizer.lua
+++ b/data/core/tokenizer.lua
@@ -133,12 +133,12 @@ function tokenizer.tokenize(incoming_syntax, text, state, resume)
   local res
   local i = 1
 
-  if #incoming_syntax.patterns == 0 then
-    return { "normal", text }
-  end
-
   state = state or string.char(0)
 
+  if #incoming_syntax.patterns == 0 then
+    return { "normal", text }, state
+  end
+
   if resume then
     res = resume.res
     -- Remove "incomplete" tokens