Allow `tokenizer` to pause and resume in the middle of a line (#1444)
This commit is contained in:
parent
6deca53303
commit
6d217204f6
|
@ -19,25 +19,34 @@ function Highlighter:start()
|
||||||
if self.running then return end
|
if self.running then return end
|
||||||
self.running = true
|
self.running = true
|
||||||
core.add_thread(function()
|
core.add_thread(function()
|
||||||
while self.first_invalid_line < self.max_wanted_line do
|
while self.first_invalid_line <= self.max_wanted_line do
|
||||||
local max = math.min(self.first_invalid_line + 40, self.max_wanted_line)
|
local max = math.min(self.first_invalid_line + 40, self.max_wanted_line)
|
||||||
local retokenized_from
|
local retokenized_from
|
||||||
for i = self.first_invalid_line, max do
|
for i = self.first_invalid_line, max do
|
||||||
local state = (i > 1) and self.lines[i - 1].state
|
local state = (i > 1) and self.lines[i - 1].state
|
||||||
local line = self.lines[i]
|
local line = self.lines[i]
|
||||||
if not (line and line.init_state == state and line.text == self.doc.lines[i]) then
|
if line and line.resume and (line.init_state ~= state or line.text ~= self.doc.lines[i]) then
|
||||||
|
-- Reset the progress if no longer valid
|
||||||
|
line.resume = nil
|
||||||
|
end
|
||||||
|
if not (line and line.init_state == state and line.text == self.doc.lines[i] and not line.resume) then
|
||||||
retokenized_from = retokenized_from or i
|
retokenized_from = retokenized_from or i
|
||||||
self.lines[i] = self:tokenize_line(i, state)
|
self.lines[i] = self:tokenize_line(i, state, line and line.resume)
|
||||||
|
if self.lines[i].resume then
|
||||||
|
self.first_invalid_line = i
|
||||||
|
goto yield
|
||||||
|
end
|
||||||
elseif retokenized_from then
|
elseif retokenized_from then
|
||||||
self:update_notify(retokenized_from, i - retokenized_from - 1)
|
self:update_notify(retokenized_from, i - retokenized_from - 1)
|
||||||
retokenized_from = nil
|
retokenized_from = nil
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
self.first_invalid_line = max + 1
|
||||||
|
::yield::
|
||||||
if retokenized_from then
|
if retokenized_from then
|
||||||
self:update_notify(retokenized_from, max - retokenized_from)
|
self:update_notify(retokenized_from, max - retokenized_from)
|
||||||
end
|
end
|
||||||
|
|
||||||
self.first_invalid_line = max + 1
|
|
||||||
core.redraw = true
|
core.redraw = true
|
||||||
coroutine.yield()
|
coroutine.yield()
|
||||||
end
|
end
|
||||||
|
@ -48,7 +57,7 @@ end
|
||||||
|
|
||||||
local function set_max_wanted_lines(self, amount)
|
local function set_max_wanted_lines(self, amount)
|
||||||
self.max_wanted_line = amount
|
self.max_wanted_line = amount
|
||||||
if self.first_invalid_line < self.max_wanted_line then
|
if self.first_invalid_line <= self.max_wanted_line then
|
||||||
self:start()
|
self:start()
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
@ -91,11 +100,11 @@ function Highlighter:update_notify(line, n)
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
||||||
function Highlighter:tokenize_line(idx, state)
|
function Highlighter:tokenize_line(idx, state, resume)
|
||||||
local res = {}
|
local res = {}
|
||||||
res.init_state = state
|
res.init_state = state
|
||||||
res.text = self.doc.lines[idx]
|
res.text = self.doc.lines[idx]
|
||||||
res.tokens, res.state = tokenizer.tokenize(self.doc.syntax, res.text, state)
|
res.tokens, res.state, res.resume = tokenizer.tokenize(self.doc.syntax, res.text, state, resume)
|
||||||
return res
|
return res
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
|
@ -1,5 +1,6 @@
|
||||||
local core = require "core"
|
local core = require "core"
|
||||||
local syntax = require "core.syntax"
|
local syntax = require "core.syntax"
|
||||||
|
local config = require "core.config"
|
||||||
|
|
||||||
local tokenizer = {}
|
local tokenizer = {}
|
||||||
local bad_patterns = {}
|
local bad_patterns = {}
|
||||||
|
@ -8,7 +9,7 @@ local function push_token(t, type, text)
|
||||||
type = type or "normal"
|
type = type or "normal"
|
||||||
local prev_type = t[#t-1]
|
local prev_type = t[#t-1]
|
||||||
local prev_text = t[#t]
|
local prev_text = t[#t]
|
||||||
if prev_type and (prev_type == type or prev_text:ufind("^%s*$")) then
|
if prev_type and (prev_type == type or (prev_text:ufind("^%s*$") and type ~= "incomplete")) then
|
||||||
t[#t-1] = type
|
t[#t-1] = type
|
||||||
t[#t] = prev_text .. text
|
t[#t] = prev_text .. text
|
||||||
else
|
else
|
||||||
|
@ -128,8 +129,8 @@ end
|
||||||
---@param incoming_syntax table
|
---@param incoming_syntax table
|
||||||
---@param text string
|
---@param text string
|
||||||
---@param state string
|
---@param state string
|
||||||
function tokenizer.tokenize(incoming_syntax, text, state)
|
function tokenizer.tokenize(incoming_syntax, text, state, resume)
|
||||||
local res = {}
|
local res
|
||||||
local i = 1
|
local i = 1
|
||||||
|
|
||||||
if #incoming_syntax.patterns == 0 then
|
if #incoming_syntax.patterns == 0 then
|
||||||
|
@ -137,6 +138,20 @@ function tokenizer.tokenize(incoming_syntax, text, state)
|
||||||
end
|
end
|
||||||
|
|
||||||
state = state or string.char(0)
|
state = state or string.char(0)
|
||||||
|
|
||||||
|
if resume then
|
||||||
|
res = resume.res
|
||||||
|
-- Remove "incomplete" tokens
|
||||||
|
while res[#res-1] == "incomplete" do
|
||||||
|
table.remove(res, #res)
|
||||||
|
table.remove(res, #res)
|
||||||
|
end
|
||||||
|
i = resume.i
|
||||||
|
state = resume.state
|
||||||
|
end
|
||||||
|
|
||||||
|
res = res or {}
|
||||||
|
|
||||||
-- incoming_syntax : the parent syntax of the file.
|
-- incoming_syntax : the parent syntax of the file.
|
||||||
-- state : a string of bytes representing syntax state (see above)
|
-- state : a string of bytes representing syntax state (see above)
|
||||||
|
|
||||||
|
@ -245,7 +260,22 @@ function tokenizer.tokenize(incoming_syntax, text, state)
|
||||||
end
|
end
|
||||||
|
|
||||||
local text_len = text:ulen()
|
local text_len = text:ulen()
|
||||||
|
local start_time = system.get_time()
|
||||||
|
local starting_i = i
|
||||||
while i <= text_len do
|
while i <= text_len do
|
||||||
|
-- Every 200 chars, check if we're out of time
|
||||||
|
if i - starting_i > 200 then
|
||||||
|
starting_i = i
|
||||||
|
if system.get_time() - start_time > 0.5 / config.fps then
|
||||||
|
-- We're out of time
|
||||||
|
push_token(res, "incomplete", string.usub(text, i))
|
||||||
|
return res, string.char(0), {
|
||||||
|
res = res,
|
||||||
|
i = i,
|
||||||
|
state = state
|
||||||
|
}
|
||||||
|
end
|
||||||
|
end
|
||||||
-- continue trying to match the end pattern of a pair if we have a state set
|
-- continue trying to match the end pattern of a pair if we have a state set
|
||||||
if current_pattern_idx > 0 then
|
if current_pattern_idx > 0 then
|
||||||
local p = current_syntax.patterns[current_pattern_idx]
|
local p = current_syntax.patterns[current_pattern_idx]
|
||||||
|
|
Loading…
Reference in New Issue