walkero
/
lite
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Moved highlighter code from `DocView` to `Doc` 

* Only one highlighter state is kept per-document as opposed
  to one per-docview
* Fixes a bug with retaining older highlighter state as a
  DocView wasn't able to detect lines changing above it's viewport
* Renames `highlighter` module to more descriptive `tokenizer`
This commit is contained in:
rxi 2020-05-07 21:14:46 +01:00
parent ae42176953
commit f5025efbb8
4 changed files with 102 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

85
data/core/doc/highlighter.lua Normal file
View File

@ -0,0 +1,85 @@
local core = require "core"
local syntax = require "core.syntax"
local config = require "core.config"
local tokenizer = require "core.tokenizer"
local Object = require "core.object"
local Highlighter = Object:extend()
function Highlighter:new(doc)
self.doc = doc
self:reset_syntax()
-- init incremental syntax highlighting
core.add_thread(function()
while true do
if self.last_valid_line > self.max_wanted_line then
self.max_wanted_line = 0
coroutine.yield(1 / config.fps)
else
local max = math.min(self.last_valid_line + 40, self.max_wanted_line)
for i = self.last_valid_line, max do
local state = (i > 1) and self.lines[i - 1].state
local line = self.lines[i]
if not (line and line.init_state == state) then
self.lines[i] = self:tokenize_line(i, state)
end
end
self.last_valid_line = max + 1
core.redraw = true
coroutine.yield()
end
end
end, self)
end
function Highlighter:reset_syntax()
local syn = syntax.get(self.doc.filename or "")
if self.syntax ~= syn then
self.syntax = syn
self.lines = {}
self.last_valid_line = 1
self.max_wanted_line = 0
end
end
function Highlighter:invalidate(idx)
self.last_valid_line = idx
end
function Highlighter:tokenize_line(idx, state)
local line = {}
line.init_state = state
line.text = self.doc.lines[idx]
line.tokens, line.state = tokenizer.tokenize(self.syntax, line.text, state)
return line
end
function Highlighter:get_line(idx)
local line = self.lines[idx]
if not line or line.text ~= self.doc.lines[idx] then
local prev = self.lines[idx - 1]
line = self:tokenize_line(idx, prev and prev.state)
self.lines[idx] = line
self.last_valid_line = math.min(self.last_valid_line, idx)
end
self.max_wanted_line = math.max(self.max_wanted_line, idx)
return line
end
function Highlighter:each_token(idx)
return tokenizer.each_token(self:get_line(idx).tokens)
end
return Highlighter

11
data/core/doc/init.lua
View File

@ -1,4 +1,5 @@
local Object = require "core.object"
local Highlighter = require "core.doc.highlighter"
local config = require "core.config"
local common = require "core.common"
@ -19,7 +20,6 @@ local function splice(t, at, remove, insert)
insert = insert or {}
local offset = #insert - remove
local old_len = #t
local new_len = old_len + offset
if offset < 0 then
for i = at - offset, old_len - offset do
t[i + offset] = t[i]
@ -49,6 +49,7 @@ function Doc:reset()
self.undo_stack = { idx = 1 }
self.redo_stack = { idx = 1 }
self.clean_change_id = 1
self.highlighter = Highlighter(self)
end
@ -68,6 +69,7 @@ function Doc:load(filename)
table.insert(self.lines, "\n")
end
fp:close()
self.highlighter:reset_syntax()
end
@ -80,6 +82,7 @@ function Doc:save(filename)
end
fp:close()
self.filename = filename or self.filename
self.highlighter:reset_syntax()
self:clean()
end
@ -233,6 +236,9 @@ local function insert(self, undo_stack, time, line, col, text)
local line2, col2 = self:position_offset(line, col, #text)
push_undo(self, undo_stack, time, "selection", self:get_selection())
push_undo(self, undo_stack, time, "remove", line, col, line2, col2)
-- update highlighter
self.highlighter:invalidate(line)
end
@ -252,6 +258,9 @@ local function remove(self, undo_stack, time, line1, col1, line2, col2)
-- splice line into line array
splice(self.lines, line1, line2 - line1 + 1, { before .. after })
-- update highlighter
self.highlighter:invalidate(line1)
end

74
data/core/docview.lua
View File

@ -5,7 +5,6 @@ local style = require "core.style"
local syntax = require "core.syntax"
local translate = require "core.doc.translate"
local View = require "core.view"
local highlighter = require "core.highlighter"
local DocView = View:extend()
@ -51,15 +50,6 @@ DocView.translate = {
local blink_period = 0.8
local function reset_syntax(self)
local syn = syntax.get(self.doc.filename or "")
if self.syntax ~= syn then
self.syntax = syn
self.cache = { last_valid = 1 }
end
end
function DocView:new(doc)
DocView.super.new(self)
self.cursor = "ibeam"
@ -68,32 +58,6 @@ function DocView:new(doc)
self.font = "code_font"
self.last_x_offset = {}
self.blink_timer = 0
reset_syntax(self)
-- init thread for incremental highlighting
self.updated_highlighting = false
core.add_thread(function()
while true do
local _, max = self:get_visible_line_range()
if self.cache.last_valid > max then
coroutine.yield(1 / config.fps)
else
max = math.min(self.cache.last_valid + 20, max)
for i = self.cache.last_valid, max do
local state = (i > 1) and self.cache[i - 1].state
local cl = self.cache[i]
if not (cl and cl.init_state == state) then
self.cache[i] = self:tokenize_line(i, state)
end
end
self.cache.last_valid = max + 1
self.updated_highlighting = true
coroutine.yield()
end
end
end, self)
end
@ -131,27 +95,6 @@ function DocView:get_scrollable_size()
end
function DocView:tokenize_line(idx, state)
local cl = {}
cl.init_state = state
cl.text = self.doc.lines[idx]
cl.tokens, cl.state = highlighter.tokenize(self.syntax, cl.text, state)
return cl
end
function DocView:get_cached_line(idx)
local cl = self.cache[idx]
if not cl or cl.text ~= self.doc.lines[idx] then
local prev = self.cache[idx-1]
cl = self:tokenize_line(idx, prev and prev.state)
self.cache[idx] = cl
self.cache.last_valid = math.min(self.cache.last_valid, idx)
end
return cl
end
function DocView:get_font()
return style[self.font]
end
@ -308,16 +251,6 @@ function DocView:update()
self.last_line, self.last_col = line, col
end
if self.updated_highlighting then
self.updated_highlighting = false
core.redraw = true
end
if self.doc.filename ~= self.last_filename then
reset_syntax(self)
self.last_filename = self.doc.filename
end
-- update blink timer
if self == core.active_view and not self.mouse_selecting then
local n = blink_period / 2
@ -339,10 +272,9 @@ end
function DocView:draw_line_text(idx, x, y)
local cl = self:get_cached_line(idx)
local tx, ty = x, y + self:get_line_text_y_offset()
local font = self:get_font()
for _, type, text in highlighter.each_token(cl.tokens) do
for _, type, text in self.doc.highlighter:each_token(idx) do
local color = style.syntax[type]
tx = renderer.draw_text(font, text, tx, ty, color)
end
@ -355,9 +287,9 @@ function DocView:draw_line_body(idx, x, y)
-- draw selection if it overlaps this line
local line1, col1, line2, col2 = self.doc:get_selection(true)
if idx >= line1 and idx <= line2 then
local cl = self:get_cached_line(idx)
local text = self.doc.lines[idx]
if line1 ~= idx then col1 = 1 end
if line2 ~= idx then col2 = #cl.text + 1 end
if line2 ~= idx then col2 = #text + 1 end
local x1 = x + self:get_col_x_offset(idx, col1)
local x2 = x + self:get_col_x_offset(idx, col2)
local lh = self:get_line_height()

8
data/core/highlighter.lua → data/core/tokenizer.lua
View File

@ -1,4 +1,4 @@
local highlighter = {}
local tokenizer = {}
local function push_token(t, type, text)
@ -38,7 +38,7 @@ local function find_non_escaped(text, pattern, offset, esc)
end
function highlighter.tokenize(syntax, text, state)
function tokenizer.tokenize(syntax, text, state)
local res = {}
local i = 1
@ -100,9 +100,9 @@ local function iter(t, i)
end
end
function highlighter.each_token(t)
function tokenizer.each_token(t)
return iter, t, -1
end
return highlighter
return tokenizer