walkero
/
lite-xl
1
0
Fork 0
Code Issues 7 Pull Requests Projects Releases 14 Wiki Activity

Implement lazy loading of directories 

When the number of files in a project directory is above the max
limit switch back to a mechanism to read directory content only
when the corresponding folder is expanded in the treeview.

When the command core:find-file is invoked the command core:open-file
is executed instead because the complete list of the project's
files is not available.

When a project search is done we search through all the files within
the project dir without indexing them.

Address issues #217 #203 #183.
This commit is contained in:
Francesco Abbate 2021-05-25 23:25:56 +02:00
parent 8acb3fae8c
commit 10fde6e264
4 changed files with 146 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
data/core/commands/core.lua
View File

@ -66,6 +66,9 @@ command.add(nil, {
end,
["core:find-file"] = function()
if core.project_files_limit then
return command.perform "core:open-file"
end
local files = {}
for dir, item in core.get_project_files() do
if item.type == "file" then

112
data/core/init.lua
View File

@ -72,6 +72,7 @@ function core.set_project_dir(new_dir, change_project_fn)
core.project_directories = {}
core.add_project_directory(new_dir)
core.project_files = {}
core.project_files_limit = false
core.reschedule_project_scan()
return true
end
@ -99,32 +100,20 @@ local function strip_trailing_slash(filename)
return filename
end
local function project_scan_thread()
local function diff_files(a, b)
if #a ~= #b then return true end
for i, v in ipairs(a) do
if b[i].filename ~= v.filename
or b[i].modified ~= v.modified then
return true
end
end
end
local function compare_file(a, b)
local function compare_file(a, b)
return a.filename < b.filename
end
end
-- "root" will by an absolute path without trailing '/'
-- "path" will be a path starting with '/' and without trailing '/'
-- or the empty string.
-- It will identifies a sub-path within "root.
-- The current path location will therefore always be: root .. path.
-- When recursing "root" will always be the same, only "path" will change.
-- Returns a list of file "items". In eash item the "filename" will be the
-- complete file path relative to "root" *without* the trailing '/'.
local function get_files(root, path, t)
coroutine.yield()
t = t or {}
-- "root" will by an absolute path without trailing '/'
-- "path" will be a path starting with '/' and without trailing '/'
-- or the empty string.
-- It will identifies a sub-path within "root.
-- The current path location will therefore always be: root .. path.
-- When recursing "root" will always be the same, only "path" will change.
-- Returns a list of file "items". In eash item the "filename" will be the
-- complete file path relative to "root" *without* the trailing '/'.
local function get_directory_files(root, path, t, recursive, begin_hook)
if begin_hook then begin_hook() end
local size_limit = config.file_size_limit * 10e5
local all = system.list_dir(root .. path) or {}
local dirs, files = {}, {}
@ -139,7 +128,7 @@ local function project_scan_thread()
info.filename = strip_leading_path(file)
table.insert(info.type == "dir" and dirs or files, info)
entries_count = entries_count + 1
if entries_count > max_entries then break end
if recursive and entries_count > max_entries then return nil, entries_count end
end
end
end
@ -147,9 +136,10 @@ local function project_scan_thread()
table.sort(dirs, compare_file)
for _, f in ipairs(dirs) do
table.insert(t, f)
if entries_count <= max_entries then
local subdir_t, subdir_count = get_files(root, PATHSEP .. f.filename, t)
if recursive and entries_count <= max_entries then
local subdir_t, subdir_count = get_directory_files(root, PATHSEP .. f.filename, t, recursive)
entries_count = entries_count + subdir_count
f.scanned = true
end
end
@ -159,16 +149,29 @@ local function project_scan_thread()
end
return t, entries_count
end
local function project_scan_thread()
local function diff_files(a, b)
if #a ~= #b then return true end
for i, v in ipairs(a) do
if b[i].filename ~= v.filename
or b[i].modified ~= v.modified then
return true
end
end
end
while true do
-- get project files and replace previous table if the new table is
-- different
for i = 1, #core.project_directories do
local i = 1
while not core.project_files_limit and i <= #core.project_directories do
local dir = core.project_directories[i]
local t, entries_count = get_files(dir.name, "")
local t, entries_count = get_directory_files(dir.name, "", {}, true)
if diff_files(dir.files, t) then
if entries_count > config.max_project_files then
core.project_files_limit = true
core.status_view:show_message("!", style.accent,
"Too many files in project directory: stopping reading at "..
config.max_project_files.." files according to config.max_project_files. "..
@ -182,6 +185,7 @@ local function project_scan_thread()
if dir.name == core.project_dir then
core.project_files = dir.files
end
i = i + 1
end
-- wait for next scan
@ -190,6 +194,46 @@ local function project_scan_thread()
end
function core.scan_project_folder(dirname, filename)
for _, dir in ipairs(core.project_directories) do
if dir.name == dirname then
for i, file in ipairs(dir.files) do
local file = dir.files[i]
if file.filename == filename then
if file.scanned then return end
local new_files = get_directory_files(dirname, PATHSEP .. filename, {})
for j, new_file in ipairs(new_files) do
table.insert(dir.files, i + j, new_file)
end
file.scanned = true
return
end
end
end
end
end
local function find_project_files_co(root, path)
local size_limit = config.file_size_limit * 10e5
local all = system.list_dir(root .. path) or {}
for _, file in ipairs(all) do
if not common.match_pattern(file, config.ignore_files) then
local file = path .. PATHSEP .. file
local info = system.get_file_info(root .. file)
if info and info.size < size_limit then
info.filename = strip_leading_path(file)
if info.type == "file" then
coroutine.yield(root, info)
else
find_project_files_co(root, PATHSEP .. info.filename)
end
end
end
end
end
local function project_files_iter(state)
local dir = core.project_directories[state.dir_index]
state.file_index = state.file_index + 1
@ -204,17 +248,27 @@ end
function core.get_project_files()
if core.project_files_limit then
return coroutine.wrap(function()
for _, dir in ipairs(core.project_directories) do
find_project_files_co(dir.name, "")
end
end)
else
local state = { dir_index = 1, file_index = 0 }
return project_files_iter, state
end
end
function core.project_files_number()
if not core.project_files_limit then
local n = 0
for i = 1, #core.project_directories do
n = n + #core.project_directories[i].files
end
return n
end
end

7
data/plugins/projectsearch.lua
View File

@ -170,12 +170,17 @@ function ResultsView:draw()
local ox, oy = self:get_content_offset()
local x, y = ox + style.padding.x, oy + style.padding.y
local files_number = core.project_files_number()
local per = self.last_file_idx / files_number
local per = files_number and self.last_file_idx / files_number or 1
local text
if self.searching then
if files_number then
text = string.format("Searching %d%% (%d of %d files, %d matches) for %q...",
per * 100, self.last_file_idx, files_number,
#self.results, self.query)
else
text = string.format("Searching (%d files, %d matches) for %q...",
self.last_file_idx, #self.results, self.query)
end
else
text = string.format("Found %d matches for %q",
#self.results, self.query)

29
data/plugins/treeview.lua
View File

@ -93,6 +93,13 @@ function TreeView:get_item_height()
end
function TreeView:invalidate_cache(dirname)
for _, v in pairs(self.cache[dirname]) do
v.skip = nil
end
end
function TreeView:check_cache()
-- invalidate cache's skip values if project_files has changed
for i = 1, #core.project_directories do
@ -102,9 +109,7 @@ function TreeView:check_cache()
self.last[dir.name] = dir.files
else
if dir.files ~= last_files then
for _, v in pairs(self.cache[dir.name]) do
v.skip = nil
end
self:invalidate_cache(dir.name)
self.last[dir.name] = dir.files
end
end
@ -208,17 +213,25 @@ function TreeView:on_mouse_pressed(button, x, y, clicks)
if caught then
return
end
if not self.hovered_item then
local hovered_item = self.hovered_item
if not hovered_item then
return
elseif self.hovered_item.type == "dir" then
elseif hovered_item.type == "dir" then
if keymap.modkeys["ctrl"] and button == "left" then
create_directory_in(self.hovered_item)
create_directory_in(hovered_item)
else
self.hovered_item.expanded = not self.hovered_item.expanded
if core.project_files_limit and not hovered_item.expanded then
local filename, abs_filename = hovered_item.filename, hovered_item.abs_filename
local index = string.find(abs_filename, filename, 1, true)
local dirname = string.sub(abs_filename, 1, index - 2)
core.scan_project_folder(dirname, filename)
self:invalidate_cache(dirname)
end
hovered_item.expanded = not hovered_item.expanded
end
else
core.try(function()
local doc_filename = common.relative_path(core.project_dir, self.hovered_item.abs_filename)
local doc_filename = common.relative_path(core.project_dir, hovered_item.abs_filename)
core.root_view:open_doc(core.open_doc(doc_filename))
end)
end