Implement project files rescan on dir changes

In theory the dmon based directory monitoring is enough to ensure that
the list of project files is always correct. In reality some events
may be missing and the project files list may get disaligned with the
real list of files.

To avoid the problem we add an additional rescan to be done later in a
thread on any project subdirectory affected by an event of directory of
file change.

In the rescan found the same files already present the thread terminates.
If a difference is found the files list is modified and a new rescan is
scheduled.
This commit is contained in:
Francesco Abbate 2021-07-23 19:36:31 +02:00
parent aa9221e785
commit 1277399bbb
2 changed files with 116 additions and 23 deletions

View File

@ -116,8 +116,7 @@ end
-- When recursing "root" will always be the same, only "path" will change. -- When recursing "root" will always be the same, only "path" will change.
-- Returns a list of file "items". In eash item the "filename" will be the -- Returns a list of file "items". In eash item the "filename" will be the
-- complete file path relative to "root" *without* the trailing '/'. -- complete file path relative to "root" *without* the trailing '/'.
local function get_directory_files(root, path, t, recursive, begin_hook) local function get_directory_files(root, path, t, recursive)
if begin_hook then begin_hook() end
local size_limit = config.file_size_limit * 10e5 local size_limit = config.file_size_limit * 10e5
local all = system.list_dir(root .. path) or {} local all = system.list_dir(root .. path) or {}
local dirs, files = {}, {} local dirs, files = {}, {}
@ -237,6 +236,7 @@ function core.scan_project_subdir(dirname, filename)
if file.scanned then return end if file.scanned then return end
local new_files = get_directory_files(dirname, PATHSEP .. filename, {}) local new_files = get_directory_files(dirname, PATHSEP .. filename, {})
for _, new_file in ipairs(new_files) do for _, new_file in ipairs(new_files) do
-- FIXME: add index bounds to limit the scope of the search.
project_scan_add_entry(dir, new_file) project_scan_add_entry(dir, new_file)
end end
file.scanned = true file.scanned = true
@ -247,6 +247,58 @@ function core.scan_project_subdir(dirname, filename)
end end
end end
-- for "a" inclusive from i1 + 1 and i2
local function files_list_match(a, i1, i2, b)
if i2 - i1 ~= #b then return false end
for i = 1, #b do
if a[i1 + i].filename ~= b[i].filename or a[i1 + i].type ~= b[i].type then
return false
end
end
return true
end
-- arguments like for files_list_match
local function files_list_replace(a, i1, i2, b)
local nmin = math.min(i2 - i1, #b)
for i = 1, nmin do
a[i1 + i] = b[i]
end
for j = 1, i2 - i1 - nmin do
table.remove(a, i1 + nmin + 1)
end
for j = 1, #b - nmin do
table.insert(a, i1 + nmin + 1, b[nmin + j])
end
end
local function rescan_project_subdir(dir, filename_rooted)
local new_files = get_directory_files(dir.name, filename_rooted, {}, true)
local index, n = 0, #dir.files
if filename_rooted ~= "" then
local filename = strip_leading_path(filename_rooted)
for i, file in ipairs(dir.files) do
local file = dir.files[i]
if file.filename == filename then
index, n = i, #dir.files - i
for j = 1, #dir.files - i do
if not common.path_belongs_to(dir.files[i + j].filename, filename) then
n = j - 1
break
end
end
break
end
end
end
if not files_list_match(dir.files, index, index + n, new_files) then
files_list_replace(dir.files, index, index + n, new_files)
dir.is_dirty = true
return true
end
end
-- Find files and directories recursively reading from the filesystem. -- Find files and directories recursively reading from the filesystem.
-- Filter files and yields file's directory and info table. This latter -- Filter files and yields file's directory and info table. This latter
-- is filled to be like required by project directories "files" list. -- is filled to be like required by project directories "files" list.
@ -323,44 +375,39 @@ function core.project_files_number()
end end
local function project_scan_remove_file(watch_id, filepath) local function project_dir_by_watch_id(watch_id)
local project_dir_entry
for i = 1, #core.project_directories do for i = 1, #core.project_directories do
if core.project_directories[i].watch_id == watch_id then if core.project_directories[i].watch_id == watch_id then
project_dir_entry = core.project_directories[i] return core.project_directories[i]
end end
end end
if not project_dir_entry then return end end
local function project_scan_remove_file(dir, filepath)
local fileinfo = { filename = filepath } local fileinfo = { filename = filepath }
for _, filetype in ipairs {"dir", "file"} do for _, filetype in ipairs {"dir", "file"} do
fileinfo.type = filetype fileinfo.type = filetype
local index, match = file_search(project_dir_entry.files, fileinfo) local index, match = file_search(dir.files, fileinfo)
if match then if match then
table.remove(project_dir_entry.files, index) table.remove(dir.files, index)
project_dir_entry.is_dirty = true dir.is_dirty = true
return return
end end
end end
end end
local function project_scan_add_file(watch_id, filepath) local function project_scan_add_file(dir, filepath)
local project_dir_entry
for i = 1, #core.project_directories do
if core.project_directories[i].watch_id == watch_id then
project_dir_entry = core.project_directories[i]
end
end
if not project_dir_entry then return end
for fragment in string.gmatch(filepath, "([^/\\]+)") do for fragment in string.gmatch(filepath, "([^/\\]+)") do
if common.match_pattern(fragment, config.ignore_files) then if common.match_pattern(fragment, config.ignore_files) then
return return
end end
end end
local size_limit = config.file_size_limit * 10e5 local size_limit = config.file_size_limit * 10e5
local fileinfo = get_project_file_info(project_dir_entry.name, PATHSEP .. filepath, size_limit) local fileinfo = get_project_file_info(dir.name, PATHSEP .. filepath, size_limit)
if fileinfo then if fileinfo then
project_scan_add_entry(project_dir_entry, fileinfo) project_scan_add_entry(dir, fileinfo)
end end
end end
@ -963,12 +1010,58 @@ function core.try(fn, ...)
return false, err return false, err
end end
local scheduled_rescan = {}
local function dir_rescan_add_job(dir, filepath)
local dirpath = filepath:match("^(.+)[/\\].+$")
local dirpath_rooted = dirpath and PATHSEP .. dirpath or ""
local abs_dirpath = dir.name .. dirpath_rooted
if dirpath then
local _, dir_match = file_search(dir.files, {filename = dirpath, type = "dir"})
if not dir_match then return end
end
local new_time = system.get_time() + 1
local remove_list = {}
for _, rescan in pairs(scheduled_rescan) do
if abs_dirpath == rescan.abs_path or common.path_belongs_to(abs_dirpath, rescan.abs_path) then
-- abs_dirpath is a subpath of a scan already ongoing: skip
rescan.time_limit = new_time
return
elseif common.path_belongs_to(rescan.abs_path, abs_dirpath) then
table.insert(remove_list, rescan.abs_path)
end
end
for _, key_path in ipairs(remove_list) do
scheduled_rescan[key_path] = nil
end
scheduled_rescan[abs_dirpath] = {dir = dir, path = dirpath_rooted, abs_path = abs_dirpath, time_limit = new_time}
core.add_thread(function()
while true do
local rescan = scheduled_rescan[abs_dirpath]
if not rescan then return end
if system.get_time() > rescan.time_limit then
local has_changes = rescan_project_subdir(rescan.dir, rescan.path)
if has_changes then
rescan.time_limit = new_time
else
scheduled_rescan[rescan.abs_path] = nil
return
end
end
coroutine.yield(0.2)
end
end, dir)
end
function core.on_dir_change(watch_id, action, filepath) function core.on_dir_change(watch_id, action, filepath)
local dir = project_dir_by_watch_id(watch_id)
if not dir then return end
dir_rescan_add_job(dir, filepath)
if action == "delete" then if action == "delete" then
project_scan_remove_file(watch_id, filepath) project_scan_remove_file(dir, filepath)
elseif action == "create" then elseif action == "create" then
project_scan_add_file(watch_id, filepath) project_scan_add_file(dir, filepath)
end end
end end

View File

@ -6,12 +6,12 @@
`core.add_project_directory`: `core.add_project_directory`:
Add a new top-level directory to the project. Add a new top-level directory to the project.
Also called from modules and commands outside core.init. Also called from modules and commands outside core.init.
`core.scan_project_folder`: local function `scan_project_folder`:
Scan all files for a given top-level project directory. Scan all files for a given top-level project directory.
Can emit a warning about file limit. Can emit a warning about file limit.
Called only from within core.init module. Called only from within core.init module.
`core.scan_project_folder`: (renamed to `core.scan_project_subdir`) `core.scan_project_subdir`: (before was named `core.scan_project_folder`)
scan a single folder, without recursion. Used when too many files. scan a single folder, without recursion. Used when too many files.
New local function `scan_project_folder`: New local function `scan_project_folder`: