Merge branch 'dmon-1' into dmon-debug

Include changes in dmon not yet merged into master.

data/core/commands/core.lua

@@ -66,7 +66,7 @@ command.add(nil, {
   end,
 
   ["core:find-file"] = function()
-    if core.project_files_limit then
+    if not core.project_files_number() then
        return command.perform "core:open-file"
     end
     local files = {}
@@ -191,8 +191,6 @@ command.add(nil, {
         return
       end
       core.add_project_directory(system.absolute_path(text))
-      -- TODO: add the name of directory to prioritize
-      core.reschedule_project_scan()
     end, suggest_directory)
   end,
 

data/core/config.lua

@@ -1,6 +1,5 @@
 local config = {}
 
-config.project_scan_rate = 5
 config.fps = 60
 config.max_log_items = 80
 config.message_timeout = 5

data/core/init.lua

@@ -52,13 +52,6 @@ local function update_recents_project(action, dir_path_abs)
 end
 
 
-function core.reschedule_project_scan()
-  if core.project_scan_thread_id then
-    core.threads[core.project_scan_thread_id].wake = 0
-  end
-end
-
-
 function core.set_project_dir(new_dir, change_project_fn)
   local chdir_ok = pcall(system.chdir, new_dir)
   if chdir_ok then
@@ -66,9 +59,6 @@ function core.set_project_dir(new_dir, change_project_fn)
     core.project_dir = common.normalize_path(new_dir)
     core.project_directories = {}
     core.add_project_directory(new_dir)
-    core.project_files = {}
-    core.project_files_limit = false
-    core.reschedule_project_scan()
     return true
   end
   return false
@@ -99,6 +89,20 @@ local function compare_file(a, b)
   return a.filename < b.filename
 end
 
+
+-- compute a file's info entry completed with "filename" to be used
+-- in project scan or falsy if it shouldn't appear in the list.
+local function get_project_file_info(root, file)
+  local info = system.get_file_info(root .. file)
+  if info then
+    info.filename = strip_leading_path(file)
+    return (info.size < config.file_size_limit * 1e6 and
+      not common.match_pattern(info.filename, config.ignore_files)
+      and info)
+  end
+end
+
+
 -- "root" will by an absolute path without trailing '/'
 -- "path" will be a path starting with '/' and without trailing '/'
 --    or the empty string.
@@ -107,34 +111,27 @@ end
 -- When recursing "root" will always be the same, only "path" will change.
 -- Returns a list of file "items". In eash item the "filename" will be the
 -- complete file path relative to "root" *without* the trailing '/'.
-local function get_directory_files(root, path, t, recursive, begin_hook)
+local function get_directory_files(dir, root, path, t, begin_hook, max_files)
   if begin_hook then begin_hook() end
-  local size_limit = config.file_size_limit * 10e5
   local all = system.list_dir(root .. path) or {}
   local dirs, files = {}, {}
 
   local entries_count = 0
-  local max_entries = config.max_project_files
   for _, file in ipairs(all) do
-    if not common.match_pattern(file, config.ignore_files) then
+    local info = get_project_file_info(root, path .. PATHSEP .. file)
-      local file = path .. PATHSEP .. file
+    if info then
-      local info = system.get_file_info(root .. file)
-      if info and info.size < size_limit then
-        info.filename = strip_leading_path(file)
       table.insert(info.type == "dir" and dirs or files, info)
       entries_count = entries_count + 1
-        if recursive and entries_count > max_entries then return nil, entries_count end
-      end
     end
   end
 
   table.sort(dirs, compare_file)
   for _, f in ipairs(dirs) do
     table.insert(t, f)
-    if recursive and entries_count <= max_entries then
+    if (not max_files or entries_count <= max_files) and core.project_subdir_is_shown(dir, f.filename) then
-      local subdir_t, subdir_count = get_directory_files(root, PATHSEP .. f.filename, t, recursive)
+      local sub_limit = max_files and max_files - entries_count
-      entries_count = entries_count + subdir_count
+      local _, n = get_directory_files(dir, root, PATHSEP .. f.filename, t, begin_hook, sub_limit)
-      f.scanned = true
+      entries_count = entries_count + n
     end
   end
 
@@ -146,133 +143,295 @@ local function get_directory_files(root, path, t, recursive, begin_hook)
   return t, entries_count
 end
 
-local function project_scan_thread()
+
-  local function diff_files(a, b)
+function core.project_subdir_set_show(dir, filename, show)
-    if #a ~= #b then return true end
+  dir.shown_subdir[filename] = show
-    for i, v in ipairs(a) do
+  if dir.files_limit and PLATFORM == "Linux" then
-      if b[i].filename ~= v.filename
+    local fullpath = dir.name .. PATHSEP .. filename
-      or b[i].modified ~= v.modified then
+    local watch_fn = show and system.watch_dir_add or system.watch_dir_rm
-        return true
+    print("DEBUG dir", dir.name, "filename", filename, "watch_id:", dir.watch_id, "show:", show)
+    local success = watch_fn(dir.watch_id, fullpath)
+    print("DEBUG: ", show and "watch_dir_add" or "watch_dir_rm", fullpath, "success:", success)
+    if not success then
+      core.log("Internal warning: error calling system.watch_dir_%s", show and "add" or "rm")
     end
   end
 end
 
-  while true do
+
-    -- get project files and replace previous table if the new table is
+function core.project_subdir_is_shown(dir, filename)
-    -- different
+  return not dir.files_limit or dir.shown_subdir[filename]
-    local i = 1
+end
-    while not core.project_files_limit and i <= #core.project_directories do
+
-      local dir = core.project_directories[i]
+
-      local t, entries_count = get_directory_files(dir.name, "", {}, true)
+local function show_max_files_warning()
-      if diff_files(dir.files, t) then
-        if entries_count > config.max_project_files then
-          core.project_files_limit = true
   core.status_view:show_message("!", style.accent,
     "Too many files in project directory: stopped reading at "..
     config.max_project_files.." files. For more information see "..
     "usage.md at github.com/franko/lite-xl."
     )
 end
-        dir.files = t
+
-        core.redraw = true
+-- Populate a project folder top directory by scanning the filesystem.
+local function scan_project_folder(index)
+  local dir = core.project_directories[index]
+  local t, entries_count = get_directory_files(dir, dir.name, "", {}, nil, config.max_project_files)
+  if entries_count > config.max_project_files then
+    print("DEBUG setting files limit FLAG for", dir.name)
+    dir.files_limit = true
+    -- Watch non-recursively on Linux only.
+    -- The reason is recursively watching with dmon on linux
+    -- doesn't work on very large directories.
+    dir.watch_id = system.watch_dir(dir.name, PLATFORM ~= "Linux")
+    if core.status_view then -- May be not yet initialized.
+      show_max_files_warning()
     end
-      if dir.name == core.project_dir then
+  else
+    dir.watch_id = system.watch_dir(dir.name, true)
+  end
+  dir.files = t
+  core.dir_rescan_add_job(dir, ".")
+end
+
+
+function core.add_project_directory(path)
+  -- top directories has a file-like "item" but the item.filename
+  -- will be simply the name of the directory, without its path.
+  -- The field item.topdir will identify it as a top level directory.
+  path = common.normalize_path(path)
+  local dir = {
+    name = path,
+    item = {filename = common.basename(path), type = "dir", topdir = true},
+    files_limit = false,
+    is_dirty = true,
+    shown_subdir = {},
+  }
+  table.insert(core.project_directories, dir)
+  scan_project_folder(#core.project_directories)
+  if path == core.project_dir then
     core.project_files = dir.files
   end
-      i = i + 1
+  core.redraw = true
 end
 
-    -- wait for next scan
+
-    coroutine.yield(config.project_scan_rate)
+local function file_search(files, info)
+  local filename, type = info.filename, info.type
+  local inf, sup = 1, #files
+  while sup - inf > 8 do
+    local curr = math.floor((inf + sup) / 2)
+    if system.path_compare(filename, type, files[curr].filename, files[curr].type) then
+      sup = curr - 1
+    else
+      inf = curr
+    end
+  end
+  repeat
+    if files[inf].filename == filename then
+      return inf, true
+    end
+    inf = inf + 1
+  until inf > sup or system.path_compare(filename, type, files[inf].filename, files[inf].type)
+  return inf, false
+end
+
+
+local function project_scan_add_entry(dir, fileinfo)
+  local index, match = file_search(dir.files, fileinfo)
+  if not match then
+    table.insert(dir.files, index, fileinfo)
+    dir.is_dirty = true
   end
 end
 
 
-function core.is_project_folder(dirname)
+local function files_info_equal(a, b)
-  for _, dir in ipairs(core.project_directories) do
+  return a.filename == b.filename and a.type == b.type
-    if dir.name == dirname then
-      return true
-    end
 end
+
+-- for "a" inclusive from i1 + 1 and i1 + n
+local function files_list_match(a, i1, n, b)
+  if n ~= #b then return false end
+  for i = 1, n do
+    if not files_info_equal(a[i1 + i], b[i]) then
       return false
     end
+  end
+  return true
+end
 
+-- arguments like for files_list_match
+local function files_list_replace(as, i1, n, bs)
+  local m = #bs
+  local i, j = 1, 1
+  while i <= m or i <= n do
+    local a, b = as[i1 + i], bs[j]
+    if i > n or (j <= m and not files_info_equal(a, b) and
+      not system.path_compare(a.filename, a.type, b.filename, b.type))
+    then
+      print("DEBUG FIX insert: ", b.filename, "before:", as[i1 + i] and as[i1 + i].filename)
+      table.insert(as, i1 + i, b)
+      i, j, n = i + 1, j + 1, n + 1
+    elseif j > m or system.path_compare(a.filename, a.type, b.filename, b.type) then
+      print("DEBUG FIX remove: ", a.filename, "before:", as[i1 + i + 1] and as[i1 + i + 1].filename)
+      table.remove(as, i1 + i)
+      n = n - 1
+    else
+      i, j = i + 1, j + 1
+    end
+  end
+end
 
-function core.scan_project_folder(dirname, filename)
+local function project_subdir_bounds(dir, filename)
-  for _, dir in ipairs(core.project_directories) do
+  local index, n = 0, #dir.files
-    if dir.name == dirname then
   for i, file in ipairs(dir.files) do
     local file = dir.files[i]
     if file.filename == filename then
-          if file.scanned then return end
+      index, n = i, #dir.files - i
-          local new_files = get_directory_files(dirname, PATHSEP .. filename, {})
+      for j = 1, #dir.files - i do
-          for j, new_file in ipairs(new_files) do
+        if not common.path_belongs_to(dir.files[i + j].filename, filename) then
-            table.insert(dir.files, i + j, new_file)
+          n = j - 1
-          end
+          break
-          file.scanned = true
-          return
         end
       end
+      return index, n, file
     end
   end
 end
 
+local function rescan_project_subdir(dir, filename_rooted)
+  local new_files = get_directory_files(dir, dir.name, filename_rooted, {}, coroutine.yield)
+  local index, n = 0, #dir.files
+  if filename_rooted ~= "" then
+    local filename = strip_leading_path(filename_rooted)
+    index, n = project_subdir_bounds(dir, filename)
+  end
 
-local function find_project_files_co(root, path)
+  if not files_list_match(dir.files, index, n, new_files) then
-  local size_limit = config.file_size_limit * 10e5
+    files_list_replace(dir.files, index, n, new_files)
+    dir.is_dirty = true
+    return true
+  end
+end
+
+
+function core.update_project_subdir(dir, filename, expanded)
+  local index, n, file = project_subdir_bounds(dir, filename)
+  if index then
+    local new_files = expanded and get_directory_files(dir, dir.name, PATHSEP .. filename, {}) or {}
+    files_list_replace(dir.files, index, n, new_files)
+    dir.is_dirty = true
+    return true
+  end
+end
+
+
+-- Find files and directories recursively reading from the filesystem.
+-- Filter files and yields file's directory and info table. This latter
+-- is filled to be like required by project directories "files" list.
+local function find_files_rec(root, path)
   local all = system.list_dir(root .. path) or {}
   for _, file in ipairs(all) do
-    if not common.match_pattern(file, config.ignore_files) then
     local file = path .. PATHSEP .. file
     local info = system.get_file_info(root .. file)
-      if info and info.size < size_limit then
+    if info then
       info.filename = strip_leading_path(file)
       if info.type == "file" then
         coroutine.yield(root, info)
       else
-          find_project_files_co(root, PATHSEP .. info.filename)
+        find_files_rec(root, PATHSEP .. info.filename)
-        end
       end
     end
   end
 end
 
 
+-- Iterator function to list all project files
 local function project_files_iter(state)
   local dir = core.project_directories[state.dir_index]
+  if state.co then
+    -- We have a coroutine to fetch for files, use the coroutine.
+    -- Used for directories that exceeds the files nuumber limit.
+    local ok, name, file = coroutine.resume(state.co, dir.name, "")
+    if ok and name then
+      return name, file
+    else
+      -- The coroutine terminated, increment file/dir counter to scan
+      -- next project directory.
+      state.co = false
+      state.file_index = 1
+      state.dir_index = state.dir_index + 1
+      dir = core.project_directories[state.dir_index]
+    end
+  else
+    -- Increase file/dir counter
     state.file_index = state.file_index + 1
     while dir and state.file_index > #dir.files do
       state.dir_index = state.dir_index + 1
       state.file_index = 1
       dir = core.project_directories[state.dir_index]
     end
+  end
   if not dir then return end
+  if dir.files_limit then
+    -- The current project directory is files limited: create a couroutine
+    -- to read files from the filesystem.
+    state.co = coroutine.create(find_files_rec)
+    return project_files_iter(state)
+  end
   return dir.name, dir.files[state.file_index]
 end
 
 
 function core.get_project_files()
-  if core.project_files_limit then
-    return coroutine.wrap(function()
-      for _, dir in ipairs(core.project_directories) do
-        find_project_files_co(dir.name, "")
-      end
-    end)
-  else
   local state = { dir_index = 1, file_index = 0 }
   return project_files_iter, state
 end
-end
 
 
 function core.project_files_number()
-  if not core.project_files_limit then
   local n = 0
   for i = 1, #core.project_directories do
+    if core.project_directories[i].files_limit then return end
     n = n + #core.project_directories[i].files
   end
   return n
 end
+
+
+local function project_dir_by_watch_id(watch_id)
+  for i = 1, #core.project_directories do
+    if core.project_directories[i].watch_id == watch_id then
+      return core.project_directories[i]
+    end
+  end
+end
+
+
+local function project_scan_remove_file(dir, filepath)
+  local fileinfo = { filename = filepath }
+  for _, filetype in ipairs {"dir", "file"} do
+    fileinfo.type = filetype
+    local index, match = file_search(dir.files, fileinfo)
+    if match then
+      table.remove(dir.files, index)
+      dir.is_dirty = true
+      return
+    end
+  end
+end
+
+
+local function project_scan_add_file(dir, filepath)
+  for fragment in string.gmatch(filepath, "([^/\\]+)") do
+    if common.match_pattern(fragment, config.ignore_files) then
+      return
+    end
+  end
+  local fileinfo = get_project_file_info(dir.name, PATHSEP .. filepath)
+  if fileinfo then
+    project_scan_add_entry(dir, fileinfo)
+  end
 end
 
 
@@ -368,19 +527,6 @@ function core.load_user_directory()
 end
 
 
-function core.add_project_directory(path)
-  -- top directories has a file-like "item" but the item.filename
-  -- will be simply the name of the directory, without its path.
-  -- The field item.topdir will identify it as a top level directory.
-  path = common.normalize_path(path)
-  table.insert(core.project_directories, {
-    name = path,
-    item = {filename = common.basename(path), type = "dir", topdir = true},
-    files = {}
-  })
-end
-
-
 function core.remove_project_directory(path)
   -- skip the fist directory because it is the project's directory
   for i = 2, #core.project_directories do
@@ -516,7 +662,6 @@ function core.init()
   cur_node = cur_node:split("down", core.command_view, {y = true})
   cur_node = cur_node:split("down", core.status_view, {y = true})
 
-  core.project_scan_thread_id = core.add_thread(project_scan_thread)
   command.add_defaults()
   local got_user_error = not core.load_user_directory()
   local plugins_success, plugins_refuse_list = core.load_plugins()
@@ -527,6 +672,12 @@ function core.init()
   end
   local got_project_error = not core.load_project_module()
 
+  -- We assume we have just a single project directory here. Now that StatusView
+  -- is there show max files warning if needed.
+  if core.project_directories[1].files_limit then
+    show_max_files_warning()
+  end
+
   for _, filename in ipairs(files) do
     core.root_view:open_doc(core.open_doc(filename))
   end
@@ -915,6 +1066,82 @@ function core.try(fn, ...)
   return false, err
 end
 
+local scheduled_rescan = {}
+
+function core.has_pending_rescan()
+  for _ in pairs(scheduled_rescan) do
+    return true
+  end
+end
+
+
+function core.dir_rescan_add_job(dir, filepath)
+  local dirpath = filepath:match("^(.+)[/\\].+$")
+  local dirpath_rooted = dirpath and PATHSEP .. dirpath or ""
+  local abs_dirpath = dir.name .. dirpath_rooted
+  if dirpath then
+    -- check if the directory is in the project files list, if not exit
+    local dir_index, dir_match = file_search(dir.files, {filename = dirpath, type = "dir"})
+    -- Note that is dir_match is false dir_index greaten than the last valid index.
+    -- We use dir_index to index dir.files below only if dir_match is true.
+    if not dir_match or not core.project_subdir_is_shown(dir, dir.files[dir_index].filename) then
+      print("DEBUG do not start a rescan job for", abs_dirpath)
+      return
+    end
+  end
+  local new_time = system.get_time() + 1
+
+  -- evaluate new rescan request versus existing rescan
+  local remove_list = {}
+  for _, rescan in pairs(scheduled_rescan) do
+    if abs_dirpath == rescan.abs_path or common.path_belongs_to(abs_dirpath, rescan.abs_path) then
+      -- abs_dirpath is a subpath of a scan already ongoing: skip
+      rescan.time_limit = new_time
+      return
+    elseif common.path_belongs_to(rescan.abs_path, abs_dirpath) then
+      -- abs_dirpath already cover this rescan: add to the list of rescan to be removed
+      table.insert(remove_list, rescan.abs_path)
+    end
+  end
+  for _, key_path in ipairs(remove_list) do
+    scheduled_rescan[key_path] = nil
+  end
+
+  print("DEBUG: adding rescan thread for", abs_dirpath)
+  scheduled_rescan[abs_dirpath] = {dir = dir, path = dirpath_rooted, abs_path = abs_dirpath, time_limit = new_time}
+  core.add_thread(function()
+    while true do
+      local rescan = scheduled_rescan[abs_dirpath]
+      if not rescan then print("DEBUG: cancel rescan for", abs_dirpath); return end
+      if system.get_time() > rescan.time_limit then
+        local has_changes = rescan_project_subdir(rescan.dir, rescan.path)
+        print("DEBUG: rescan done for", abs_dirpath, " changes", has_changes)
+        if has_changes then
+          core.redraw = true -- we run without an event, from a thread
+          rescan.time_limit = new_time
+        else
+          print("DEBUG: terminating rescan for", abs_dirpath)
+          scheduled_rescan[rescan.abs_path] = nil
+          return
+        end
+      end
+      coroutine.yield(0.2)
+    end
+  end)
+end
+
+
+function core.on_dir_change(watch_id, action, filepath)
+  local dir = project_dir_by_watch_id(watch_id)
+  if not dir then return end
+  core.dir_rescan_add_job(dir, filepath)
+  if action == "delete" then
+    project_scan_remove_file(dir, filepath)
+  elseif action == "create" then
+    project_scan_add_file(dir, filepath)
+  end
+end
+
 
 function core.on_event(type, ...)
   local did_keymap = false
@@ -951,6 +1178,8 @@ function core.on_event(type, ...)
     end
   elseif type == "focuslost" then
     core.root_view:on_focus_lost(...)
+  elseif type == "dirchange" then
+    core.on_dir_change(...)
   elseif type == "quit" then
     core.quit()
   end
@@ -1054,10 +1283,11 @@ end)
 
 function core.run()
   local idle_iterations = 0
+  local debug_count = 0
   while true do
     core.frame_start = system.get_time()
     local did_redraw = core.step()
-    local need_more_work = run_threads()
+    local need_more_work = run_threads() or core.has_pending_rescan()
     if core.restart_request or core.quit_request then break end
     if not did_redraw and not need_more_work then
       idle_iterations = idle_iterations + 1
@@ -1071,6 +1301,8 @@ function core.run()
           local dt = math.ceil(t / h) * h - t
           system.wait_event(dt + 1 / config.fps)
         else
+          print("DEBUG:", debug_count, " application WAITING")
+          debug_count = debug_count + 1
           system.wait_event()
         end
       end

data/core/keymap.lua

@@ -5,7 +5,7 @@ keymap.modkeys = {}
 keymap.map = {}
 keymap.reverse_map = {}
 
-local macos = rawget(_G, "MACOS")
+local macos = PLATFORM:match("^[Mm]ac")
 
 -- Thanks to mathewmariani, taken from his lite-macos github repository.
 local modkeys_os = require("core.modkeys-" .. (macos and "macos" or "generic"))

data/plugins/autoreload.lua

@@ -3,9 +3,10 @@ local core = require "core"
 local config = require "core.config"
 local Doc = require "core.doc"
 
-
 local times = setmetatable({}, { __mode = "k" })
 
+local autoreload_scan_rate = 5
+
 local function update_time(doc)
   local info = system.get_file_info(doc.filename)
   times[doc] = info.modified
@@ -40,7 +41,7 @@ core.add_thread(function()
     end
 
     -- wait for next scan
-    coroutine.yield(config.project_scan_rate)
+    coroutine.yield(autoreload_scan_rate)
   end
 end)
 

data/plugins/treeview.lua

@@ -41,7 +41,6 @@ function TreeView:new()
   self.init_size = true
   self.target_size = default_treeview_size
   self.cache = {}
-  self.last = {}
   self.tooltip = { x = 0, y = 0, begin = 0, alpha = 0 }
 end
 
@@ -54,7 +53,7 @@ function TreeView:set_target_size(axis, value)
 end
 
 
-function TreeView:get_cached(item, dirname)
+function TreeView:get_cached(dir, item, dirname)
   local dir_cache = self.cache[dirname]
   if not dir_cache then
     dir_cache = {}
@@ -80,6 +79,7 @@ function TreeView:get_cached(item, dirname)
     end
     t.name = basename
     t.type = item.type
+    t.dir = dir -- points to top level "dir" item
     dir_cache[cache_name] = t
   end
   return t
@@ -104,18 +104,13 @@ end
 
 
 function TreeView:check_cache()
-  -- invalidate cache's skip values if project_files has changed
   for i = 1, #core.project_directories do
     local dir = core.project_directories[i]
-    local last_files = self.last[dir.name]
+    -- invalidate cache's skip values if directory is declared dirty
-    if not last_files then
+    if dir.is_dirty and self.cache[dir.name] then
-      self.last[dir.name] = dir.files
-    else
-      if dir.files ~= last_files then
       self:invalidate_cache(dir.name)
-        self.last[dir.name] = dir.files
-      end
     end
+    dir.is_dirty = false
   end
 end
 
@@ -131,14 +126,14 @@ function TreeView:each_item()
 
     for k = 1, #core.project_directories do
       local dir = core.project_directories[k]
-      local dir_cached = self:get_cached(dir.item, dir.name)
+      local dir_cached = self:get_cached(dir, dir.item, dir.name)
       coroutine.yield(dir_cached, ox, y, w, h)
       count_lines = count_lines + 1
       y = y + h
       local i = 1
       while i <= #dir.files and dir_cached.expanded do
         local item = dir.files[i]
-        local cached = self:get_cached(item, dir.name)
+        local cached = self:get_cached(dir, item, dir.name)
 
         coroutine.yield(cached, ox, y, w, h)
         count_lines = count_lines + 1
@@ -206,7 +201,6 @@ local function create_directory_in(item)
       core.error("cannot create directory %q: %s", dirname, err)
     end
     item.expanded = true
-    core.reschedule_project_scan()
   end)
 end
 
@@ -223,23 +217,12 @@ function TreeView:on_mouse_pressed(button, x, y, clicks)
     if keymap.modkeys["ctrl"] and button == "left" then
       create_directory_in(hovered_item)
     else
-      if core.project_files_limit and not hovered_item.expanded then
-        local filename, abs_filename = hovered_item.filename, hovered_item.abs_filename
-        local index = 0
-        -- The loop below is used to find the first match starting from the end
-        -- in case there are multiple matches.
-        while index and index + #filename < #abs_filename do
-          index = string.find(abs_filename, filename, index + 1, true)
-        end
-        -- we assume here index is not nil because the abs_filename must contain the
-        -- relative filename
-        local dirname = string.sub(abs_filename, 1, index - 2)
-        if core.is_project_folder(dirname) then
-          core.scan_project_folder(dirname, filename)
-          self:invalidate_cache(dirname)
-        end
-      end
       hovered_item.expanded = not hovered_item.expanded
+      if hovered_item.dir.files_limit then
+        core.update_project_subdir(hovered_item.dir, hovered_item.filename, hovered_item.expanded)
+        print("DEBUG setting show flag to", hovered_item.expanded)
+        core.project_subdir_set_show(hovered_item.dir, hovered_item.filename, hovered_item.expanded)
+      end
     end
   else
     core.try(function()
@@ -460,7 +443,6 @@ command.add(function() return view.hovered_item ~= nil end, {
       else
         core.error("Error while renaming \"%s\" to \"%s\": %s", old_abs_filename, abs_filename, err)
       end
-      core.reschedule_project_scan()
     end, common.path_suggest)
   end,
 
@@ -475,7 +457,6 @@ command.add(function() return view.hovered_item ~= nil end, {
       file:write("")
       file:close()
       core.root_view:open_doc(core.open_doc(doc_filename))
-      core.reschedule_project_scan()
       core.log("Created %s", doc_filename)
     end, common.path_suggest)
   end,
@@ -488,7 +469,6 @@ command.add(function() return view.hovered_item ~= nil end, {
     core.command_view:enter("Folder Name", function(filename)
       local dir_path = core.project_dir .. PATHSEP .. filename
       common.mkdirp(dir_path)
-      core.reschedule_project_scan()
       core.log("Created %s", dir_path)
     end, common.path_suggest)
   end,
@@ -525,7 +505,6 @@ command.add(function() return view.hovered_item ~= nil end, {
               return
             end
           end
-          core.reschedule_project_scan()
           core.log("Deleted \"%s\"", filename)
         end
       end

licenses/licenses.md

@@ -22,6 +22,33 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
 
+## septag/dmon
+
+Copyright 2019 Sepehr Taghdisian. All rights reserved.
+
+https://github.com/septag/dmon
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+   1. Redistributions of source code must retain the above copyright notice,
+      this list of conditions and the following disclaimer.
+
+   2. Redistributions in binary form must reproduce the above copyright notice,
+      this list of conditions and the following disclaimer in the documentation
+      and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY COPYRIGHT HOLDER "AS IS" AND ANY EXPRESS OR
+IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+EVENT SHALL COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
 ## Fira Sans
 
 Digitized data copyright (c) 2012-2015, The Mozilla Foundation and Telefonica S.A.

meson.build

@@ -45,6 +45,7 @@ endif
 if not get_option('source-only')
     libm = cc.find_library('m', required : false)
     libdl = cc.find_library('dl', required : false)
+    threads_dep = dependency('threads')
     lua_dep = dependency('lua5.2', fallback: ['lua', 'lua_dep'],
         default_options: ['shared=false', 'use_readline=false', 'app=false']
     )
@@ -57,7 +58,7 @@ if not get_option('source-only')
         ]
     )
 
-    lite_deps = [lua_dep, sdl_dep, reproc_dep, pcre2_dep, libm, libdl]
+    lite_deps = [lua_dep, sdl_dep, reproc_dep, pcre2_dep, libm, libdl, threads_dep]
 
     if host_machine.system() == 'windows'
         # Note that we need to explicitly add the windows socket DLL because

resources/notes-dmon-integration-2.md

@@ -0,0 +1,10 @@
+## from core/init.lua
+
+- `scan_project_folder` set the `watch_id` recursively or not
+  * called from `core.add_project_directory`
+
+## from treeview.lua
+
+`TreeView:on_mouse_pressed`
+  * calls `core.scan_project_subdir` only in `files_limit` mode
+  * calls `core.project_subdir_set_show`

resources/notes-dmon-integration.md

@@ -0,0 +1,54 @@
+
+`core.set_project_dir`:
+    Reset project directories and set its directory.
+    It chdir into the directory, empty the `core.project_directories` and add
+    the given directory.
+    `core.add_project_directory`:
+        Add a new top-level directory to the project.
+        Also called from modules and commands outside core.init.
+        local function `scan_project_folder`:
+            Scan all files for a given top-level project directory.
+            Can emit a warning about file limit.
+            Called only from within core.init module.
+
+`core.scan_project_subdir`: (before was named `core.scan_project_folder`)
+    scan a single folder, without recursion. Used when too many files.
+
+Local function `scan_project_folder`:
+    Populate the project folder top directory. Done only once when the directory
+    is added to the project.
+
+`core.add_project_directory`:
+    Add a new top-level folder to the project.
+
+`core.set_project_dir`:
+    Set the initial project directory.
+
+`core.dir_rescan_add_job`:
+    Add a job to rescan after an elapsed time a project's subdirectory to fix for any
+    changes.
+
+Local function `rescan_project_subdir`:
+    Rescan a project's subdirectory, compare to the current version and patch the list if
+    a difference is found.
+
+
+`core.project_scan_thread`:
+    Should disappear now that we use dmon.
+
+
+`core.project_scan_topdir`:
+    New function to scan a top level project folder.
+
+
+`config.project_scan_rate`:
+`core.project_scan_thread_id`:
+`core.reschedule_project_scan`:
+`core.project_files_limit`:
+    A eliminer.
+
+`core.get_project_files`:
+    To be fixed. Use `find_project_files_co` for a single directory
+
+In TreeView remove usage of self.last to detect new scan that changed the files list.
+

scripts/dmon-test.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+IFS=$'\r\n' GLOBIGNORE='*' command eval  'DIRS=($(find . -type d))'
+
+n=${#DIRS[@]}
+files=()
+for i in {1..20}; do
+  files[$i]="${DIRS[$(( $RANDOM % $n ))]}/dmon-test-$i.txt"
+done
+
+for name in "${files[@]}"; do
+  echo "create ${name#./}"
+  touch "${name#./}"
+done
+
+echo "Files created"
+sleep 2;
+
+
+for name in "${files[@]}"; do
+  echo "remove ${name#./}"
+  rm "${name#./}"
+done
+
+echo "Files removed"

src/api/system.c

@@ -6,6 +6,7 @@
 #include <errno.h>
 #include <sys/stat.h>
 #include "api.h"
+#include "dirmonitor.h"
 #include "rencache.h"
 #ifdef _WIN32
   #include <direct.h>
@@ -222,6 +223,14 @@ top:
       lua_pushnumber(L, e.wheel.y);
       return 2;
 
+    case SDL_USEREVENT:
+      lua_pushstring(L, "dirchange");
+      lua_pushnumber(L, e.user.code >> 16);
+      lua_pushstring(L, (e.user.code & 0xffff) == DMON_ACTION_DELETE ? "delete" : "create");
+      lua_pushstring(L, e.user.data1);
+      free(e.user.data1);
+      return 4;
+
     default:
       goto top;
   }
@@ -637,6 +646,92 @@ static int f_set_window_opacity(lua_State *L) {
   return 1;
 }
 
+static int f_watch_dir(lua_State *L) {
+  const char *path = luaL_checkstring(L, 1);
+  const int recursive = lua_toboolean(L, 2);
+  uint32_t dmon_flags = (recursive ? DMON_WATCHFLAGS_RECURSIVE : 0);
+  dmon_watch_id watch_id = dmon_watch(path, dirmonitor_watch_callback, dmon_flags, NULL);
+  if (watch_id.id == 0) { luaL_error(L, "directory monitoring watch failed"); }
+  lua_pushnumber(L, watch_id.id);
+  return 1;
+}
+
+#if __linux__
+static int f_watch_dir_add(lua_State *L) {
+  dmon_watch_id watch_id;
+  watch_id.id = luaL_checkinteger(L, 1);
+  const char *subdir = luaL_checkstring(L, 2);
+  lua_pushboolean(L, dmon_watch_add(watch_id, subdir));
+  return 1;
+}
+
+static int f_watch_dir_rm(lua_State *L) {
+  dmon_watch_id watch_id;
+  watch_id.id = luaL_checkinteger(L, 1);
+  const char *subdir = luaL_checkstring(L, 2);
+  fprintf(stderr, "f_watch_dir_rm id: %d, subdir: %s\n", watch_id.id, subdir);
+  lua_pushboolean(L, dmon_watch_rm(watch_id, subdir));
+  return 1;
+}
+#endif
+
+#ifdef _WIN32
+#define PATHSEP '\\'
+#else
+#define PATHSEP '/'
+#endif
+
+/* Special purpose filepath compare function. Corresponds to the
+   order used in the TreeView view of the project's files. Returns true iff
+   path1 < path2 in the TreeView order. */
+static int f_path_compare(lua_State *L) {
+  const char *path1 = luaL_checkstring(L, 1);
+  const char *type1_s = luaL_checkstring(L, 2);
+  const char *path2 = luaL_checkstring(L, 3);
+  const char *type2_s = luaL_checkstring(L, 4);
+  const int len1 = strlen(path1), len2 = strlen(path2);
+  int type1 = strcmp(type1_s, "dir") != 0;
+  int type2 = strcmp(type2_s, "dir") != 0;
+  /* Find the index of the common part of the path. */
+  int offset = 0, i;
+  for (i = 0; i < len1 && i < len2; i++) {
+    if (path1[i] != path2[i]) break;
+    if (path1[i] == PATHSEP) {
+      offset = i + 1;
+    }
+  }
+  /* If a path separator is present in the name after the common part we consider
+     the entry like a directory. */
+  if (strchr(path1 + offset, PATHSEP)) {
+    type1 = 0;
+  }
+  if (strchr(path2 + offset, PATHSEP)) {
+    type2 = 0;
+  }
+  /* If types are different "dir" types comes before "file" types. */
+  if (type1 != type2) {
+    lua_pushboolean(L, type1 < type2);
+    return 1;
+  }
+  /* If types are the same compare the files' path alphabetically. */
+  int cfr = 0;
+  int len_min = (len1 < len2 ? len1 : len2);
+  for (int j = offset; j <= len_min; j++) {
+    if (path1[j] == path2[j]) continue;
+    if (path1[j] == 0 || path2[j] == 0) {
+      cfr = (path1[j] == 0);
+    } else if (path1[j] == PATHSEP || path2[j] == PATHSEP) {
+      /* For comparison we treat PATHSEP as if it was the string terminator. */
+      cfr = (path1[j] == PATHSEP);
+    } else {
+      cfr = (path1[j] < path2[j]);
+    }
+    break;
+  }
+  lua_pushboolean(L, cfr);
+  return 1;
+}
+
 
 static const luaL_Reg lib[] = {
   { "poll_event",          f_poll_event          },
@@ -664,6 +759,12 @@ static const luaL_Reg lib[] = {
   { "exec",                f_exec                },
   { "fuzzy_match",         f_fuzzy_match         },
   { "set_window_opacity",  f_set_window_opacity  },
+  { "watch_dir",           f_watch_dir           },
+  { "path_compare",        f_path_compare        },
+#if __linux__
+  { "watch_dir_add",       f_watch_dir_add       },
+  { "watch_dir_rm",        f_watch_dir_rm        },
+#endif
   { NULL, NULL }
 };
 

src/dirmonitor.c

@@ -0,0 +1,60 @@
+#include <stdio.h>
+#include <string.h>
+
+#include <SDL.h>
+
+#define DMON_IMPL
+#include "dmon.h"
+
+#include "dirmonitor.h"
+
+static void send_sdl_event(dmon_watch_id watch_id, dmon_action action, const char *filepath) {
+  SDL_Event ev;
+  const int size = strlen(filepath) + 1;
+  /* The string allocated below should be deallocated as soon as the event is
+     treated in the SDL main loop. */
+  char *new_filepath = malloc(size);
+  if (!new_filepath) return;
+  memcpy(new_filepath, filepath, size);
+#ifdef _WIN32
+  for (int i = 0; i < size; i++) {
+    if (new_filepath[i] == '/') {
+      new_filepath[i] = '\\';
+    }
+  }
+#endif
+  SDL_zero(ev);
+  ev.type = SDL_USEREVENT;
+  ev.user.code = ((watch_id.id & 0xffff) << 16) | (action & 0xffff);
+  ev.user.data1 = new_filepath;
+  SDL_PushEvent(&ev);
+}
+
+void dirmonitor_init() {
+  dmon_init();
+  /* In theory we should register our user event but since we
+     have just one type of user event this is not really needed. */
+  /* sdl_dmon_event_type = SDL_RegisterEvents(1); */
+}
+
+void dirmonitor_deinit() {
+  dmon_deinit();
+}
+
+void dirmonitor_watch_callback(dmon_watch_id watch_id, dmon_action action, const char *rootdir,
+  const char *filepath, const char *oldfilepath, void *user)
+{
+  (void) rootdir;
+  (void) user;
+  switch (action) {
+  case DMON_ACTION_MOVE:
+    send_sdl_event(watch_id, DMON_ACTION_DELETE, oldfilepath);
+    send_sdl_event(watch_id, DMON_ACTION_CREATE, filepath);
+    break;
+  case DMON_ACTION_MODIFY:
+    break;
+  default:
+    send_sdl_event(watch_id, action, filepath);
+  }
+}
+

src/dirmonitor.h

@@ -0,0 +1,14 @@
+#ifndef DIRMONITOR_H
+#define DIRMONITOR_H
+
+#include <stdint.h>
+
+#include "dmon.h"
+
+void dirmonitor_init();
+void dirmonitor_deinit();
+void dirmonitor_watch_callback(dmon_watch_id watch_id, dmon_action action, const char *rootdir,
+  const char *filepath, const char *oldfilepath, void *user);
+
+#endif
+

src/main.c

@@ -14,6 +14,8 @@
   #include <mach-o/dyld.h>
 #endif
 
+#include "dirmonitor.h"
+
 
 SDL_Window *window;
 
@@ -108,6 +110,8 @@ int main(int argc, char **argv) {
   SDL_DisplayMode dm;
   SDL_GetCurrentDisplayMode(0, &dm);
 
+  dirmonitor_init();
+
   window = SDL_CreateWindow(
     "", SDL_WINDOWPOS_UNDEFINED, SDL_WINDOWPOS_UNDEFINED, dm.w * 0.8, dm.h * 0.8,
     SDL_WINDOW_RESIZABLE | SDL_WINDOW_ALLOW_HIGHDPI | SDL_WINDOW_HIDDEN);
@@ -191,6 +195,7 @@ init_lua:
 
   lua_close(L);
   ren_free_window_resources();
+  dirmonitor_deinit();
 
   return EXIT_SUCCESS;
 }

src/meson.build

@@ -6,6 +6,7 @@ lite_sources = [
     'api/regex.c',
     'api/system.c',
     'api/process.c',
+    'dirmonitor.c',
     'renderer.c',
     'renwindow.c',
     'fontdesc.c',