Implement lazy loading of directories

When the number of files in a project directory is above the max
limit switch back to a mechanism to read directory content only
when the corresponding folder is expanded in the treeview.

When the command core:find-file is invoked the command core:open-file
is executed instead because the complete list of the project's
files is not available.

When a project search is done we search through all the files within
the project dir without indexing them.

Address issues #217 #203 #183.

data/core/commands/core.lua

@@ -66,6 +66,9 @@ command.add(nil, {
   end,
 
   ["core:find-file"] = function()
+    if core.project_files_limit then
+      return command.perform "core:open-file"
+    end
     local files = {}
     for dir, item in core.get_project_files() do
       if item.type == "file" then

data/core/init.lua

@@ -72,6 +72,7 @@ function core.set_project_dir(new_dir, change_project_fn)
     core.project_directories = {}
     core.add_project_directory(new_dir)
     core.project_files = {}
+    core.project_files_limit = false
     core.reschedule_project_scan()
     return true
   end
@@ -99,6 +100,57 @@ local function strip_trailing_slash(filename)
   return filename
 end
 
+local function compare_file(a, b)
+  return a.filename < b.filename
+end
+
+-- "root" will by an absolute path without trailing '/'
+-- "path" will be a path starting with '/' and without trailing '/'
+--    or the empty string.
+--    It will identifies a sub-path within "root.
+-- The current path location will therefore always be: root .. path.
+-- When recursing "root" will always be the same, only "path" will change.
+-- Returns a list of file "items". In eash item the "filename" will be the
+-- complete file path relative to "root" *without* the trailing '/'.
+local function get_directory_files(root, path, t, recursive, begin_hook)
+  if begin_hook then begin_hook() end
+  local size_limit = config.file_size_limit * 10e5
+  local all = system.list_dir(root .. path) or {}
+  local dirs, files = {}, {}
+
+  local entries_count = 0
+  local max_entries = config.max_project_files
+  for _, file in ipairs(all) do
+    if not common.match_pattern(file, config.ignore_files) then
+      local file = path .. PATHSEP .. file
+      local info = system.get_file_info(root .. file)
+      if info and info.size < size_limit then
+        info.filename = strip_leading_path(file)
+        table.insert(info.type == "dir" and dirs or files, info)
+        entries_count = entries_count + 1
+        if recursive and entries_count > max_entries then return nil, entries_count end
+      end
+    end
+  end
+
+  table.sort(dirs, compare_file)
+  for _, f in ipairs(dirs) do
+    table.insert(t, f)
+    if recursive and entries_count <= max_entries then
+      local subdir_t, subdir_count = get_directory_files(root, PATHSEP .. f.filename, t, recursive)
+      entries_count = entries_count + subdir_count
+      f.scanned = true
+    end
+  end
+
+  table.sort(files, compare_file)
+  for _, f in ipairs(files) do
+    table.insert(t, f)
+  end
+
+  return t, entries_count
+end
+
 local function project_scan_thread()
   local function diff_files(a, b)
     if #a ~= #b then return true end
@@ -110,65 +162,16 @@ local function project_scan_thread()
     end
   end
 
-  local function compare_file(a, b)
-    return a.filename < b.filename
-  end
-
-  -- "root" will by an absolute path without trailing '/'
-  -- "path" will be a path starting with '/' and without trailing '/'
-  --    or the empty string.
-  --    It will identifies a sub-path within "root.
-  -- The current path location will therefore always be: root .. path.
-  -- When recursing "root" will always be the same, only "path" will change.
-  -- Returns a list of file "items". In eash item the "filename" will be the
-  -- complete file path relative to "root" *without* the trailing '/'.
-  local function get_files(root, path, t)
-    coroutine.yield()
-    t = t or {}
-    local size_limit = config.file_size_limit * 10e5
-    local all = system.list_dir(root .. path) or {}
-    local dirs, files = {}, {}
-
-    local entries_count = 0
-    local max_entries = config.max_project_files
-    for _, file in ipairs(all) do
-      if not common.match_pattern(file, config.ignore_files) then
-        local file = path .. PATHSEP .. file
-        local info = system.get_file_info(root .. file)
-        if info and info.size < size_limit then
-          info.filename = strip_leading_path(file)
-          table.insert(info.type == "dir" and dirs or files, info)
-          entries_count = entries_count + 1
-          if entries_count > max_entries then break end
-        end
-      end
-    end
-
-    table.sort(dirs, compare_file)
-    for _, f in ipairs(dirs) do
-      table.insert(t, f)
-      if entries_count <= max_entries then
-        local subdir_t, subdir_count = get_files(root, PATHSEP .. f.filename, t)
-        entries_count = entries_count + subdir_count
-      end
-    end
-
-    table.sort(files, compare_file)
-    for _, f in ipairs(files) do
-      table.insert(t, f)
-    end
-
-    return t, entries_count
-  end
-
   while true do
     -- get project files and replace previous table if the new table is
     -- different
-    for i = 1, #core.project_directories do
+    local i = 1
+    while not core.project_files_limit and i <= #core.project_directories do
       local dir = core.project_directories[i]
-      local t, entries_count = get_files(dir.name, "")
+      local t, entries_count = get_directory_files(dir.name, "", {}, true)
       if diff_files(dir.files, t) then
         if entries_count > config.max_project_files then
+          core.project_files_limit = true
           core.status_view:show_message("!", style.accent,
             "Too many files in project directory: stopping reading at "..
             config.max_project_files.." files according to config.max_project_files. "..
@@ -182,6 +185,7 @@ local function project_scan_thread()
       if dir.name == core.project_dir then
         core.project_files = dir.files
       end
+      i = i + 1
     end
 
     -- wait for next scan
@@ -190,6 +194,46 @@ local function project_scan_thread()
 end
 
 
+function core.scan_project_folder(dirname, filename)
+  for _, dir in ipairs(core.project_directories) do
+    if dir.name == dirname then
+      for i, file in ipairs(dir.files) do
+        local file = dir.files[i]
+        if file.filename == filename then
+          if file.scanned then return end
+          local new_files = get_directory_files(dirname, PATHSEP .. filename, {})
+          for j, new_file in ipairs(new_files) do
+            table.insert(dir.files, i + j, new_file)
+          end
+          file.scanned = true
+          return
+        end
+      end
+    end
+  end
+end
+
+
+local function find_project_files_co(root, path)
+  local size_limit = config.file_size_limit * 10e5
+  local all = system.list_dir(root .. path) or {}
+  for _, file in ipairs(all) do
+    if not common.match_pattern(file, config.ignore_files) then
+      local file = path .. PATHSEP .. file
+      local info = system.get_file_info(root .. file)
+      if info and info.size < size_limit then
+        info.filename = strip_leading_path(file)
+        if info.type == "file" then
+          coroutine.yield(root, info)
+        else
+          find_project_files_co(root, PATHSEP .. info.filename)
+        end
+      end
+    end
+  end
+end
+
+
 local function project_files_iter(state)
   local dir = core.project_directories[state.dir_index]
   state.file_index = state.file_index + 1
@@ -204,17 +248,27 @@ end
 
 
 function core.get_project_files()
-  local state = { dir_index = 1, file_index = 0 }
+  if core.project_files_limit then
-  return project_files_iter, state
+    return coroutine.wrap(function()
+      for _, dir in ipairs(core.project_directories) do
+        find_project_files_co(dir.name, "")
+      end
+    end)
+  else
+    local state = { dir_index = 1, file_index = 0 }
+    return project_files_iter, state
+  end
 end
 
 
 function core.project_files_number()
-  local n = 0
+  if not core.project_files_limit then
-  for i = 1, #core.project_directories do
+    local n = 0
-    n = n + #core.project_directories[i].files
+    for i = 1, #core.project_directories do
+      n = n + #core.project_directories[i].files
+    end
+    return n
   end
-  return n
 end
 
 

data/plugins/projectsearch.lua

@@ -170,12 +170,17 @@ function ResultsView:draw()
   local ox, oy = self:get_content_offset()
   local x, y = ox + style.padding.x, oy + style.padding.y
   local files_number = core.project_files_number()
-  local per = self.last_file_idx / files_number
+  local per = files_number and self.last_file_idx / files_number or 1
   local text
   if self.searching then
-    text = string.format("Searching %d%% (%d of %d files, %d matches) for %q...",
+    if files_number then
-      per * 100, self.last_file_idx, files_number,
+      text = string.format("Searching %d%% (%d of %d files, %d matches) for %q...",
-      #self.results, self.query)
+        per * 100, self.last_file_idx, files_number,
+        #self.results, self.query)
+    else
+      text = string.format("Searching (%d files, %d matches) for %q...",
+        self.last_file_idx, #self.results, self.query)
+    end
   else
     text = string.format("Found %d matches for %q",
       #self.results, self.query)

data/plugins/treeview.lua

@@ -93,6 +93,13 @@ function TreeView:get_item_height()
 end
 
 
+function TreeView:invalidate_cache(dirname)
+  for _, v in pairs(self.cache[dirname]) do
+    v.skip = nil
+  end
+end
+
+
 function TreeView:check_cache()
   -- invalidate cache's skip values if project_files has changed
   for i = 1, #core.project_directories do
@@ -102,9 +109,7 @@ function TreeView:check_cache()
       self.last[dir.name] = dir.files
     else
       if dir.files ~= last_files then
-        for _, v in pairs(self.cache[dir.name]) do
+        self:invalidate_cache(dir.name)
-          v.skip = nil
-        end
         self.last[dir.name] = dir.files
       end
     end
@@ -208,17 +213,25 @@ function TreeView:on_mouse_pressed(button, x, y, clicks)
   if caught then
     return
   end
-  if not self.hovered_item then
+  local hovered_item = self.hovered_item
+  if not hovered_item then
     return
-  elseif self.hovered_item.type == "dir" then
+  elseif hovered_item.type == "dir" then
     if keymap.modkeys["ctrl"] and button == "left" then
-      create_directory_in(self.hovered_item)
+      create_directory_in(hovered_item)
     else
-      self.hovered_item.expanded = not self.hovered_item.expanded
+      if core.project_files_limit and not hovered_item.expanded then
+        local filename, abs_filename = hovered_item.filename, hovered_item.abs_filename
+        local index = string.find(abs_filename, filename, 1, true)
+        local dirname = string.sub(abs_filename, 1, index - 2)
+        core.scan_project_folder(dirname, filename)
+        self:invalidate_cache(dirname)
+      end
+      hovered_item.expanded = not hovered_item.expanded
     end
   else
     core.try(function()
-      local doc_filename = common.relative_path(core.project_dir, self.hovered_item.abs_filename)
+      local doc_filename = common.relative_path(core.project_dir, hovered_item.abs_filename)
       core.root_view:open_doc(core.open_doc(doc_filename))
     end)
   end