Implement lazy loading of directories

When the number of files in a project directory is above the max
limit switch back to a mechanism to read directory content only
when the corresponding folder is expanded in the treeview.

When the command core:find-file is invoked the command core:open-file
is executed instead because the complete list of the project's
files is not available.

When a project search is done we search through all the files within
the project dir without indexing them.

Address issues #217 #203 #183.

data/core/commands/core.lua

@@ -66,6 +66,9 @@ command.add(nil, {
   end,
 
   ["core:find-file"] = function()
+    if core.project_files_limit then
+      return command.perform "core:open-file"
+    end
     local files = {}
     for dir, item in core.get_project_files() do
       if item.type == "file" then

data/core/init.lua

@@ -72,6 +72,7 @@ function core.set_project_dir(new_dir, change_project_fn)
     core.project_directories = {}
     core.add_project_directory(new_dir)
     core.project_files = {}
+    core.project_files_limit = false
     core.reschedule_project_scan()
     return true
   end
@@ -99,17 +100,6 @@ local function strip_trailing_slash(filename)
   return filename
 end
 
-local function project_scan_thread()
-  local function diff_files(a, b)
-    if #a ~= #b then return true end
-    for i, v in ipairs(a) do
-      if b[i].filename ~= v.filename
-      or b[i].modified ~= v.modified then
-        return true
-      end
-    end
-  end
-
 local function compare_file(a, b)
   return a.filename < b.filename
 end
@@ -122,9 +112,8 @@ local function project_scan_thread()
 -- When recursing "root" will always be the same, only "path" will change.
 -- Returns a list of file "items". In eash item the "filename" will be the
 -- complete file path relative to "root" *without* the trailing '/'.
-  local function get_files(root, path, t)
+local function get_directory_files(root, path, t, recursive, begin_hook)
-    coroutine.yield()
+  if begin_hook then begin_hook() end
-    t = t or {}
   local size_limit = config.file_size_limit * 10e5
   local all = system.list_dir(root .. path) or {}
   local dirs, files = {}, {}
@@ -139,7 +128,7 @@ local function project_scan_thread()
         info.filename = strip_leading_path(file)
         table.insert(info.type == "dir" and dirs or files, info)
         entries_count = entries_count + 1
-          if entries_count > max_entries then break end
+        if recursive and entries_count > max_entries then return nil, entries_count end
       end
     end
   end
@@ -147,9 +136,10 @@ local function project_scan_thread()
   table.sort(dirs, compare_file)
   for _, f in ipairs(dirs) do
     table.insert(t, f)
-      if entries_count <= max_entries then
+    if recursive and entries_count <= max_entries then
-        local subdir_t, subdir_count = get_files(root, PATHSEP .. f.filename, t)
+      local subdir_t, subdir_count = get_directory_files(root, PATHSEP .. f.filename, t, recursive)
       entries_count = entries_count + subdir_count
+      f.scanned = true
     end
   end
 
@@ -161,14 +151,27 @@ local function project_scan_thread()
   return t, entries_count
 end
 
+local function project_scan_thread()
+  local function diff_files(a, b)
+    if #a ~= #b then return true end
+    for i, v in ipairs(a) do
+      if b[i].filename ~= v.filename
+      or b[i].modified ~= v.modified then
+        return true
+      end
+    end
+  end
+
   while true do
     -- get project files and replace previous table if the new table is
     -- different
-    for i = 1, #core.project_directories do
+    local i = 1
+    while not core.project_files_limit and i <= #core.project_directories do
       local dir = core.project_directories[i]
-      local t, entries_count = get_files(dir.name, "")
+      local t, entries_count = get_directory_files(dir.name, "", {}, true)
       if diff_files(dir.files, t) then
         if entries_count > config.max_project_files then
+          core.project_files_limit = true
           core.status_view:show_message("!", style.accent,
             "Too many files in project directory: stopping reading at "..
             config.max_project_files.." files according to config.max_project_files. "..
@@ -182,6 +185,7 @@ local function project_scan_thread()
       if dir.name == core.project_dir then
         core.project_files = dir.files
       end
+      i = i + 1
     end
 
     -- wait for next scan
@@ -190,6 +194,46 @@ local function project_scan_thread()
 end
 
 
+function core.scan_project_folder(dirname, filename)
+  for _, dir in ipairs(core.project_directories) do
+    if dir.name == dirname then
+      for i, file in ipairs(dir.files) do
+        local file = dir.files[i]
+        if file.filename == filename then
+          if file.scanned then return end
+          local new_files = get_directory_files(dirname, PATHSEP .. filename, {})
+          for j, new_file in ipairs(new_files) do
+            table.insert(dir.files, i + j, new_file)
+          end
+          file.scanned = true
+          return
+        end
+      end
+    end
+  end
+end
+
+
+local function find_project_files_co(root, path)
+  local size_limit = config.file_size_limit * 10e5
+  local all = system.list_dir(root .. path) or {}
+  for _, file in ipairs(all) do
+    if not common.match_pattern(file, config.ignore_files) then
+      local file = path .. PATHSEP .. file
+      local info = system.get_file_info(root .. file)
+      if info and info.size < size_limit then
+        info.filename = strip_leading_path(file)
+        if info.type == "file" then
+          coroutine.yield(root, info)
+        else
+          find_project_files_co(root, PATHSEP .. info.filename)
+        end
+      end
+    end
+  end
+end
+
+
 local function project_files_iter(state)
   local dir = core.project_directories[state.dir_index]
   state.file_index = state.file_index + 1
@@ -204,18 +248,28 @@ end
 
 
 function core.get_project_files()
+  if core.project_files_limit then
+    return coroutine.wrap(function()
+      for _, dir in ipairs(core.project_directories) do
+        find_project_files_co(dir.name, "")
+      end
+    end)
+  else
     local state = { dir_index = 1, file_index = 0 }
     return project_files_iter, state
   end
+end
 
 
 function core.project_files_number()
+  if not core.project_files_limit then
     local n = 0
     for i = 1, #core.project_directories do
       n = n + #core.project_directories[i].files
     end
     return n
   end
+end
 
 
 -- create a directory using mkdir but may need to create the parent

data/plugins/projectsearch.lua

@@ -170,12 +170,17 @@ function ResultsView:draw()
   local ox, oy = self:get_content_offset()
   local x, y = ox + style.padding.x, oy + style.padding.y
   local files_number = core.project_files_number()
-  local per = self.last_file_idx / files_number
+  local per = files_number and self.last_file_idx / files_number or 1
   local text
   if self.searching then
+    if files_number then
       text = string.format("Searching %d%% (%d of %d files, %d matches) for %q...",
         per * 100, self.last_file_idx, files_number,
         #self.results, self.query)
+    else
+      text = string.format("Searching (%d files, %d matches) for %q...",
+        self.last_file_idx, #self.results, self.query)
+    end
   else
     text = string.format("Found %d matches for %q",
       #self.results, self.query)

data/plugins/treeview.lua

@@ -93,6 +93,13 @@ function TreeView:get_item_height()
 end
 
 
+function TreeView:invalidate_cache(dirname)
+  for _, v in pairs(self.cache[dirname]) do
+    v.skip = nil
+  end
+end
+
+
 function TreeView:check_cache()
   -- invalidate cache's skip values if project_files has changed
   for i = 1, #core.project_directories do
@@ -102,9 +109,7 @@ function TreeView:check_cache()
       self.last[dir.name] = dir.files
     else
       if dir.files ~= last_files then
-        for _, v in pairs(self.cache[dir.name]) do
+        self:invalidate_cache(dir.name)
-          v.skip = nil
-        end
         self.last[dir.name] = dir.files
       end
     end
@@ -208,17 +213,25 @@ function TreeView:on_mouse_pressed(button, x, y, clicks)
   if caught then
     return
   end
-  if not self.hovered_item then
+  local hovered_item = self.hovered_item
+  if not hovered_item then
     return
-  elseif self.hovered_item.type == "dir" then
+  elseif hovered_item.type == "dir" then
     if keymap.modkeys["ctrl"] and button == "left" then
-      create_directory_in(self.hovered_item)
+      create_directory_in(hovered_item)
     else
-      self.hovered_item.expanded = not self.hovered_item.expanded
+      if core.project_files_limit and not hovered_item.expanded then
+        local filename, abs_filename = hovered_item.filename, hovered_item.abs_filename
+        local index = string.find(abs_filename, filename, 1, true)
+        local dirname = string.sub(abs_filename, 1, index - 2)
+        core.scan_project_folder(dirname, filename)
+        self:invalidate_cache(dirname)
+      end
+      hovered_item.expanded = not hovered_item.expanded
     end
   else
     core.try(function()
-      local doc_filename = common.relative_path(core.project_dir, self.hovered_item.abs_filename)
+      local doc_filename = common.relative_path(core.project_dir, hovered_item.abs_filename)
       core.root_view:open_doc(core.open_doc(doc_filename))
     end)
   end