Asynchronous Reads for Dirmonitor (#930)

Change dirmonitor reads to be synchronous, in a secondary thread.

data/core/dirwatch.lua

@@ -85,7 +85,9 @@ end
 
 -- designed to be run inside a coroutine.
 function dirwatch:check(change_callback, scan_time, wait_time)
+  local had_change = false
   self.monitor:check(function(id)
+    had_change = true
     if PLATFORM == "Windows" then
       change_callback(common.dirname(self.windows_watch_top .. PATHSEP .. id))
     elseif self.reverse_watched[id] then
@@ -98,6 +100,7 @@ function dirwatch:check(change_callback, scan_time, wait_time)
       local new_modified = system.get_file_info(directory).modified
       if old_modified < new_modified then
         change_callback(directory)
+        had_change = true
         self.scanned[directory] = new_modified
       end
     end
@@ -106,6 +109,7 @@ function dirwatch:check(change_callback, scan_time, wait_time)
       start_time = system.get_time()
     end
   end
+  return had_change
 end
 
 

data/core/init.lua

@@ -269,7 +269,7 @@ function core.add_project_directory(path)
   -- time; the watch will yield in this coroutine after 0.01 second, for 0.1 seconds.
   topdir.watch_thread = core.add_thread(function()
     while true do
-      topdir.watch:check(function(target)
+      local changed = topdir.watch:check(function(target)
         if target == topdir.name then return refresh_directory(topdir) end
         local dirpath = target:sub(#topdir.name + 2)
         local abs_dirpath = topdir.name .. PATHSEP .. dirpath
@@ -280,7 +280,7 @@ function core.add_project_directory(path)
         end
         return refresh_directory(topdir, dirpath)
       end, 0.01, 0.01)
-      coroutine.yield(0.05)
+      coroutine.yield(changed and 0.05 or 0)
     end
   end)
 
@@ -1120,14 +1120,6 @@ function core.try(fn, ...)
   return false, err
 end
 
-local scheduled_rescan = {}
-
-function core.has_pending_rescan()
-  for _ in pairs(scheduled_rescan) do
-    return true
-  end
-end
-
 function core.on_event(type, ...)
   local did_keymap = false
   if type == "textinput" then
@@ -1274,8 +1266,8 @@ function core.run()
   local idle_iterations = 0
   while true do
     core.frame_start = system.get_time()
+    local need_more_work = run_threads()
     local did_redraw = core.step()
-    local need_more_work = run_threads() or core.has_pending_rescan()
     if core.restart_request or core.quit_request then break end
     if not did_redraw and not need_more_work then
       idle_iterations = idle_iterations + 1

src/api/dirmonitor.c

@@ -1,78 +1,115 @@
 #include "api.h"
+#include <SDL.h>
 #include <stdlib.h>
-#ifdef DIRMONITOR_WIN32
-  #include <windows.h>
-#endif
 #include <unistd.h>
 #include <errno.h>
-#include <dirent.h>
-#include <fcntl.h>
 #include <string.h>
 #include <stdbool.h>
 
-#ifndef DIRMONITOR_BACKEND
+static unsigned int DIR_EVENT_TYPE = 0;
-#error No dirmonitor backend defined
-#endif
 
-#define GLUE_HELPER(x, y) x##y
+struct dirmonitor {
-#define GLUE(x, y) GLUE_HELPER(x, y)
+  SDL_Thread* thread;
+  SDL_mutex* mutex;
+  char buffer[64512];
+  volatile int length;
+  struct dirmonitor_internal* internal;
+};
 
-#define init_dirmonitor GLUE(init_dirmonitor_, DIRMONITOR_BACKEND)
-#define deinit_dirmonitor GLUE(deinit_dirmonitor_, DIRMONITOR_BACKEND)
-#define check_dirmonitor GLUE(check_dirmonitor_, DIRMONITOR_BACKEND)
-#define add_dirmonitor GLUE(add_dirmonitor_, DIRMONITOR_BACKEND)
-#define remove_dirmonitor GLUE(remove_dirmonitor_, DIRMONITOR_BACKEND)
 
-struct dirmonitor {}; // dirmonitor struct is defined in each backend
+struct dirmonitor_internal* init_dirmonitor();
+void deinit_dirmonitor(struct dirmonitor_internal*);
+int get_changes_dirmonitor(struct dirmonitor_internal*, char*, int);
+int translate_changes_dirmonitor(struct dirmonitor_internal*, char*, int, int (*)(int, const char*, void*), void*);
+int add_dirmonitor(struct dirmonitor_internal*, const char*);
+void remove_dirmonitor(struct dirmonitor_internal*, int);
 
-// define functions so we know their signature
-struct dirmonitor* init_dirmonitor();
-void deinit_dirmonitor(struct dirmonitor*);
-int check_dirmonitor(struct dirmonitor*, int (*)(int, const char*, void*), void*);
-int add_dirmonitor(struct dirmonitor*, const char*);
-void remove_dirmonitor(struct dirmonitor*, int);
 
 static int f_check_dir_callback(int watch_id, const char* path, void* L) {
   lua_pushvalue(L, -1);
-  #ifdef DIRMONITOR_WIN32
+  if (path)
-    char buffer[PATH_MAX*4];
+    lua_pushlstring(L, path, watch_id);
-    int count = WideCharToMultiByte(CP_UTF8, 0, (WCHAR*)path, watch_id, buffer, PATH_MAX*4 - 1, NULL, NULL);
+  else
-    lua_pushlstring(L, buffer, count);
-  #else
     lua_pushnumber(L, watch_id);
-  #endif
   lua_call(L, 1, 1);
   int result = lua_toboolean(L, -1);
   lua_pop(L, 1);
   return !result;
 }
 
+
+static int dirmonitor_check_thread(void* data) {
+  struct dirmonitor* monitor = data;
+  while (monitor->length >= 0) {
+    if (monitor->length == 0) {
+      int result = get_changes_dirmonitor(monitor->internal, monitor->buffer, sizeof(monitor->buffer));
+      SDL_LockMutex(monitor->mutex);
+      if (monitor->length == 0)
+        monitor->length = result;
+      SDL_UnlockMutex(monitor->mutex);
+    }
+    SDL_Delay(1);
+    SDL_Event event = { .type = DIR_EVENT_TYPE };
+    SDL_PushEvent(&event);
+  }
+  return 0;
+}
+
+
 static int f_dirmonitor_new(lua_State* L) {
-  struct dirmonitor** monitor = lua_newuserdata(L, sizeof(struct dirmonitor**));
+  if (DIR_EVENT_TYPE == 0)
-  *monitor = init_dirmonitor();
+    DIR_EVENT_TYPE = SDL_RegisterEvents(1);
+  struct dirmonitor* monitor = lua_newuserdata(L, sizeof(struct dirmonitor));
   luaL_setmetatable(L, API_TYPE_DIRMONITOR);
+  memset(monitor, 0, sizeof(struct dirmonitor));
+  monitor->internal = init_dirmonitor();
+  if (monitor->internal)
+    monitor->thread = SDL_CreateThread(dirmonitor_check_thread, "dirmonitor_check_thread", monitor);
   return 1;
 }
 
+
 static int f_dirmonitor_gc(lua_State* L) {
-  deinit_dirmonitor(*((struct dirmonitor**)luaL_checkudata(L, 1, API_TYPE_DIRMONITOR)));
+  struct dirmonitor* monitor = luaL_checkudata(L, 1, API_TYPE_DIRMONITOR);
+  SDL_LockMutex(monitor->mutex);
+  monitor->length = -1;
+  deinit_dirmonitor(monitor->internal);
+  SDL_UnlockMutex(monitor->mutex);
+  SDL_WaitThread(monitor->thread, NULL);
+  free(monitor->internal);
+  SDL_DestroyMutex(monitor->mutex);
   return 0;
 }
 
+
 static int f_dirmonitor_watch(lua_State *L) {
-  lua_pushnumber(L, add_dirmonitor(*(struct dirmonitor**)luaL_checkudata(L, 1, API_TYPE_DIRMONITOR), luaL_checkstring(L, 2)));
+  lua_pushnumber(L, add_dirmonitor(((struct dirmonitor*)luaL_checkudata(L, 1, API_TYPE_DIRMONITOR))->internal, luaL_checkstring(L, 2)));
   return 1;
 }
 
+
 static int f_dirmonitor_unwatch(lua_State *L) {
-  remove_dirmonitor(*(struct dirmonitor**)luaL_checkudata(L, 1, API_TYPE_DIRMONITOR), lua_tonumber(L, 2));
+  remove_dirmonitor(((struct dirmonitor*)luaL_checkudata(L, 1, API_TYPE_DIRMONITOR))->internal, lua_tonumber(L, 2));
   return 0;
 }
 
+
 static int f_dirmonitor_check(lua_State* L) {
-  lua_pushnumber(L, check_dirmonitor(*(struct dirmonitor**)luaL_checkudata(L, 1, API_TYPE_DIRMONITOR), f_check_dir_callback, L));
+  struct dirmonitor* monitor = luaL_checkudata(L, 1, API_TYPE_DIRMONITOR);
+  SDL_LockMutex(monitor->mutex);
+  if (monitor->length < 0) 
+    lua_pushnil(L);
+  else if (monitor->length > 0) {
+    if (translate_changes_dirmonitor(monitor->internal, monitor->buffer, monitor->length, f_check_dir_callback, L) == 0)
+      monitor->length = 0;
+    lua_pushboolean(L, 1);
+  } else
+    lua_pushboolean(L, 0);
+  SDL_UnlockMutex(monitor->mutex);
   return 1;
 }
+
+
 static const luaL_Reg dirmonitor_lib[] = {
   { "new",      f_dirmonitor_new         },
   { "__gc",     f_dirmonitor_gc          },
@@ -82,6 +119,7 @@ static const luaL_Reg dirmonitor_lib[] = {
   {NULL, NULL}
 };
 
+
 int luaopen_dirmonitor(lua_State* L) {
   luaL_newmetatable(L, API_TYPE_DIRMONITOR);
   luaL_setfuncs(L, dirmonitor_lib, 0);

src/api/dirmonitor/dummy.c

@@ -1,22 +1,8 @@
 #include <stdlib.h>
 
-struct dirmonitor {
+struct dirmonitor_internal* init_dirmonitor() { return NULL; }
-};
+void deinit_dirmonitor(struct dirmonitor_internal*) { }
-
+int get_changes_dirmonitor(struct dirmonitor_internal*, char*, size_t) { return -1; }
-struct dirmonitor* init_dirmonitor_dummy() {
+int translate_changes_dirmonitor(struct dirmonitor_internal*, char*, int, int (*)(int, const char*, void*), void*) { return -1; }
-  return NULL;
+int add_dirmonitor(struct dirmonitor_internal*, const char*) { return -1; }
-}
+void remove_dirmonitor(struct dirmonitor_internal*, int) { }
-
-void deinit_dirmonitor_dummy(struct dirmonitor* monitor) {
-}
-
-int check_dirmonitor_dummy(struct dirmonitor* monitor, int (*change_callback)(int, const char*, void*), void* data) {
-  return -1;
-}
-
-int add_dirmonitor_dummy(struct dirmonitor* monitor, const char* path) {
-  return -1;
-}
-
-void remove_dirmonitor_dummy(struct dirmonitor* monitor, int fd) {
-}

src/api/dirmonitor/inotify.c

@@ -1,58 +1,53 @@
 #include <sys/inotify.h>
-#include <limits.h>
+#include <sys/select.h>
-#include <unistd.h>
 #include <stdlib.h>
-#include <string.h>
+#include <unistd.h>
-#include <errno.h>
-#include <fcntl.h>
 
-struct dirmonitor {
+
+struct dirmonitor_internal {
   int fd;
+  // a pipe is used to wake the thread in case of exit
+  int sig[2];
 };
 
-struct dirmonitor* init_dirmonitor_inotify() {
-  struct dirmonitor* monitor = calloc(sizeof(struct dirmonitor), 1);
 
+struct dirmonitor_internal* init_dirmonitor() {
+  struct dirmonitor_internal* monitor = calloc(sizeof(struct dirmonitor_internal), 1);
   monitor->fd = inotify_init();
-  fcntl(monitor->fd, F_SETFL, O_NONBLOCK);
+  pipe(monitor->sig);
-
-
   return monitor;
 }
 
-void deinit_dirmonitor_inotify(struct dirmonitor* monitor) {
+
+void deinit_dirmonitor(struct dirmonitor_internal* monitor) {
+  close(monitor->sig[0]);
+  close(monitor->sig[1]);
   close(monitor->fd);
-  free(monitor);
 }
 
-int check_dirmonitor_inotify(struct dirmonitor* monitor, int (*change_callback)(int, const char*, void*), void* data) {
-  char buf[PATH_MAX + sizeof(struct inotify_event)];
-  ssize_t offset = 0;
 
-  while (1) {
+int get_changes_dirmonitor(struct dirmonitor_internal* monitor, char* buffer, int length) {
-    ssize_t len = read(monitor->fd, &buf[offset], sizeof(buf) - offset);
+  fd_set set;
-
+  FD_ZERO(&set);
-    if (len == -1 && errno != EAGAIN) {
+  FD_SET(monitor->fd, &set);
-      return errno;
+  FD_SET(monitor->sig[0], &set);
-    }
+  select(FD_SETSIZE, &set, NULL, NULL, NULL);
-
+  return read(monitor->fd, buffer, length);
-    if (len <= 0) {
-      return 0;
-    }
-
-    while (len > sizeof(struct inotify_event) && len >= ((struct inotify_event*)buf)->len + sizeof(struct inotify_event)) {
-      change_callback(((const struct inotify_event *)buf)->wd, NULL, data);
-      len -= sizeof(struct inotify_event) + ((struct inotify_event*)buf)->len;
-      memmove(buf, &buf[sizeof(struct inotify_event) + ((struct inotify_event*)buf)->len], len);
-      offset = len;
-    }
-  }
 }
 
-int add_dirmonitor_inotify(struct dirmonitor* monitor, const char* path) {
+
+int translate_changes_dirmonitor(struct dirmonitor_internal* monitor, char* buffer, int length, int (*change_callback)(int, const char*, void*), void* data) {
+  for (struct inotify_event* info = (struct inotify_event*)buffer; (char*)info < buffer + length; info = (struct inotify_event*)((char*)info + sizeof(struct inotify_event)))
+    change_callback(info->wd, NULL, data);
+  return 0;
+}
+
+
+int add_dirmonitor(struct dirmonitor_internal* monitor, const char* path) {
   return inotify_add_watch(monitor->fd, path, IN_CREATE | IN_DELETE | IN_MOVED_FROM | IN_MOVED_TO);
 }
 
-void remove_dirmonitor_inotify(struct dirmonitor* monitor, int fd) {
+
+void remove_dirmonitor(struct dirmonitor_internal* monitor, int fd) {
   inotify_rm_watch(monitor->fd, fd);
-}
+}

src/api/dirmonitor/kqueue.c

@@ -5,40 +5,41 @@
 #include <fcntl.h>
 #include <unistd.h>
 
-struct dirmonitor {
+struct dirmonitor_internal {
   int fd;
 };
 
-struct dirmonitor* init_dirmonitor_kqueue() {
+
-  struct dirmonitor* monitor = calloc(sizeof(struct dirmonitor), 1);
+struct dirmonitor_internal* init_dirmonitor() {
+  struct dirmonitor_internal* monitor = calloc(sizeof(struct dirmonitor_internal), 1);
   monitor->fd = kqueue();
   return monitor;
 }
 
-void deinit_dirmonitor_kqueue(struct dirmonitor* monitor) {
+
+void deinit_dirmonitor(struct dirmonitor_internal* monitor) {
   close(monitor->fd);
-  free(monitor);
 }
 
-int check_dirmonitor_kqueue(struct dirmonitor* monitor, int (*change_callback)(int, const char*, void*), void* data) {
-  struct kevent event;
-  while (1) {
-    struct timespec tm = {0};
-    int nev = kevent(monitor->fd, NULL, 0, &event, 1, &tm);
 
-    if (nev == -1) {
+int get_changes_dirmonitor(struct dirmonitor_internal* monitor, char* buffer, int buffer_size) {
-      return errno;
+  int nev = kevent(monitor->fd, NULL, 0, (struct kevent*)buffer, buffer_size / sizeof(kevent), NULL);
-    }
+  if (nev == -1)
-
+    return -1;
-    if (nev <= 0) {
+  if (nev <= 0)
-      return 0;
+    return 0;
-    }
+  return nev * sizeof(struct kevent);
-
-    change_callback(event.ident, NULL, data);
-  }
 }
 
-int add_dirmonitor_kqueue(struct dirmonitor* monitor, const char* path) {
+
+int translate_changes_dirmonitor(struct dirmonitor_internal* monitor, char* buffer, int buffer_size, int (*change_callback)(int, const char*, void*), void* data) {
+  for (struct kevent* info = (struct kevent*)buffer; (char*)info < buffer + buffer_size; info = (struct kevent*)(((char*)info) + sizeof(kevent)))
+    change_callback(info->ident, NULL, data);
+  return 0;
+}
+
+
+int add_dirmonitor(struct dirmonitor_internal* monitor, const char* path) {
   int fd = open(path, O_RDONLY);
   struct kevent change;
 
@@ -48,6 +49,7 @@ int add_dirmonitor_kqueue(struct dirmonitor* monitor, const char* path) {
   return fd;
 }
 
-void remove_dirmonitor_kqueue(struct dirmonitor* monitor, int fd) {
+
+void remove_dirmonitor(struct dirmonitor_internal* monitor, int fd) {
   close(fd);
-}
+}

src/api/dirmonitor/win32.c

@@ -1,77 +1,62 @@
-#include <stdbool.h>
 #include <windows.h>
 
-struct dirmonitor {
+
+struct dirmonitor_internal {
   HANDLE handle;
-  char buffer[64512];
-  OVERLAPPED overlapped;
-  bool running;
 };
 
-struct dirmonitor* init_dirmonitor_win32() {
-  struct dirmonitor* monitor = calloc(sizeof(struct dirmonitor), 1);
 
-  return monitor;
+int get_changes_dirmonitor(struct dirmonitor_internal* monitor, char* buffer, int buffer_size) {
-}
+  HANDLE handle = monitor->handle;
-
+  if (handle && handle != INVALID_HANDLE_VALUE) {
-static void close_monitor_handle(struct dirmonitor* monitor) {
+    DWORD bytes_transferred;
-  if (monitor->handle) {
+    if (ReadDirectoryChangesW(handle, buffer, buffer_size, TRUE,  FILE_NOTIFY_CHANGE_FILE_NAME | FILE_NOTIFY_CHANGE_DIR_NAME, &bytes_transferred, NULL, NULL) == 0)
-      if (monitor->running) {
+      return 0;
-        BOOL result = CancelIoEx(monitor->handle, &monitor->overlapped);
+    return bytes_transferred;
-        DWORD error = GetLastError();
-        if (result == TRUE || error != ERROR_NOT_FOUND) {
-          DWORD bytes_transferred;
-          GetOverlappedResult( monitor->handle, &monitor->overlapped, &bytes_transferred, TRUE );
-        }
-        monitor->running = false;
-      }
-      CloseHandle(monitor->handle);
-    }
-  monitor->handle = NULL;
-}
-
-void deinit_dirmonitor_win32(struct dirmonitor* monitor) {
-  close_monitor_handle(monitor);
-  free(monitor);
-}
-
-int check_dirmonitor_win32(struct dirmonitor* monitor, int (*change_callback)(int, const char*, void*), void* data) {
-  if (!monitor->running) {
-    if (ReadDirectoryChangesW(monitor->handle, monitor->buffer, sizeof(monitor->buffer), TRUE,  FILE_NOTIFY_CHANGE_FILE_NAME | FILE_NOTIFY_CHANGE_DIR_NAME, NULL, &monitor->overlapped, NULL) == 0) {
-      return GetLastError();
-    }
-    monitor->running = true;
   }
-
-  DWORD bytes_transferred;
-
-  if (!GetOverlappedResult(monitor->handle, &monitor->overlapped, &bytes_transferred, FALSE)) {
-    int error = GetLastError();
-    return error == ERROR_IO_PENDING || error == ERROR_IO_INCOMPLETE ? 0 : error;
-  }
-
-  monitor->running = false;
-
-  for (FILE_NOTIFY_INFORMATION* info = (FILE_NOTIFY_INFORMATION*)monitor->buffer; (char*)info < monitor->buffer + sizeof(monitor->buffer); info = (FILE_NOTIFY_INFORMATION*)(((char*)info) + info->NextEntryOffset)) {
-    change_callback(info->FileNameLength / sizeof(WCHAR), (char*)info->FileName, data);
-    if (!info->NextEntryOffset)
-      break;
-  }
-
-  monitor->running = false;
   return 0;
 }
 
-int add_dirmonitor_win32(struct dirmonitor* monitor, const char* path) {
+
-  close_monitor_handle(monitor);
+struct dirmonitor* init_dirmonitor() {
-  monitor->handle = CreateFileA(path, FILE_LIST_DIRECTORY, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, NULL, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OVERLAPPED, NULL);
+  return calloc(sizeof(struct dirmonitor_internal), 1);
-  if (monitor->handle && monitor->handle != INVALID_HANDLE_VALUE) {
-    return 1;
-  }
-  monitor->handle = NULL;
-  return -1;
 }
 
-void remove_dirmonitor_win32(struct dirmonitor* monitor, int fd) {
+
+static void close_monitor_handle(struct dirmonitor_internal* monitor) {
+  if (monitor->handle && monitor->handle != INVALID_HANDLE_VALUE) {
+    HANDLE handle = monitor->handle;
+    monitor->handle = NULL;
+    CancelIoEx(handle, NULL);
+    CloseHandle(handle);
+  }
+}
+
+
+void deinit_dirmonitor(struct dirmonitor_internal* monitor) {
+  close_monitor_handle(monitor);
+}
+
+
+int translate_changes_dirmonitor(struct dirmonitor_internal* monitor, char* buffer, int buffer_size, int (*change_callback)(int, const char*, void*), void* data) {
+  for (FILE_NOTIFY_INFORMATION* info = (FILE_NOTIFY_INFORMATION*)buffer; (char*)info < buffer + buffer_size; info = (FILE_NOTIFY_INFORMATION*)(((char*)info) + info->NextEntryOffset)) {
+    char transform_buffer[PATH_MAX*4];
+    int count = WideCharToMultiByte(CP_UTF8, 0, (WCHAR*)info->FileName, info->FileNameLength, transform_buffer, PATH_MAX*4 - 1, NULL, NULL);
+    change_callback(count, buffer, data);
+    if (!info->NextEntryOffset)
+      break;
+  }
+  return 0;
+}
+
+
+int add_dirmonitor(struct dirmonitor_internal* monitor, const char* path) {
+  close_monitor_handle(monitor);
+  monitor->handle = CreateFileA(path, FILE_LIST_DIRECTORY, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, NULL, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL);
+  return !monitor->handle || monitor->handle == INVALID_HANDLE_VALUE ? -1 : 1;
+}
+
+
+void remove_dirmonitor(struct dirmonitor_internal* monitor, int fd) {
   close_monitor_handle(monitor);
 }

src/meson.build

@@ -42,8 +42,6 @@ lite_sources += [
     'api/dirmonitor.c',
     'api/dirmonitor/' + dirmonitor_backend + '.c',
 ]
-lite_cargs += '-DDIRMONITOR_BACKEND=' + dirmonitor_backend
-lite_cargs += '-DDIRMONITOR_' + dirmonitor_backend.to_upper()
 
 
 lite_rc = []