nghttpd: Check validity of cached file descriptor periodically

This commit adds ability to check status of cached file descriptor to
make sure that it can be reused.  We inspect last modification time
and number of hard links.  If last modification is changed from the
last validation time, or number of hard links gets 0, we don't reuse
file descriptor.  We also capped upper limit of the cached file
descriptors.  If the limit is reached, we will close file descriptor
which is least recently used, and its usecount is 0.

src/HttpServer.cc

@@ -183,6 +183,43 @@ namespace {
 void release_fd_cb(struct ev_loop *loop, ev_timer *w, int revents);
 } // namespace
 
+namespace {
+constexpr ev_tstamp FILE_ENTRY_MAX_AGE = 10.;
+} // namespace
+
+namespace {
+constexpr size_t FILE_ENTRY_EVICT_THRES = 2048;
+} // namespace
+
+namespace {
+bool need_validation_file_entry(const FileEntry *ent, ev_tstamp now) {
+  return ent->last_valid + FILE_ENTRY_MAX_AGE < now;
+}
+} // namespace
+
+namespace {
+bool validate_file_entry(FileEntry *ent, ev_tstamp now) {
+  struct stat stbuf;
+  int rv;
+
+  rv = fstat(ent->fd, &stbuf);
+  if (rv != 0) {
+    ent->stale = true;
+    return false;
+  }
+
+  if (stbuf.st_nlink == 0 || ent->mtime != stbuf.st_mtime) {
+    ent->stale = true;
+    return false;
+  }
+
+  ent->mtime = stbuf.st_mtime;
+  ent->last_valid = now;
+
+  return true;
+}
+} // namespace
+
 class Sessions {
 public:
   Sessions(HttpServer *sv, struct ev_loop *loop, const Config *config,
@@ -269,14 +306,39 @@ public:
     return cached_date_;
   }
   FileEntry *get_cached_fd(const std::string &path) {
-    auto i = fd_cache_.find(path);
+    auto range = fd_cache_.equal_range(path);
-    if (i == std::end(fd_cache_)) {
+    if (range.first == range.second) {
       return nullptr;
     }
-    auto &ent = (*i).second;
+
+    auto now = ev_now(loop_);
+
+    for (auto it = range.first; it != range.second;) {
+      auto &ent = (*it).second;
+      if (ent.stale) {
+        ++it;
+        continue;
+      }
+      if (need_validation_file_entry(&ent, now) &&
+          !validate_file_entry(&ent, now)) {
+        if (ent.usecount == 0) {
+          fd_cache_lru_.remove(&ent);
+          close(ent.fd);
+          it = fd_cache_.erase(it);
+          continue;
+        }
+        ++it;
+        continue;
+      }
+
+      fd_cache_lru_.remove(&ent);
+      fd_cache_lru_.append(&ent);
+
       ++ent.usecount;
       return &ent;
     }
+    return nullptr;
+  }
   FileEntry *cache_fd(const std::string &path, const FileEntry &ent) {
 #ifdef HAVE_STD_MAP_EMPLACE
     auto rv = fd_cache_.emplace(path, ent);
@@ -284,23 +346,31 @@ public:
     // for gcc-4.7
     auto rv = fd_cache_.insert(std::make_pair(path, ent));
 #endif // !HAVE_STD_MAP_EMPLACE
-    return &(*rv.first).second;
+    auto &res = (*rv).second;
-  }
+    res.it = rv;
-  void release_fd(const std::string &path) {
+    fd_cache_lru_.append(&res);
-    auto i = fd_cache_.find(path);
-    if (i == std::end(fd_cache_)) {
-      return;
-    }
-    auto &ent = (*i).second;
-    if (ent.usecount == 0) {
-      // temporary fd, close it immediately
-      close(ent.fd);
-      fd_cache_.erase(i);
 
-      return;
+    while (fd_cache_.size() > FILE_ENTRY_EVICT_THRES) {
+      auto ent = fd_cache_lru_.head;
+      if (ent->usecount) {
+        break;
+      }
+      fd_cache_lru_.remove(ent);
+      close(ent->fd);
+      fd_cache_.erase(ent->it);
     }
 
-    --ent.usecount;
+    return &res;
+  }
+  void release_fd(FileEntry *target) {
+    --target->usecount;
+
+    if (target->usecount == 0 && target->stale) {
+      fd_cache_lru_.remove(target);
+      close(target->fd);
+      fd_cache_.erase(target->it);
+      return;
+    }
 
     // We use timer to close file descriptor and delete the entry from
     // cache.  The timer will be started when there is no handler.
@@ -313,6 +383,7 @@ public:
         continue;
       }
 
+      fd_cache_lru_.remove(&ent);
       close(ent.fd);
       i = fd_cache_.erase(i);
     }
@@ -323,7 +394,8 @@ public:
 private:
   std::set<Http2Handler *> handlers_;
   // cache for file descriptors to read file.
-  std::map<std::string, FileEntry> fd_cache_;
+  std::multimap<std::string, FileEntry> fd_cache_;
+  DList<FileEntry> fd_cache_lru_;
   HttpServer *sv_;
   struct ev_loop *loop_;
   const Config *config_;
@@ -366,7 +438,7 @@ Stream::Stream(Http2Handler *handler, int32_t stream_id)
 Stream::~Stream() {
   if (file_ent != nullptr) {
     auto sessions = handler->get_sessions();
-    sessions->release_fd(file_ent->path);
+    sessions->release_fd(file_ent);
   }
 
   auto loop = handler->get_loop();
@@ -1034,7 +1106,7 @@ bool prepare_upload_temp_store(Stream *stream, Http2Handler *hd) {
   // now.  We will update it when we get whole request body.
   auto path = std::string("echo:") + tempfn;
   stream->file_ent =
-      sessions->cache_fd(path, FileEntry(path, 0, 0, fd, nullptr, 0));
+      sessions->cache_fd(path, FileEntry(path, 0, 0, fd, nullptr, 0, true));
   stream->echo_upload = true;
   return true;
 }
@@ -1104,7 +1176,7 @@ void prepare_response(Stream *stream, Http2Handler *hd,
   url = util::percentDecode(std::begin(url), std::end(url));
   if (!util::check_path(url)) {
     if (stream->file_ent) {
-      sessions->release_fd(stream->file_ent->path);
+      sessions->release_fd(stream->file_ent);
       stream->file_ent = nullptr;
     }
     prepare_status_response(stream, hd, 404);
@@ -1186,7 +1258,8 @@ void prepare_response(Stream *stream, Http2Handler *hd,
     }
 
     file_ent = sessions->cache_fd(
-        path, FileEntry(path, buf.st_size, buf.st_mtime, file, content_type));
+        path, FileEntry(path, buf.st_size, buf.st_mtime, file, content_type,
+                        ev_now(sessions->get_loop())));
   }
 
   stream->file_ent = file_ent;
@@ -1710,7 +1783,7 @@ FileEntry make_status_body(int status, uint16_t port) {
     assert(0);
   }
 
-  return FileEntry(util::utos(status), nwrite, 0, fd, nullptr);
+  return FileEntry(util::utos(status), nwrite, 0, fd, nullptr, 0);
 }
 } // namespace
 

src/HttpServer.h

@@ -83,18 +83,22 @@ struct Config {
 class Http2Handler;
 
 struct FileEntry {
-  // To close fd immediately when nothing refers to this entry, give 0
-  // to |usecount|.
   FileEntry(std::string path, int64_t length, int64_t mtime, int fd,
-            const std::string *content_type, int usecount = 1)
+            const std::string *content_type, ev_tstamp last_valid,
+            bool stale = false)
       : path(std::move(path)), length(length), mtime(mtime),
-        content_type(content_type), fd(fd), usecount(usecount) {}
+        last_valid(last_valid), content_type(content_type), dlnext(nullptr),
+        dlprev(nullptr), fd(fd), usecount(1), stale(stale) {}
   std::string path;
+  std::multimap<std::string, FileEntry>::iterator it;
   int64_t length;
   int64_t mtime;
+  ev_tstamp last_valid;
   const std::string *content_type;
+  FileEntry *dlnext, *dlprev;
   int fd;
   int usecount;
+  bool stale;
 };
 
 struct Stream {