nghttpx: Unload BPF objects on reload to avoid running out of memlock

src/shrpx.cc

@@ -3769,6 +3769,10 @@ void reload_config(WorkerProcess *wp) {
 #endif // ENABLE_HTTP3
                                                      ));
 
+#ifdef ENABLE_HTTP3
+  ipc_send(last_wp.get(), SHRPX_IPC_UNLOAD_BPF_OBJECT);
+#endif // ENABLE_HTTP3
+
   if (!get_config()->pid_file.empty()) {
     save_pid();
   }

src/shrpx_connection_handler.cc

@@ -1079,6 +1079,26 @@ ConnectionHandler::match_quic_lingering_worker_process_cid_prefix(
 std::vector<BPFRef> &ConnectionHandler::get_quic_bpf_refs() {
   return quic_bpf_refs_;
 }
+
+void ConnectionHandler::unload_bpf_objects() {
+  std::array<char, STRERROR_BUFSIZE> errbuf;
+
+  LOG(NOTICE) << "Unloading BPF objects";
+
+  for (auto &ref : quic_bpf_refs_) {
+    if (ref.obj == nullptr) {
+      continue;
+    }
+
+    if (bpf_object__unload(ref.obj) != 0) {
+      LOG(WARN) << "Failed to unload bpf object: "
+                << xsi_strerror(errno, errbuf.data(), errbuf.size());
+      continue;
+    }
+
+    ref.obj = nullptr;
+  }
+}
 #  endif // HAVE_LIBBPF
 
 void ConnectionHandler::set_quic_ipc_fd(int fd) { quic_ipc_fd_ = fd; }

src/shrpx_connection_handler.h

@@ -106,6 +106,7 @@ struct SerialEvent {
 #ifdef ENABLE_HTTP3
 #  ifdef HAVE_LIBBPF
 struct BPFRef {
+  bpf_object *obj;
   int reuseport_array;
   int cid_prefix_map;
 };
@@ -225,6 +226,7 @@ public:
 
 #  ifdef HAVE_LIBBPF
   std::vector<BPFRef> &get_quic_bpf_refs();
+  void unload_bpf_objects();
 #  endif // HAVE_LIBBPF
 #endif   // ENABLE_HTTP3
 

src/shrpx_process.h

@@ -31,6 +31,7 @@ namespace shrpx {
 
 constexpr uint8_t SHRPX_IPC_REOPEN_LOG = 1;
 constexpr uint8_t SHRPX_IPC_GRACEFUL_SHUTDOWN = 2;
+constexpr uint8_t SHRPX_IPC_UNLOAD_BPF_OBJECT = 3;
 
 } // namespace shrpx
 

src/shrpx_worker.cc

@@ -890,6 +890,8 @@ int Worker::create_quic_server_socket(UpstreamAddr &faddr) {
 
       auto &ref = quic_bpf_refs[faddr.index];
 
+      ref.obj = obj;
+
       auto reuseport_array =
           bpf_object__find_map_by_name(obj, "reuseport_array");
       err = libbpf_get_error(reuseport_array);

src/shrpx_worker_process.cc

@@ -175,6 +175,9 @@ void ipc_readcb(struct ev_loop *loop, ev_io *w, int revents) {
     case SHRPX_IPC_REOPEN_LOG:
       reopen_log(conn_handler);
       break;
+    case SHRPX_IPC_UNLOAD_BPF_OBJECT:
+      conn_handler->unload_bpf_objects();
+      break;
     }
   }
 }