diff --git a/src/HttpServer.cc b/src/HttpServer.cc
index 3044567f..95e75fdb 100644
--- a/src/HttpServer.cc
+++ b/src/HttpServer.cc
@@ -832,7 +832,6 @@ ssize_t file_read_callback(nghttp2_session *session, int32_t stream_id,
 
     auto config = hd->get_config();
     if (!config->trailer.empty()) {
-      *data_flags |= NGHTTP2_DATA_FLAG_NO_END_STREAM;
       std::vector<nghttp2_nv> nva;
       nva.reserve(config->trailer.size());
       for (auto &kv : config->trailer) {
@@ -840,7 +839,11 @@ ssize_t file_read_callback(nghttp2_session *session, int32_t stream_id,
       }
       rv = nghttp2_submit_trailer(session, stream_id, nva.data(), nva.size());
       if (rv != 0) {
-        *data_flags &= ~NGHTTP2_DATA_FLAG_NO_END_STREAM;
+        if (nghttp2_is_fatal(rv)) {
+          return NGHTTP2_ERR_CALLBACK_FAILURE;
+        }
+      } else {
+        *data_flags |= NGHTTP2_DATA_FLAG_NO_END_STREAM;
       }
     }
 
diff --git a/src/nghttp.cc b/src/nghttp.cc
index faa9c649..883565f8 100644
--- a/src/nghttp.cc
+++ b/src/nghttp.cc
@@ -2128,7 +2128,6 @@ ssize_t file_read_callback(nghttp2_session *session, int32_t stream_id,
   if (nread == 0) {
     *data_flags |= NGHTTP2_DATA_FLAG_EOF;
     if (!config.trailer.empty()) {
-      *data_flags |= NGHTTP2_DATA_FLAG_NO_END_STREAM;
       std::vector<nghttp2_nv> nva;
       nva.reserve(config.trailer.size());
       for (auto &kv : config.trailer) {
@@ -2136,7 +2135,11 @@ ssize_t file_read_callback(nghttp2_session *session, int32_t stream_id,
       }
       rv = nghttp2_submit_trailer(session, stream_id, nva.data(), nva.size());
       if (rv != 0) {
-        *data_flags &= ~NGHTTP2_DATA_FLAG_NO_END_STREAM;
+        if (nghttp2_is_fatal(rv)) {
+          return NGHTTP2_ERR_CALLBACK_FAILURE;
+        }
+      } else {
+        *data_flags |= NGHTTP2_DATA_FLAG_NO_END_STREAM;
       }
     }
   } else {