Fix bug that idle self-depending PRIORITY is not handled gracefully

Previously, we did not handle PRIORITY frame which depends on itself
and for idle stream.  As a result, nghttp2_session_mem_recv (or
nghttp2_session_recv) returne NGHTTP2_ERR_NOMEM.  The error code was
still misleading.  It was not out of memory, and we failed to insert
hash map because of duplicated key, which was treated as out of
memory.  This commit fixes this issue, by explicitly checking
dependency for incoming PRIORITY for all cases.

lib/nghttp2_session.c

@@ -3674,6 +3674,11 @@ int nghttp2_session_on_priority_received(nghttp2_session *session,
                                              "PRIORITY: stream_id == 0");
   }
 
+  if (frame->priority.pri_spec.stream_id == frame->hd.stream_id) {
+    return nghttp2_session_terminate_session_with_reason(
+        session, NGHTTP2_PROTOCOL_ERROR, "depend on itself");
+  }
+
   if (!session->server) {
     /* Re-prioritization works only in server */
     return session_call_on_frame_received(session, frame);

tests/nghttp2_session_test.c

@@ -2163,6 +2163,22 @@ void test_nghttp2_session_on_priority_received(void) {
 
   nghttp2_frame_priority_free(&frame.priority);
   nghttp2_session_del(session);
+
+  /* Check again dep_stream_id == stream_id, and stream_id is idle */
+  nghttp2_session_server_new(&session, &callbacks, &user_data);
+
+  nghttp2_priority_spec_init(&pri_spec, 1, 16, 0);
+
+  nghttp2_frame_priority_init(&frame.priority, 1, &pri_spec);
+
+  CU_ASSERT(0 == nghttp2_session_on_priority_received(session, &frame));
+
+  item = nghttp2_session_get_next_ob_item(session);
+
+  CU_ASSERT(NGHTTP2_GOAWAY == item->frame.hd.type);
+
+  nghttp2_frame_priority_free(&frame.priority);
+  nghttp2_session_del(session);
 }
 
 void test_nghttp2_session_on_rst_stream_received(void) {