walkero
/
nghttp2
1
0
Fork 0
Code Issues Pull Requests Projects Releases 1 Wiki Activity

Use hash table for dynamic table lookup

This commit is contained in:
Tatsuhiro Tsujikawa 2015-09-15 01:05:41 +09:00
parent 4ac8edfe27
commit 230b1f927f
3 changed files with 249 additions and 104 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

317
lib/nghttp2_hd.c
View File

@ -32,10 +32,10 @@
#include "nghttp2_int.h"
/* Make scalar initialization form of nghttp2_hd_entry */
#define MAKE_STATIC_ENT(N, V, T) \
#define MAKE_STATIC_ENT(N, V, T, H) \
{ \
{ (uint8_t *)(N), (uint8_t *)(V), sizeof((N)) - 1, sizeof((V)) - 1, 0 } \
, (T), 1, NGHTTP2_HD_FLAG_NONE \
, NULL, 0, (H), (T), 1, NGHTTP2_HD_FLAG_NONE \
}
/* Generated by mkstatictbl.py */
@ -43,67 +43,67 @@
first enum value if same header names are repeated (e.g.,
:status). */
static nghttp2_hd_entry static_table[] = {
MAKE_STATIC_ENT(":authority", "", 0),
MAKE_STATIC_ENT(":method", "GET", 1),
MAKE_STATIC_ENT(":method", "POST", 1),
MAKE_STATIC_ENT(":path", "/", 3),
MAKE_STATIC_ENT(":path", "/index.html", 3),
MAKE_STATIC_ENT(":scheme", "http", 5),
MAKE_STATIC_ENT(":scheme", "https", 5),
MAKE_STATIC_ENT(":status", "200", 7),
MAKE_STATIC_ENT(":status", "204", 7),
MAKE_STATIC_ENT(":status", "206", 7),
MAKE_STATIC_ENT(":status", "304", 7),
MAKE_STATIC_ENT(":status", "400", 7),
MAKE_STATIC_ENT(":status", "404", 7),
MAKE_STATIC_ENT(":status", "500", 7),
MAKE_STATIC_ENT("accept-charset", "", 14),
MAKE_STATIC_ENT("accept-encoding", "gzip, deflate", 15),
MAKE_STATIC_ENT("accept-language", "", 16),
MAKE_STATIC_ENT("accept-ranges", "", 17),
MAKE_STATIC_ENT("accept", "", 18),
MAKE_STATIC_ENT("access-control-allow-origin", "", 19),
MAKE_STATIC_ENT("age", "", 20),
MAKE_STATIC_ENT("allow", "", 21),
MAKE_STATIC_ENT("authorization", "", 22),
MAKE_STATIC_ENT("cache-control", "", 23),
MAKE_STATIC_ENT("content-disposition", "", 24),
MAKE_STATIC_ENT("content-encoding", "", 25),
MAKE_STATIC_ENT("content-language", "", 26),
MAKE_STATIC_ENT("content-length", "", 27),
MAKE_STATIC_ENT("content-location", "", 28),
MAKE_STATIC_ENT("content-range", "", 29),
MAKE_STATIC_ENT("content-type", "", 30),
MAKE_STATIC_ENT("cookie", "", 31),
MAKE_STATIC_ENT("date", "", 32),
MAKE_STATIC_ENT("etag", "", 33),
MAKE_STATIC_ENT("expect", "", 34),
MAKE_STATIC_ENT("expires", "", 35),
MAKE_STATIC_ENT("from", "", 36),
MAKE_STATIC_ENT("host", "", 37),
MAKE_STATIC_ENT("if-match", "", 38),
MAKE_STATIC_ENT("if-modified-since", "", 39),
MAKE_STATIC_ENT("if-none-match", "", 40),
MAKE_STATIC_ENT("if-range", "", 41),
MAKE_STATIC_ENT("if-unmodified-since", "", 42),
MAKE_STATIC_ENT("last-modified", "", 43),
MAKE_STATIC_ENT("link", "", 44),
MAKE_STATIC_ENT("location", "", 45),
MAKE_STATIC_ENT("max-forwards", "", 46),
MAKE_STATIC_ENT("proxy-authenticate", "", 47),
MAKE_STATIC_ENT("proxy-authorization", "", 48),
MAKE_STATIC_ENT("range", "", 49),
MAKE_STATIC_ENT("referer", "", 50),
MAKE_STATIC_ENT("refresh", "", 51),
MAKE_STATIC_ENT("retry-after", "", 52),
MAKE_STATIC_ENT("server", "", 53),
MAKE_STATIC_ENT("set-cookie", "", 54),
MAKE_STATIC_ENT("strict-transport-security", "", 55),
MAKE_STATIC_ENT("transfer-encoding", "", 56),
MAKE_STATIC_ENT("user-agent", "", 57),
MAKE_STATIC_ENT("vary", "", 58),
MAKE_STATIC_ENT("via", "", 59),
MAKE_STATIC_ENT("www-authenticate", "", 60),
MAKE_STATIC_ENT(":authority", "", 0, 3153725150u),
MAKE_STATIC_ENT(":method", "GET", 1, 695666056u),
MAKE_STATIC_ENT(":method", "POST", 1, 695666056u),
MAKE_STATIC_ENT(":path", "/", 3, 3292848686u),
MAKE_STATIC_ENT(":path", "/index.html", 3, 3292848686u),
MAKE_STATIC_ENT(":scheme", "http", 5, 2510477674u),
MAKE_STATIC_ENT(":scheme", "https", 5, 2510477674u),
MAKE_STATIC_ENT(":status", "200", 7, 4000288983u),
MAKE_STATIC_ENT(":status", "204", 7, 4000288983u),
MAKE_STATIC_ENT(":status", "206", 7, 4000288983u),
MAKE_STATIC_ENT(":status", "304", 7, 4000288983u),
MAKE_STATIC_ENT(":status", "400", 7, 4000288983u),
MAKE_STATIC_ENT(":status", "404", 7, 4000288983u),
MAKE_STATIC_ENT(":status", "500", 7, 4000288983u),
MAKE_STATIC_ENT("accept-charset", "", 14, 3664010344u),
MAKE_STATIC_ENT("accept-encoding", "gzip, deflate", 15, 3379649177u),
MAKE_STATIC_ENT("accept-language", "", 16, 1979086614u),
MAKE_STATIC_ENT("accept-ranges", "", 17, 1713753958u),
MAKE_STATIC_ENT("accept", "", 18, 136609321u),
MAKE_STATIC_ENT("access-control-allow-origin", "", 19, 2710797292u),
MAKE_STATIC_ENT("age", "", 20, 742476188u),
MAKE_STATIC_ENT("allow", "", 21, 2930878514u),
MAKE_STATIC_ENT("authorization", "", 22, 2436257726u),
MAKE_STATIC_ENT("cache-control", "", 23, 1355326669u),
MAKE_STATIC_ENT("content-disposition", "", 24, 3889184348u),
MAKE_STATIC_ENT("content-encoding", "", 25, 65203592u),
MAKE_STATIC_ENT("content-language", "", 26, 24973587u),
MAKE_STATIC_ENT("content-length", "", 27, 1308181789u),
MAKE_STATIC_ENT("content-location", "", 28, 2302364718u),
MAKE_STATIC_ENT("content-range", "", 29, 3555523146u),
MAKE_STATIC_ENT("content-type", "", 30, 4244048277u),
MAKE_STATIC_ENT("cookie", "", 31, 2007449791u),
MAKE_STATIC_ENT("date", "", 32, 3564297305u),
MAKE_STATIC_ENT("etag", "", 33, 113792960u),
MAKE_STATIC_ENT("expect", "", 34, 2530896728u),
MAKE_STATIC_ENT("expires", "", 35, 1049544579u),
MAKE_STATIC_ENT("from", "", 36, 2513272949u),
MAKE_STATIC_ENT("host", "", 37, 2952701295u),
MAKE_STATIC_ENT("if-match", "", 38, 3597694698u),
MAKE_STATIC_ENT("if-modified-since", "", 39, 2213050793u),
MAKE_STATIC_ENT("if-none-match", "", 40, 2536202615u),
MAKE_STATIC_ENT("if-range", "", 41, 2340978238u),
MAKE_STATIC_ENT("if-unmodified-since", "", 42, 3794814858u),
MAKE_STATIC_ENT("last-modified", "", 43, 3226950251u),
MAKE_STATIC_ENT("link", "", 44, 232457833u),
MAKE_STATIC_ENT("location", "", 45, 200649126u),
MAKE_STATIC_ENT("max-forwards", "", 46, 1826162134u),
MAKE_STATIC_ENT("proxy-authenticate", "", 47, 2709445359u),
MAKE_STATIC_ENT("proxy-authorization", "", 48, 2686392507u),
MAKE_STATIC_ENT("range", "", 49, 4208725202u),
MAKE_STATIC_ENT("referer", "", 50, 3969579366u),
MAKE_STATIC_ENT("refresh", "", 51, 3572655668u),
MAKE_STATIC_ENT("retry-after", "", 52, 3336180598u),
MAKE_STATIC_ENT("server", "", 53, 1085029842u),
MAKE_STATIC_ENT("set-cookie", "", 54, 1848371000u),
MAKE_STATIC_ENT("strict-transport-security", "", 55, 4138147361u),
MAKE_STATIC_ENT("transfer-encoding", "", 56, 3719590988u),
MAKE_STATIC_ENT("user-agent", "", 57, 606444526u),
MAKE_STATIC_ENT("vary", "", 58, 1085005381u),
MAKE_STATIC_ENT("via", "", 59, 1762798611u),
MAKE_STATIC_ENT("www-authenticate", "", 60, 779865858u),
};
static int memeq(const void *s1, const void *s2, size_t n) {
@ -540,6 +540,8 @@ int nghttp2_hd_entry_init(nghttp2_hd_entry *ent, uint8_t flags, uint8_t *name,
ent->token = token;
ent->ref = 1;
ent->flags = flags;
ent->next = NULL;
ent->hash = 0;
return 0;
@ -562,6 +564,97 @@ void nghttp2_hd_entry_free(nghttp2_hd_entry *ent, nghttp2_mem *mem) {
}
}
static int name_eq(const nghttp2_nv *a, const nghttp2_nv *b) {
return a->namelen == b->namelen && memeq(a->name, b->name, a->namelen);
}
static int value_eq(const nghttp2_nv *a, const nghttp2_nv *b) {
return a->valuelen == b->valuelen && memeq(a->value, b->value, a->valuelen);
}
static uint32_t name_hash(const nghttp2_nv *nv) {
/* 32 bit FNV-1a: http://isthe.com/chongo/tech/comp/fnv/ */
uint32_t h = 2166136261;
size_t i;
for (i = 0; i < nv->namelen; ++i) {
h ^= nv->name[i];
h += (h << 1) + (h << 4) + (h << 7) + (h << 8) + (h << 24);
}
return h;
}
static void hd_map_init(nghttp2_hd_map *map) {
memset(map, 0, sizeof(nghttp2_hd_map));
}
static void hd_map_insert(nghttp2_hd_map *map, nghttp2_hd_entry *ent) {
nghttp2_hd_entry **bucket;
bucket = &map->table[ent->hash & (HD_MAP_SIZE - 1)];
if (*bucket == NULL) {
*bucket = ent;
return;
}
/* lower index is linked near the root */
ent->next = *bucket;
*bucket = ent;
}
static nghttp2_hd_entry *hd_map_find(nghttp2_hd_map *map, int *exact_match,
const nghttp2_nv *nv, int token,
uint32_t hash) {
nghttp2_hd_entry *p;
nghttp2_hd_entry *res = NULL;
*exact_match = 0;
for (p = map->table[hash & (HD_MAP_SIZE - 1)]; p; p = p->next) {
if (hash != p->hash || token != p->token ||
(token == -1 && !name_eq(&p->nv, nv))) {
continue;
}
if (!res) {
res = p;
}
if (value_eq(&p->nv, nv)) {
res = p;
*exact_match = 1;
break;
}
}
return res;
}
static void hd_map_remove(nghttp2_hd_map *map, nghttp2_hd_entry *ent) {
nghttp2_hd_entry **bucket;
nghttp2_hd_entry *p;
bucket = &map->table[ent->hash & (HD_MAP_SIZE - 1)];
if (*bucket == NULL) {
return;
}
if (*bucket == ent) {
*bucket = ent->next;
ent->next = NULL;
return;
}
for (p = *bucket; p; p = p->next) {
if (p->next == ent) {
p->next = ent->next;
ent->next = NULL;
return;
}
}
}
static int hd_ringbuf_init(nghttp2_hd_ringbuf *ringbuf, size_t bufsize,
nghttp2_mem *mem) {
size_t size;
@ -656,6 +749,8 @@ static int hd_context_init(nghttp2_hd_context *context, nghttp2_mem *mem) {
}
context->hd_table_bufsize = 0;
context->next_seq = 0;
return 0;
}
@ -677,6 +772,8 @@ int nghttp2_hd_deflate_init2(nghttp2_hd_deflater *deflater,
return rv;
}
hd_map_init(&deflater->map);
if (deflate_hd_table_bufsize_max < NGHTTP2_HD_DEFAULT_MAX_BUFFER_SIZE) {
deflater->notify_table_size_change = 1;
deflater->ctx.hd_table_bufsize_max = deflate_hd_table_bufsize_max;
@ -1081,10 +1178,10 @@ static int emit_newname_block(nghttp2_bufs *bufs, const nghttp2_nv *nv,
return 0;
}
static nghttp2_hd_entry *add_hd_table_incremental(nghttp2_hd_context *context,
const nghttp2_nv *nv,
int token,
uint8_t entry_flags) {
static nghttp2_hd_entry *
add_hd_table_incremental(nghttp2_hd_context *context, const nghttp2_nv *nv,
int token, uint8_t entry_flags, nghttp2_hd_map *map,
uint32_t hash) {
int rv;
nghttp2_hd_entry *new_ent;
size_t room;
@ -1105,6 +1202,9 @@ static nghttp2_hd_entry *add_hd_table_incremental(nghttp2_hd_context *context,
ent->nv.name, ent->nv.value));
hd_ringbuf_pop_back(&context->hd_table);
if (map) {
hd_map_remove(map, ent);
}
if (--ent->ref == 0) {
nghttp2_hd_entry_free(ent, mem);
nghttp2_mem_free(mem, ent);
@ -1152,19 +1252,21 @@ static nghttp2_hd_entry *add_hd_table_incremental(nghttp2_hd_context *context,
return NULL;
}
new_ent->seq = context->next_seq++;
new_ent->hash = hash;
DEBUGF(fprintf(stderr, "deflatehd: indexed at %zu\n",
context->hd_table.len + NGHTTP2_STATIC_TABLE_LENGTH));
if (map) {
hd_map_insert(map, new_ent);
}
context->hd_table_bufsize += room;
}
return new_ent;
}
static int name_eq(const nghttp2_nv *a, const nghttp2_nv *b) {
return a->namelen == b->namelen && memeq(a->name, b->name, a->namelen);
}
static int value_eq(const nghttp2_nv *a, const nghttp2_nv *b) {
return a->valuelen == b->valuelen && memeq(a->value, b->value, a->valuelen);
}
typedef struct {
ssize_t index;
/* Nonzero if both name and value are matched. */
@ -1194,9 +1296,11 @@ static search_result search_static_table(const nghttp2_nv *nv, int token,
static search_result search_hd_table(nghttp2_hd_context *context,
const nghttp2_nv *nv, int token,
int indexing_mode) {
int indexing_mode, nghttp2_hd_map *map,
uint32_t hash) {
search_result res = {-1, 0};
size_t i;
nghttp2_hd_entry *ent;
int exact_match;
if (token >= 0 && token <= NGHTTP2_TOKEN_WWW_AUTHENTICATE) {
res = search_static_table(nv, token, indexing_mode);
@ -1205,27 +1309,27 @@ static search_result search_hd_table(nghttp2_hd_context *context,
}
}
for (i = 0; i < context->hd_table.len; ++i) {
nghttp2_hd_entry *ent = hd_ringbuf_get(&context->hd_table, i);
if (ent->token != token || (token == -1 && !name_eq(&ent->nv, nv))) {
continue;
}
exact_match = 0;
ent = hd_map_find(map, &exact_match, nv, token, hash);
if (ent == NULL) {
return res;
}
if (res.index == -1) {
res.index = (ssize_t)(i + NGHTTP2_STATIC_TABLE_LENGTH);
}
if (res.index != -1 && !exact_match) {
return res;
}
if (indexing_mode != NGHTTP2_HD_NEVER_INDEXING && value_eq(&ent->nv, nv)) {
res.index = (ssize_t)(i + NGHTTP2_STATIC_TABLE_LENGTH);
res.name_value_match = 1;
return res;
}
res.index = context->next_seq - 1 - ent->seq + NGHTTP2_STATIC_TABLE_LENGTH;
if (exact_match) {
res.name_value_match = 1;
}
return res;
}
static void hd_context_shrink_table_size(nghttp2_hd_context *context) {
static void hd_context_shrink_table_size(nghttp2_hd_context *context,
nghttp2_hd_map *map) {
nghttp2_mem *mem;
mem = context->mem;
@ -1236,6 +1340,9 @@ static void hd_context_shrink_table_size(nghttp2_hd_context *context) {
nghttp2_hd_entry *ent = hd_ringbuf_get(&context->hd_table, idx);
context->hd_table_bufsize -= entry_room(ent->nv.namelen, ent->nv.valuelen);
hd_ringbuf_pop_back(&context->hd_table);
if (map) {
hd_map_remove(map, ent);
}
if (--ent->ref == 0) {
nghttp2_hd_entry_free(ent, mem);
nghttp2_mem_free(mem, ent);
@ -1255,7 +1362,7 @@ int nghttp2_hd_deflate_change_table_size(nghttp2_hd_deflater *deflater,
deflater->notify_table_size_change = 1;
hd_context_shrink_table_size(&deflater->ctx);
hd_context_shrink_table_size(&deflater->ctx, &deflater->map);
return 0;
}
@ -1272,7 +1379,7 @@ int nghttp2_hd_inflate_change_table_size(nghttp2_hd_inflater *inflater,
inflater->state = NGHTTP2_HD_STATE_EXPECT_TABLE_SIZE;
inflater->settings_hd_table_bufsize_max = settings_hd_table_bufsize_max;
inflater->ctx.hd_table_bufsize_max = settings_hd_table_bufsize_max;
hd_context_shrink_table_size(&inflater->ctx);
hd_context_shrink_table_size(&inflater->ctx, NULL);
return 0;
}
@ -1317,12 +1424,18 @@ static int deflate_nv(nghttp2_hd_deflater *deflater, nghttp2_bufs *bufs,
int indexing_mode;
int token;
nghttp2_mem *mem;
uint32_t hash;
DEBUGF(fprintf(stderr, "deflatehd: deflating %s: %s\n", nv->name, nv->value));
mem = deflater->ctx.mem;
token = lookup_token(nv->name, nv->namelen);
if (token == -1 || token > NGHTTP2_TOKEN_WWW_AUTHENTICATE) {
hash = name_hash(nv);
} else {
hash = static_table[token].hash;
}
/* Don't index authorization header field since it may contain low
entropy secret data (e.g., id/password). Also cookie header
@ -1335,7 +1448,8 @@ static int deflate_nv(nghttp2_hd_deflater *deflater, nghttp2_bufs *bufs,
? NGHTTP2_HD_NEVER_INDEXING
: hd_deflate_decide_indexing(deflater, nv, token);
res = search_hd_table(&deflater->ctx, nv, token, indexing_mode);
res = search_hd_table(&deflater->ctx, nv, token, indexing_mode,
&deflater->map, hash);
idx = res.index;
@ -1362,11 +1476,13 @@ static int deflate_nv(nghttp2_hd_deflater *deflater, nghttp2_bufs *bufs,
nv_indname = *nv;
nv_indname.name = nghttp2_hd_table_get(&deflater->ctx, idx)->nv.name;
new_ent = add_hd_table_incremental(&deflater->ctx, &nv_indname, token,
NGHTTP2_HD_FLAG_VALUE_ALLOC);
NGHTTP2_HD_FLAG_VALUE_ALLOC,
&deflater->map, hash);
} else {
new_ent = add_hd_table_incremental(&deflater->ctx, nv, token,
NGHTTP2_HD_FLAG_NAME_ALLOC |
NGHTTP2_HD_FLAG_VALUE_ALLOC);
NGHTTP2_HD_FLAG_VALUE_ALLOC,
&deflater->map, hash);
}
if (!new_ent) {
return NGHTTP2_ERR_HEADER_COMP;
@ -1816,8 +1932,9 @@ static int hd_inflate_commit_newname(nghttp2_hd_inflater *inflater,
management. */
ent_flags = NGHTTP2_HD_FLAG_NAME_ALLOC | NGHTTP2_HD_FLAG_NAME_GIFT;
new_ent = add_hd_table_incremental(
&inflater->ctx, &nv, lookup_token(nv.name, nv.namelen), ent_flags);
new_ent = add_hd_table_incremental(&inflater->ctx, &nv,
lookup_token(nv.name, nv.namelen),
ent_flags, NULL, 0);
if (new_ent) {
emit_indexed_header(nv_out, token_out, new_ent);
@ -1892,7 +2009,7 @@ static int hd_inflate_commit_indname(nghttp2_hd_inflater *inflater,
}
new_ent = add_hd_table_incremental(&inflater->ctx, &nv, ent_name->token,
ent_flags);
ent_flags, NULL, 0);
/* At this point, ent_name might be deleted. */
@ -2021,7 +2138,7 @@ ssize_t nghttp2_hd_inflate_hd2(nghttp2_hd_inflater *inflater,
}
DEBUGF(fprintf(stderr, "inflatehd: table_size=%zu\n", inflater->left));
inflater->ctx.hd_table_bufsize_max = inflater->left;
hd_context_shrink_table_size(&inflater->ctx);
hd_context_shrink_table_size(&inflater->ctx, NULL);
inflater->state = NGHTTP2_HD_STATE_INFLATE_START;
break;
case NGHTTP2_HD_STATE_READ_INDEX: {

21
lib/nghttp2_hd.h
View File

@ -126,15 +126,25 @@ typedef enum {
NGHTTP2_HD_FLAG_VALUE_GIFT = 1 << 3
} nghttp2_hd_flags;
typedef struct {
struct nghttp2_hd_entry;
typedef struct nghttp2_hd_entry nghttp2_hd_entry;
struct nghttp2_hd_entry {
nghttp2_nv nv;
/* The next entry which shares same bucket in hash table. */
nghttp2_hd_entry *next;
/* The sequence number. We will increment it by one whenever we
store nghttp2_hd_entry to dynamic header table. */
uint32_t seq;
/* The hash value for header name (nv.name). */
uint32_t hash;
/* nghttp2_token value for nv.name. It could be -1 if we have no
token for that header field name. */
int token;
/* Reference count */
uint8_t ref;
uint8_t flags;
} nghttp2_hd_entry;
};
typedef struct {
nghttp2_hd_entry **buffer;
@ -183,14 +193,21 @@ typedef struct {
size_t hd_table_bufsize;
/* The effective header table size. */
size_t hd_table_bufsize_max;
/* Next sequence number for nghttp2_hd_entry */
uint32_t next_seq;
/* If inflate/deflate error occurred, this value is set to 1 and
further invocation of inflate/deflate will fail with
NGHTTP2_ERR_HEADER_COMP. */
uint8_t bad;
} nghttp2_hd_context;
#define HD_MAP_SIZE 128
typedef struct { nghttp2_hd_entry *table[HD_MAP_SIZE]; } nghttp2_hd_map;
struct nghttp2_hd_deflater {
nghttp2_hd_context ctx;
nghttp2_hd_map map;
/* The upper limit of the header table size the deflater accepts. */
size_t deflate_hd_table_bufsize_max;
/* Minimum header table size notified in the next context update */

15
mkstatichdtbl.py
View File

@ -10,6 +10,17 @@
from __future__ import unicode_literals
import re, sys
def hd_map_hash(name):
h = 2166136261
# FNV hash variant: http://isthe.com/chongo/tech/comp/fnv/
for c in name:
h ^= ord(c)
h *= 16777619
h &= 0xffffffff
return h
entries = []
for line in sys.stdin:
m = re.match(r'(\d+)\s+(\S+)\s+(\S.*)?', line)
@ -21,6 +32,6 @@ idx = 0
for i, ent in enumerate(entries):
if entries[idx][1] != ent[1]:
idx = i
print 'MAKE_STATIC_ENT("{}", "{}", {}),'\
.format(ent[1], ent[2], entries[idx][0] - 1)
print 'MAKE_STATIC_ENT("{}", "{}", {}, {}u),'\
.format(ent[1], ent[2], entries[idx][0] - 1, hd_map_hash(ent[1]))
print '};'