walkero
/
nghttp2
1
0
Fork 0
Code Issues Pull Requests Projects Releases 1 Wiki Activity

nghttpx: Add TLS ticket key sharing among nghttpx instances using memcached

This commit is contained in:
Tatsuhiro Tsujikawa 2015-07-28 00:54:44 +09:00
parent a4a9cfd650
commit 2f2a300e83
8 changed files with 322 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
gennghttpxfun.py
View File

@ -94,6 +94,7 @@ OPTIONS = [
"tls-ticket-cipher",
"host-rewrite",
"tls-session-cache-memcached",
"tls-ticket-key-memcached",
"conf",
]

205
src/shrpx.cc
View File

@ -83,6 +83,8 @@
#include "shrpx_accept_handler.h"
#include "shrpx_http2_upstream.h"
#include "shrpx_http2_session.h"
#include "shrpx_memcached_dispatcher.h"
#include "shrpx_memcached_request.h"
#include "util.h"
#include "app_helper.h"
#include "ssl.h"
@ -689,6 +691,116 @@ void renew_ticket_key_cb(struct ev_loop *loop, ev_timer *w, int revents) {
}
} // namespace
namespace {
void memcached_get_ticket_key_cb(struct ev_loop *loop, ev_timer *w,
int revents) {
auto conn_handler = static_cast<ConnectionHandler *>(w->data);
auto dispatcher = conn_handler->get_tls_ticket_key_memcached_dispatcher();
auto req = make_unique<MemcachedRequest>();
req->key = "nghttpx:tls-ticket-key";
req->op = MEMCACHED_OP_GET;
req->cb = [conn_handler, dispatcher, w](MemcachedRequest *req,
MemcachedResult res) {
switch (res.status_code) {
case MEMCACHED_ERR_NO_ERROR:
break;
case MEMCACHED_ERR_EXT_NETWORK_ERROR:
conn_handler->on_tls_ticket_key_network_error(w);
return;
default:
conn_handler->on_tls_ticket_key_not_found(w);
return;
}
// |version (4bytes)|len (2bytes)|key (variable length)|...
// (len, key) pairs are repeated as necessary.
auto &value = res.value;
if (value.size() < 4) {
LOG(WARN) << "Memcached: tls ticket key value is too small: got "
<< value.size();
conn_handler->on_tls_ticket_key_not_found(w);
return;
}
auto p = value.data();
auto version = util::get_uint32(p);
// Currently supported version is 1.
if (version != 1) {
LOG(WARN) << "Memcached: tls ticket key version: want 1, got " << version;
conn_handler->on_tls_ticket_key_not_found(w);
return;
}
auto end = p + value.size();
p += 4;
size_t expectedlen;
size_t enc_keylen;
size_t hmac_keylen;
if (get_config()->tls_ticket_cipher == EVP_aes_128_cbc()) {
expectedlen = 48;
enc_keylen = 16;
hmac_keylen = 16;
} else if (get_config()->tls_ticket_cipher == EVP_aes_256_cbc()) {
expectedlen = 80;
enc_keylen = 32;
hmac_keylen = 32;
} else {
return;
}
auto ticket_keys = std::make_shared<TicketKeys>();
for (; p != end;) {
if (end - p < 2) {
LOG(WARN) << "Memcached: tls ticket key data is too small";
conn_handler->on_tls_ticket_key_not_found(w);
return;
}
auto len = util::get_uint16(p);
p += 2;
if (len != expectedlen) {
LOG(WARN) << "Memcached: wrong tls ticket key size: want "
<< expectedlen << ", got " << len;
conn_handler->on_tls_ticket_key_not_found(w);
return;
}
if (p + len > end) {
LOG(WARN) << "Memcached: too short tls ticket key payload: want " << len
<< ", got " << (end - p);
conn_handler->on_tls_ticket_key_not_found(w);
return;
}
auto key = TicketKey();
key.cipher = get_config()->tls_ticket_cipher;
key.hmac = EVP_sha256();
key.hmac_keylen = EVP_MD_size(key.hmac);
std::copy_n(p, key.data.name.size(), key.data.name.data());
p += key.data.name.size();
std::copy_n(p, enc_keylen, key.data.enc_key.data());
p += enc_keylen;
std::copy_n(p, hmac_keylen, key.data.hmac_key.data());
p += hmac_keylen;
ticket_keys->keys.push_back(std::move(key));
}
conn_handler->on_tls_ticket_key_get_success(ticket_keys, w);
};
if (LOG_ENABLED(INFO)) {
LOG(INFO) << "Memcached: tls ticket key get request sent";
}
dispatcher->add_request(std::move(req));
}
} // namespace
namespace {
int call_daemon() {
#ifdef __sgi
@ -749,34 +861,47 @@ int event_loop() {
ev_timer renew_ticket_key_timer;
if (!get_config()->upstream_no_tls) {
bool auto_tls_ticket_key = true;
if (!get_config()->tls_ticket_key_files.empty()) {
if (!get_config()->tls_ticket_cipher_given) {
LOG(WARN) << "It is strongly recommended to specify "
"--tls-ticket-cipher=aes-128-cbc (or "
"tls-ticket-cipher=aes-128-cbc in configuration file) "
"when --tls-ticket-key-file is used for the smooth "
"transition when the default value of --tls-ticket-cipher "
"becomes aes-256-cbc";
}
auto ticket_keys = read_tls_ticket_key_file(
get_config()->tls_ticket_key_files, get_config()->tls_ticket_cipher,
EVP_sha256());
if (!ticket_keys) {
LOG(WARN) << "Use internal session ticket key generator";
} else {
conn_handler->set_ticket_keys(std::move(ticket_keys));
auto_tls_ticket_key = false;
}
}
if (auto_tls_ticket_key) {
// Generate new ticket key every 1hr.
ev_timer_init(&renew_ticket_key_timer, renew_ticket_key_cb, 0., 1_h);
renew_ticket_key_timer.data = conn_handler.get();
ev_timer_again(loop, &renew_ticket_key_timer);
if (get_config()->tls_ticket_key_memcached_host) {
conn_handler->set_tls_ticket_key_memcached_dispatcher(
make_unique<MemcachedDispatcher>(
&get_config()->tls_ticket_key_memcached_addr, loop));
// Generate first session ticket key before running workers.
renew_ticket_key_cb(loop, &renew_ticket_key_timer, 0);
ev_timer_init(&renew_ticket_key_timer, memcached_get_ticket_key_cb, 0.,
0.);
renew_ticket_key_timer.data = conn_handler.get();
// Get first ticket keys.
memcached_get_ticket_key_cb(loop, &renew_ticket_key_timer, 0);
} else {
bool auto_tls_ticket_key = true;
if (!get_config()->tls_ticket_key_files.empty()) {
if (!get_config()->tls_ticket_cipher_given) {
LOG(WARN)
<< "It is strongly recommended to specify "
"--tls-ticket-cipher=aes-128-cbc (or "
"tls-ticket-cipher=aes-128-cbc in configuration file) "
"when --tls-ticket-key-file is used for the smooth "
"transition when the default value of --tls-ticket-cipher "
"becomes aes-256-cbc";
}
auto ticket_keys = read_tls_ticket_key_file(
get_config()->tls_ticket_key_files, get_config()->tls_ticket_cipher,
EVP_sha256());
if (!ticket_keys) {
LOG(WARN) << "Use internal session ticket key generator";
} else {
conn_handler->set_ticket_keys(std::move(ticket_keys));
auto_tls_ticket_key = false;
}
}
if (auto_tls_ticket_key) {
// Generate new ticket key every 1hr.
ev_timer_init(&renew_ticket_key_timer, renew_ticket_key_cb, 0., 1_h);
renew_ticket_key_timer.data = conn_handler.get();
ev_timer_again(loop, &renew_ticket_key_timer);
// Generate first session ticket key before running workers.
renew_ticket_key_cb(loop, &renew_ticket_key_timer, 0);
}
}
}
@ -1020,6 +1145,9 @@ void fill_default_config() {
mod_config()->tls_ticket_cipher = EVP_aes_128_cbc();
mod_config()->tls_ticket_cipher_given = false;
mod_config()->tls_session_timeout = std::chrono::hours(12);
mod_config()->tls_ticket_key_memcached_max_retry = 3;
mod_config()->tls_ticket_key_memcached_max_fail = 2;
mod_config()->tls_ticket_key_memcached_interval = 10_min;
}
} // namespace
@ -1368,6 +1496,15 @@ SSL/TLS:
Specify address of memcached server to store session
cache. This enables shared session cache between
multiple nghttpx instances.
--tls-ticket-key-memcached=<HOST>,<PORT>
Specify address of memcached server to store session
cache. This enables shared TLS ticket key between
multiple nghttpx instances. nghttpx does not set TLS
ticket key to memcached. The external ticket key
generator is required. nghttpx just gets TLS ticket
keys from memcached, and use them, possibly replacing
current set of keys. It is up to extern TLS ticket key
generator to rotate keys frequently.
HTTP/2 and SPDY:
-c, --http2-max-concurrent-streams=<N>
@ -1732,6 +1869,7 @@ int main(int argc, char **argv) {
{SHRPX_OPT_TLS_TICKET_CIPHER, required_argument, &flag, 84},
{SHRPX_OPT_HOST_REWRITE, no_argument, &flag, 85},
{SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED, required_argument, &flag, 86},
{SHRPX_OPT_TLS_TICKET_KEY_MEMCACHED, required_argument, &flag, 87},
{nullptr, 0, nullptr, 0}};
int option_index = 0;
@ -2110,6 +2248,10 @@ int main(int argc, char **argv) {
// --tls-session-cache-memcached
cmdcfgs.emplace_back(SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED, optarg);
break;
case 87:
// --tls-ticket-key-memcached
cmdcfgs.emplace_back(SHRPX_OPT_TLS_TICKET_KEY_MEMCACHED, optarg);
break;
default:
break;
}
@ -2396,6 +2538,15 @@ int main(int argc, char **argv) {
}
}
if (get_config()->tls_ticket_key_memcached_host) {
if (resolve_hostname(&mod_config()->tls_ticket_key_memcached_addr,
get_config()->tls_ticket_key_memcached_host.get(),
get_config()->tls_ticket_key_memcached_port,
AF_UNSPEC) == -1) {
exit(EXIT_FAILURE);
}
}
if (get_config()->rlimit_nofile) {
struct rlimit lim = {static_cast<rlim_t>(get_config()->rlimit_nofile),
static_cast<rlim_t>(get_config()->rlimit_nofile)};

17
src/shrpx_config.cc
View File

@ -706,6 +706,7 @@ enum {
SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED,
SHRPX_OPTID_TLS_TICKET_CIPHER,
SHRPX_OPTID_TLS_TICKET_KEY_FILE,
SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED,
SHRPX_OPTID_USER,
SHRPX_OPTID_VERIFY_CLIENT,
SHRPX_OPTID_VERIFY_CLIENT_CACERT,
@ -1138,6 +1139,11 @@ int option_lookup_token(const char *name, size_t namelen) {
break;
case 24:
switch (name[23]) {
case 'd':
if (util::strieq_l("tls-ticket-key-memcache", name, 23)) {
return SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED;
}
break;
case 'e':
if (util::strieq_l("fetch-ocsp-response-fil", name, 23)) {
return SHRPX_OPTID_FETCH_OCSP_RESPONSE_FILE;
@ -1881,6 +1887,17 @@ int parse_config(const char *opt, const char *optarg,
return 0;
}
case SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED: {
if (split_host_port(host, sizeof(host), &port, optarg, strlen(optarg)) ==
-1) {
return -1;
}
mod_config()->tls_ticket_key_memcached_host = strcopy(host);
mod_config()->tls_ticket_key_memcached_port = port;
return 0;
}
case SHRPX_OPTID_CONF:
LOG(WARN) << "conf: ignored";

12
src/shrpx_config.h
View File

@ -175,6 +175,8 @@ constexpr char SHRPX_OPT_TLS_TICKET_CIPHER[] = "tls-ticket-cipher";
constexpr char SHRPX_OPT_HOST_REWRITE[] = "host-rewrite";
constexpr char SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED[] =
"tls-session-cache-memcached";
constexpr char SHRPX_OPT_TLS_TICKET_KEY_MEMCACHED[] =
"tls-ticket-key-memcached";
union sockaddr_union {
sockaddr_storage storage;
@ -260,6 +262,7 @@ struct Config {
// binary form of http proxy host and port
Address downstream_http_proxy_addr;
Address session_cache_memcached_addr;
Address tls_ticket_key_memcached_addr;
std::chrono::seconds tls_session_timeout;
ev_tstamp http2_upstream_read_timeout;
ev_tstamp upstream_read_timeout;
@ -271,6 +274,7 @@ struct Config {
ev_tstamp downstream_idle_read_timeout;
ev_tstamp listener_disable_timeout;
ev_tstamp ocsp_update_interval;
ev_tstamp tls_ticket_key_memcached_interval;
// address of frontend connection. This could be a path to UNIX
// domain socket. In this case, |host_unix| must be true.
std::unique_ptr<char[]> host;
@ -303,6 +307,7 @@ struct Config {
std::unique_ptr<char[]> fetch_ocsp_response_file;
std::unique_ptr<char[]> user;
std::unique_ptr<char[]> session_cache_memcached_host;
std::unique_ptr<char[]> tls_ticket_key_memcached_host;
FILE *http2_upstream_dump_request_header;
FILE *http2_upstream_dump_response_header;
nghttp2_session_callbacks *http2_upstream_callbacks;
@ -339,6 +344,12 @@ struct Config {
size_t max_header_fields;
// The index of catch-all group in downstream_addr_groups.
size_t downstream_addr_group_catch_all;
// Maximum number of retries when getting TLS ticket key from
// mamcached, due to network error.
size_t tls_ticket_key_memcached_max_retry;
// Maximum number of consecutive error from memcached, when this
// limit reached, TLS ticket is disabled.
size_t tls_ticket_key_memcached_max_fail;
// Bit mask to disable SSL/TLS protocol versions. This will be
// passed to SSL_CTX_set_options().
long int tls_proto_mask;
@ -356,6 +367,7 @@ struct Config {
// port in http proxy URI
uint16_t downstream_http_proxy_port;
uint16_t session_cache_memcached_port;
uint16_t tls_ticket_key_memcached_port;
bool verbose;
bool daemon;
bool verify_client;

97
src/shrpx_connection_handler.cc
View File

@ -32,6 +32,7 @@
#include <cerrno>
#include <thread>
#include <random>
#include "shrpx_client_handler.h"
#include "shrpx_ssl.h"
@ -41,6 +42,7 @@
#include "shrpx_connect_blocker.h"
#include "shrpx_downstream_connection.h"
#include "shrpx_accept_handler.h"
#include "shrpx_memcached_dispatcher.h"
#include "util.h"
#include "template.h"
@ -94,7 +96,9 @@ void ocsp_chld_cb(struct ev_loop *loop, ev_child *w, int revent) {
} // namespace
ConnectionHandler::ConnectionHandler(struct ev_loop *loop)
: single_worker_(nullptr), loop_(loop), worker_round_robin_cnt_(0),
: single_worker_(nullptr), loop_(loop),
tls_ticket_key_memcached_get_retry_count_(0),
tls_ticket_key_memcached_fail_count_(0), worker_round_robin_cnt_(0),
graceful_shutdown_(false) {
ev_timer_init(&disable_acceptor_timer_, acceptor_disable_cb, 0., 0.);
disable_acceptor_timer_.data = this;
@ -553,4 +557,95 @@ void ConnectionHandler::proceed_next_cert_ocsp() {
}
}
void ConnectionHandler::set_tls_ticket_key_memcached_dispatcher(
std::unique_ptr<MemcachedDispatcher> dispatcher) {
tls_ticket_key_memcached_dispatcher_ = std::move(dispatcher);
}
MemcachedDispatcher *
ConnectionHandler::get_tls_ticket_key_memcached_dispatcher() const {
return tls_ticket_key_memcached_dispatcher_.get();
}
namespace {
std::random_device rd;
} // namespace
void ConnectionHandler::on_tls_ticket_key_network_error(ev_timer *w) {
if (++tls_ticket_key_memcached_get_retry_count_ >=
get_config()->tls_ticket_key_memcached_max_retry) {
LOG(WARN) << "Memcached: tls ticket get retry all failed "
<< tls_ticket_key_memcached_get_retry_count_ << " times.";
on_tls_ticket_key_not_found(w);
return;
}
auto dist = std::uniform_int_distribution<int>(
1, std::min(60, 1 << tls_ticket_key_memcached_get_retry_count_));
auto t = dist(rd);
LOG(WARN)
<< "Memcached: tls ticket get failed due to network error, retrying in "
<< t << " seconds";
ev_timer_set(w, t, 0.);
ev_timer_start(loop_, w);
}
void ConnectionHandler::on_tls_ticket_key_not_found(ev_timer *w) {
tls_ticket_key_memcached_get_retry_count_ = 0;
if (++tls_ticket_key_memcached_fail_count_ >=
get_config()->tls_ticket_key_memcached_max_fail) {
LOG(WARN) << "Memcached: could not get tls ticket; disable tls ticket";
tls_ticket_key_memcached_fail_count_ = 0;
set_ticket_keys(nullptr);
set_ticket_keys_to_worker(nullptr);
}
LOG(WARN) << "Memcached: tls ticket get failed, schedule next";
schedule_next_tls_ticket_key_memcached_get(w);
}
void ConnectionHandler::on_tls_ticket_key_get_success(
const std::shared_ptr<TicketKeys> &ticket_keys, ev_timer *w) {
if (LOG_ENABLED(INFO)) {
LOG(INFO) << "Memcached: tls ticket get success";
}
tls_ticket_key_memcached_get_retry_count_ = 0;
tls_ticket_key_memcached_fail_count_ = 0;
schedule_next_tls_ticket_key_memcached_get(w);
if (!ticket_keys || ticket_keys->keys.empty()) {
LOG(WARN) << "Memcached: tls ticket keys are empty; tls ticket disabled";
set_ticket_keys(nullptr);
set_ticket_keys_to_worker(nullptr);
return;
}
if (LOG_ENABLED(INFO)) {
LOG(INFO) << "ticket keys get done";
LOG(INFO) << 0 << " enc+dec: "
<< util::format_hex(ticket_keys->keys[0].data.name);
for (size_t i = 1; i < ticket_keys->keys.size(); ++i) {
auto &key = ticket_keys->keys[i];
LOG(INFO) << i << " dec: " << util::format_hex(key.data.name);
}
}
set_ticket_keys(ticket_keys);
set_ticket_keys_to_worker(ticket_keys);
}
void
ConnectionHandler::schedule_next_tls_ticket_key_memcached_get(ev_timer *w) {
ev_timer_set(w, get_config()->tls_ticket_key_memcached_interval, 0.);
ev_timer_start(loop_, w);
}
} // namespace shrpx

15
src/shrpx_connection_handler.h
View File

@ -49,6 +49,7 @@ class AcceptHandler;
class Worker;
struct WorkerStat;
struct TicketKeys;
class MemcachedDispatcher;
struct OCSPUpdateContext {
// ocsp response buffer
@ -111,6 +112,17 @@ public:
// update.
void proceed_next_cert_ocsp();
void set_tls_ticket_key_memcached_dispatcher(
std::unique_ptr<MemcachedDispatcher> dispatcher);
MemcachedDispatcher *get_tls_ticket_key_memcached_dispatcher() const;
void on_tls_ticket_key_network_error(ev_timer *w);
void on_tls_ticket_key_not_found(ev_timer *w);
void
on_tls_ticket_key_get_success(const std::shared_ptr<TicketKeys> &ticket_keys,
ev_timer *w);
void schedule_next_tls_ticket_key_memcached_get(ev_timer *w);
private:
// Stores all SSL_CTX objects.
std::vector<SSL_CTX *> all_ssl_ctx_;
@ -120,6 +132,7 @@ private:
// Worker instance used when single threaded mode (-n1) is used.
// Otherwise, nullptr and workers_ has instances of Worker instead.
std::unique_ptr<Worker> single_worker_;
std::unique_ptr<MemcachedDispatcher> tls_ticket_key_memcached_dispatcher_;
// Current TLS session ticket keys. Note that TLS connection does
// not refer to this field directly. They use TicketKeys object in
// Worker object.
@ -131,6 +144,8 @@ private:
std::unique_ptr<AcceptHandler> acceptor6_;
ev_timer disable_acceptor_timer_;
ev_timer ocsp_timer_;
size_t tls_ticket_key_memcached_get_retry_count_;
size_t tls_ticket_key_memcached_fail_count_;
unsigned int worker_round_robin_cnt_;
bool graceful_shutdown_;
};

2
src/shrpx_memcached_connection.cc
View File

@ -100,7 +100,7 @@ namespace {
void clear_request(std::deque<std::unique_ptr<MemcachedRequest>> &q) {
for (auto &req : q) {
if (req->cb) {
req->cb(req.get(), MemcachedResult(MEMCACHED_ERR_ERROR));
req->cb(req.get(), MemcachedResult(MEMCACHED_ERR_EXT_NETWORK_ERROR));
}
}
q.clear();

4
src/shrpx_memcached_result.h
View File

@ -32,8 +32,8 @@
namespace shrpx {
enum MemcachedStatusCode {
MEMCACHED_ERR_OK,
MEMCACHED_ERR_ERROR = 0x1001,
MEMCACHED_ERR_NO_ERROR,
MEMCACHED_ERR_EXT_NETWORK_ERROR = 0x1001,
};
struct MemcachedResult {