【发布时间】：2019-10-12 20:11:31
【问题描述】：
我正在开发一个简单的多线程 HTTP/s 服务器，使用非阻塞套接字来处理基于 Linux Epoll 的 HTTP 请求。它创建了4/8个线程（struct SSLWorker），每个线程都可以接受和处理连接，除了SSL_CTX（struct SSLContext）之外，连接之间没有共享数据。
每个新连接都会增加内存，断开连接后，内存永远不会释放。我想不出原因。好像和 SSL 代码有关，因为在使用 HTTP 请求时不会发生。
使用 valgrind、heaptrack 和leak sanitizers 并没有帮助，因为似乎内存仍然可以访问并且没有检测到泄漏。 清理 Connection struct SSL 和 BIO 数据的正确方法和顺序是什么？ (freeSsl())。 对我所缺少的有什么帮助吗？
注意事项： - 尝试了来自 Debian 和自建的 Openssl 1.1 和 1.1.1，它们的行为相同。 - 当客户端使用 TLS 1.2 时，内存增长更加明显 - 尝试禁用内部 SSL SESSION 缓存并设置 SSL_MODE_RELEASE_BUFFERS。 - 客户端断开连接。
struct SSLContext{ //Shared between worker threads
SSL_CTX *ssl_ctx{nullptr};
SSLContext(){
ERR_load_crypto_strings();
ERR_load_SSL_strings();
SSL_load_error_strings();
OpenSSL_add_all_algorithms();
int r = SSL_library_init();
ssl_ctx = SSL_CTX_new(SSLv23_method());
int r = SSL_CTX_use_certificate_file(ssl_ctx, global::cert_file.c_str(),
SSL_FILETYPE_PEM);
r = SSL_CTX_use_PrivateKey_file(ssl_ctx, global::key_file.c_str(), SSL_FILETYPE_PEM);
r = SSL_CTX_check_private_key(ssl_ctx);
}
virtual ~SSLContext(){
SSL_CTX_free(ssl_ctx);
ERR_free_strings();
}
};
struct Connection //Data associated to every connection
{
int sock_fd;
SSL *ssl{nullptr};
BIO *sbio{nullptr};// socket bio
BIO *io{nullptr};// buffer bio
BIO *ssl_bio{nullptr};// ssl bio
~Connection(){ //free ssl objects
if (ssl != nullptr) {
SSL_set_shutdown(ssl, SSL_RECEIVED_SHUTDOWN);
freeSsl();
if(sock_fd > 0) ::close(sock_fd);
}
}
void freeSsl(){ //free connection
//SSL_shutdown(ssl); it seems to be done by bellow free call.
//SSL_free(ssl); it seems to be done by bellow free call.
if(sbio != nullptr) {
BIO_free(sbio);
sbio = nullptr;
}
if(io != nullptr) {
BIO_flush(io);
BIO_free(io);
io = nullptr;
}
if(ssl_bio != nullptr) {
BIO_flush(ssl_bio);
BIO_free(ssl_bio);
ssl_bio = nullptr;
}
ssl = nullptr;
}
bool enableReadEvent(); //enable socket read events, set EPOLLIN | EPOLLET
bool enableWriteEvent();//enable socket write event, set EPOLLOUT | EPOLLET
};
struct SSLConnectionManager //ssl operations hanldler.
{
static SSLContext ssl_context;
bool handleHandshake(Connection &ssl_connection) //Initialize connection and do hanshake
{
if(ssl_connection.ssl != nullptr) ssl_connection.freeSsl();
ssl_connection.ssl = SSL_new(ssl_context->ssl_ctx);
ssl_connection.sbio = BIO_new_socket(ssl_connection.sock_fd, BIO_CLOSE);
SSL_set_bio(ssl_connection.ssl, ssl_connection.sbio, ssl_connection.sbio);
ssl_connection.io = BIO_new(BIO_f_buffer());
ssl_connection.ssl_bio = BIO_new(BIO_f_ssl());
BIO_set_ssl(ssl_connection.ssl_bio, ssl_connection.ssl, BIO_CLOSE);
BIO_push(ssl_connection.io, ssl_connection.ssl_bio);
SSL_set_accept_state(ssl_connection.ssl);
int r = SSL_do_handshake(ssl_connection.ssl);
if (r == 1) {
ssl_connection.ssl_connected = true;
ssl_connection.enableReadEvent();
return true;
}
int err = SSL_get_error(ssl_connection.ssl, r);
if (err == SSL_ERROR_WANT_WRITE || err == SSL_ERROR_WANT_READ) {
ssl_connection.enableReadEvent();
return true;
}
//"SSL_do_handshake error, abort connection
return false;
}
IO::IO_RESULT SSLConnectionManager::handleRead(Connection &ssl_connection) {
if (!ssl_connection.ssl_connected) {
return IO::IO_RESULT::SSL_NEED_HANDSHAKE;
}
int rc = -1;
int bytes_read = 0;
for (;;) {
rc = BIO_read(ssl_connection.io,
ssl_connection.buffer + ssl_connection.buffer_size,
static_cast<int>(MAX_DATA_SIZE - ssl_connection.buffer_size));
if (rc == 0) {
return bytes_read > 0 ? IO::IO_RESULT::SUCCESS : IO::IO_RESULT::ZERO_DATA_READ;
}else if (rc < 0) {
if (BIO_should_retry(ssl_connection.io)) {
return IO::IO_RESULT::DONE_TRY_AGAIN;
}
return IO::IO_RESULT::ERROR;
}
bytes_read += rc;
ssl_connection.buffer_size += static_cast<size_t>(rc);
}
return IO::IO_RESULT::SUCCESS;
}
IO::IO_RESULT SSLConnectionManager::handleWrite(Connection &ssl_connection,
const char *data, size_t data_size, size_t &written) {
if (!ssl_connection.ssl_connected) {
return IO::IO_RESULT::SSL_NEED_HANDSHAKE; // after we call handleHanshake
}
IO::IO_RESULT result;
int rc = -1;
written = 0;
for (;;) {
rc = BIO_write(ssl_connection.io, data + written, static_cast<int>(data_size - written));
if (rc == 0) {
result = IO::IO_RESULT::DONE_TRY_AGAIN;
break;
} else if (rc < 0) {
if (BIO_should_retry(ssl_connection.io)) {
result = IO::IO_RESULT::DONE_TRY_AGAIN;
break;
} else {
return IO::IO_RESULT::ERROR;
}
} else {
written += rc;
if ((data_size - written) == 0) {
result = IO::IO_RESULT::SUCCESS;
break;
};
}
}
BIO_flush(ssl_connection.io);
return result;
}
};
struct SSLWorker : EpollManager{ //Thread worker task.
SSLConnectionManager ssl_connection_manager;
bool onConnectEvent(Connection &ssl_connection){
auto sock_fd = accept(...);
Connection * new_connection = new Connection(sock_fd);
addToEventManager(*new_connection, EV_READ);
}
void doWork() {
is_running = true;
int res = 0;
epoll_event events[1024];
while (is_running) {
res = ::epoll_wait(epoll_fd, events, 1024, -1);
if (res < 0)
return;
for (int i = 0; i < res; i++) {
auto conn = static_cast<Connection *>(events[i].data.ptr);
if (events[i].events & (EPOLLHUP | EPOLLERR | EPOLLRDHUP)) {
delete conn; //remote closed connection, free all
} else {
if (events[i].events & EPOLLIN) {
if (conn->fd == global::listen_fd) {
onConnectEvent();
continue;
} else {
ssl_connection_manager.handleRead(*conn);
processRequest();
conn.enableWriteEvent();
}
}
if (events[i].events & EPOLLOUT) {
ssl_connection_manager.handleWrite(*conn);
}
}
}
}
}
};
在这里您可以在短暂执行后看到什么 heaptrack 报告泄漏。
【问题讨论】：
-
您可能想要使用
BIO_free_all而不是BIO_free。我怀疑后者是否也像您假设的那样释放了 SSL 对象。 -
我试过了，但是我得到了一个双重免费运行时错误，如果我在链的每个 BIO 对象上调用 free 就不会发生这种情况。我在“BIO* io”上调用 BIO_free_all。你会怎么做？
-
在 SSL_do_handshake 之后，如果发生故障，除了连接 BIO 之外，我是否需要释放任何东西？我怀疑“SSL_do_hanshake 中的泄漏已关闭”。我在切诺基服务器上看到的相同行为，几个月前我打开了一个问题。github.com/cherokee/webserver/issues/1217
标签： c++ ssl memory-leaks openssl epoll