From c09fde39e1125acf1dec6f7c32a1d35c21ba850b Mon Sep 17 00:00:00 2001 From: Scott Stack Date: Fri, 13 Mar 2026 13:39:33 -0400 Subject: [PATCH] tls: fix hang caused by infinite TLS retries This set of changes addresses an issue where `flb_tls_net_read|write` functions can hang and consume 100% CPU. The issue occurs when a TLS connection is lost, and the underlying openssl implementation repeatedly returns `SSL_ERROR_WANT_READ|WRITE`. If no `io_timeout` is configured, then the thread will enter a tight infinite loop retrying the read/write indefinitely until the process is restarted. This can be addressed by setting net.io_timeout config setting to something other than the default, but this set of changes attempts to address the case where no default is specified. The solution here is to simply default to a high value for the timeout if the setting is zero. This does not modify the net.io_timeout value, and only applies to this set of functions. Reasoning is that there should not be a case where the user would want to spin forever here. This change also adds a small delay in between retries so that even for the timeout case, it doesn't load the CPU unnecessarily while waiting for the next bit of data. Signed-off-by: Scott Stack --- include/fluent-bit/tls/flb_tls.h | 2 ++ src/tls/flb_tls.c | 48 +++++++++++++++++++++++--------- 2 files changed, 37 insertions(+), 13 deletions(-) diff --git a/include/fluent-bit/tls/flb_tls.h b/include/fluent-bit/tls/flb_tls.h index 018231218e2..6505bdb8a65 100644 --- a/include/fluent-bit/tls/flb_tls.h +++ b/include/fluent-bit/tls/flb_tls.h @@ -47,6 +47,8 @@ #define FLB_TLS_CLIENT_MODE 0 #define FLB_TLS_SERVER_MODE 1 +#define FLB_TLS_DEFAULT_IO_TIMEOUT_S 60 + struct flb_tls; struct flb_connection; diff --git a/src/tls/flb_tls.c b/src/tls/flb_tls.c index 40961b6a12a..33a085a88db 100644 --- a/src/tls/flb_tls.c +++ b/src/tls/flb_tls.c @@ -20,6 +20,7 @@ #include #include #include +#include #include "openssl.c" @@ -349,30 +350,32 @@ int flb_tls_net_read(struct flb_tls_session *session, void *buf, size_t len) time_t current_timestamp; struct flb_tls *tls; int ret; + int timeout_value; tls = session->tls; if (session->connection->net->io_timeout > 0) { - timeout_timestamp = time(NULL) + session->connection->net->io_timeout; + timeout_value = session->connection->net->io_timeout; } else { - timeout_timestamp = 0; + timeout_value = FLB_TLS_DEFAULT_IO_TIMEOUT_S; } + timeout_timestamp = time(NULL) + timeout_value; retry_read: ret = tls->api->net_read(session, buf, len); current_timestamp = time(NULL); - if (ret == FLB_TLS_WANT_READ) { - if (timeout_timestamp > 0 && - timeout_timestamp <= current_timestamp) { - return ret; + if (ret == FLB_TLS_WANT_READ || ret == FLB_TLS_WANT_WRITE) { + if (timeout_timestamp <= current_timestamp) { + flb_warn("[tls] Read timed out after %d seconds", timeout_value); + return -1; } - goto retry_read; - } - else if (ret == FLB_TLS_WANT_WRITE) { + /* add short delay to prevent CPU hogging */ + flb_time_msleep(5); + goto retry_read; } else if (ret < 0) { @@ -455,22 +458,41 @@ int flb_tls_net_read_async(struct flb_coro *co, int flb_tls_net_write(struct flb_tls_session *session, const void *data, size_t len, size_t *out_len) { + time_t timeout_timestamp; + time_t current_timestamp; size_t total; int ret; struct flb_tls *tls; + int timeout_value; total = 0; tls = session->tls; + if (session->connection->net->io_timeout > 0) { + timeout_value = session->connection->net->io_timeout; + } + else { + timeout_value = FLB_TLS_DEFAULT_IO_TIMEOUT_S; + } + timeout_timestamp = time(NULL) + timeout_value; + retry_write: ret = tls->api->net_write(session, (unsigned char *) data + total, len - total); - if (ret == FLB_TLS_WANT_WRITE) { - goto retry_write; - } - else if (ret == FLB_TLS_WANT_READ) { + current_timestamp = time(NULL); + + if (ret == FLB_TLS_WANT_WRITE || ret == FLB_TLS_WANT_READ) { + if (timeout_timestamp <= current_timestamp) { + flb_warn("[tls] Write timed out after %d seconds", timeout_value); + *out_len = total; + return -1; + } + + /* add short delay to prevent CPU hogging */ + flb_time_msleep(5); + goto retry_write; } else if (ret < 0) {