1
0
mirror of https://github.com/esphome/esphome.git synced 2026-06-23 17:38:54 +02:00

[logger] Move task log buffer storage to BSS (#15153)

This commit is contained in:
J. Nick Koston
2026-03-24 14:52:37 -10:00
committed by GitHub
parent 26e78c840c
commit 690dc324c9
15 changed files with 69 additions and 127 deletions

View File

@@ -331,9 +331,8 @@ async def to_code(config: ConfigType) -> None:
CORE.data.setdefault(CONF_LOGGER, {})[CONF_LEVEL] = level
tx_buffer_size = config[CONF_TX_BUFFER_SIZE]
cg.add_define("ESPHOME_LOGGER_TX_BUFFER_SIZE", tx_buffer_size)
# Determine task log buffer size and define USE_ESPHOME_TASK_LOG_BUFFER early
# so the constructor can allocate the buffer immediately, preventing a race
# where another task logs before the buffer is initialized.
# Determine task log buffer size. The buffer is a direct member of Logger
# (no separate heap allocation).
task_log_buffer_size = 0
if CORE.is_esp32 or CORE.is_libretiny or CORE.is_nrf52:
task_log_buffer_size = config[CONF_TASK_LOG_BUFFER_SIZE]
@@ -341,12 +340,7 @@ async def to_code(config: ConfigType) -> None:
task_log_buffer_size = 64 # Fixed 64 slots for host
if task_log_buffer_size > 0:
cg.add_define("USE_ESPHOME_TASK_LOG_BUFFER")
log = cg.new_Pvariable(
config[CONF_ID],
baud_rate,
task_log_buffer_size,
)
else:
cg.add_define("ESPHOME_TASK_LOG_BUFFER_SIZE", task_log_buffer_size)
log = cg.new_Pvariable(
config[CONF_ID],
baud_rate,

View File

@@ -83,7 +83,7 @@ void Logger::log_vprintf_non_main_thread_(uint8_t level, const char *tag, int li
#ifdef USE_ESPHOME_TASK_LOG_BUFFER
// For non-main threads/tasks, queue the message for callbacks
message_sent =
this->log_buffer_->send_message_thread_safe(level, tag, static_cast<uint16_t>(line), thread_name, format, args);
this->log_buffer_.send_message_thread_safe(level, tag, static_cast<uint16_t>(line), thread_name, format, args);
if (message_sent) {
// Enable logger loop to process the buffered message
// This is safe to call from any context including ISRs
@@ -152,23 +152,13 @@ inline uint8_t Logger::level_for(const char *tag) {
return this->current_level_;
}
#ifdef USE_ESPHOME_TASK_LOG_BUFFER
Logger::Logger(uint32_t baud_rate, size_t task_log_buffer_size) : baud_rate_(baud_rate) {
#else
Logger::Logger(uint32_t baud_rate) : baud_rate_(baud_rate) {
#endif
#if defined(USE_ESP32) || defined(USE_LIBRETINY)
this->main_task_ = xTaskGetCurrentTaskHandle();
#elif defined(USE_ZEPHYR)
this->main_task_ = k_current_get();
#elif defined(USE_HOST)
this->main_thread_ = pthread_self();
#endif
#ifdef USE_ESPHOME_TASK_LOG_BUFFER
// NOLINTNEXTLINE(cppcoreguidelines-owning-memory) - allocated once, never freed
this->log_buffer_ = new logger::TaskLogBuffer(task_log_buffer_size);
// Note: we don't disable loop here because the component isn't registered with App yet.
// The loop self-disables on its first iteration when it finds no messages to process.
this->main_thread_ = pthread_self();
#endif
}
@@ -184,16 +174,16 @@ void Logger::loop() {
void Logger::process_messages_() {
#ifdef USE_ESPHOME_TASK_LOG_BUFFER
// Process any buffered messages when available
if (this->log_buffer_->has_messages()) {
if (this->log_buffer_.has_messages()) {
logger::TaskLogBuffer::LogMessage *message;
uint16_t text_length;
while (this->log_buffer_->borrow_message_main_loop(message, text_length)) {
while (this->log_buffer_.borrow_message_main_loop(message, text_length)) {
const char *thread_name = message->thread_name[0] != '\0' ? message->thread_name : nullptr;
LogBuffer buf{this->tx_buffer_, ESPHOME_LOGGER_TX_BUFFER_SIZE};
this->format_buffered_message_and_notify_(message->level, message->tag, message->line, thread_name,
message->text_data(), text_length, buf);
// Release the message to allow other tasks to use it as soon as possible
this->log_buffer_->release_message_main_loop();
this->log_buffer_.release_message_main_loop();
this->write_log_buffer_to_console_(buf);
}
}
@@ -239,13 +229,11 @@ void Logger::dump_config() {
this->baud_rate_, LOG_STR_ARG(get_uart_selection_()));
#endif
#ifdef USE_ESPHOME_TASK_LOG_BUFFER
if (this->log_buffer_) {
#ifdef USE_HOST
ESP_LOGCONFIG(TAG, " Task Log Buffer Slots: %u", static_cast<unsigned int>(this->log_buffer_->size()));
ESP_LOGCONFIG(TAG, " Task Log Buffer Slots: %u", static_cast<unsigned int>(this->log_buffer_.size()));
#else
ESP_LOGCONFIG(TAG, " Task Log Buffer Size: %u bytes", static_cast<unsigned int>(this->log_buffer_->size()));
ESP_LOGCONFIG(TAG, " Task Log Buffer Size: %u bytes", static_cast<unsigned int>(this->log_buffer_.size()));
#endif
}
#endif
#ifdef USE_LOGGER_RUNTIME_TAG_LEVELS

View File

@@ -143,11 +143,7 @@ enum UARTSelection : uint8_t {
*/
class Logger final : public Component {
public:
#ifdef USE_ESPHOME_TASK_LOG_BUFFER
explicit Logger(uint32_t baud_rate, size_t task_log_buffer_size);
#else
explicit Logger(uint32_t baud_rate);
#endif
#if defined(USE_ESPHOME_TASK_LOG_BUFFER) || (defined(USE_ZEPHYR) && defined(USE_LOGGER_UART_SELECTION_USB_CDC))
void loop() override;
#endif
@@ -353,10 +349,6 @@ class Logger final : public Component {
#ifdef USE_LOGGER_LEVEL_LISTENERS
std::vector<LoggerLevelListener *> level_listeners_; // Log level change listeners
#endif
#ifdef USE_ESPHOME_TASK_LOG_BUFFER
logger::TaskLogBuffer *log_buffer_{nullptr}; // Allocated once, never freed
#endif
// Group smaller types together at the end
uint8_t current_level_{ESPHOME_LOG_LEVEL_VERY_VERBOSE};
#if defined(USE_ESP32) || defined(USE_ESP8266) || defined(USE_RP2040) || defined(USE_ZEPHYR)
@@ -374,8 +366,11 @@ class Logger final : public Component {
bool global_recursion_guard_{false}; // Simple global recursion guard for single-task platforms
#endif
// Large buffer placed last to keep frequently-accessed member offsets small
// Large buffers placed last to keep frequently-accessed member offsets small
char tx_buffer_[ESPHOME_LOGGER_TX_BUFFER_SIZE + 1]; // +1 for null terminator
#ifdef USE_ESPHOME_TASK_LOG_BUFFER
logger::TaskLogBuffer log_buffer_; // Embedded in Logger (no separate heap allocation)
#endif
// --- get_thread_name_ overloads (per-platform) ---

View File

@@ -1,33 +1,23 @@
#ifdef USE_ESP32
#include "task_log_buffer_esp32.h"
#include "esphome/core/helpers.h"
#include "esphome/core/log.h"
#ifdef USE_ESPHOME_TASK_LOG_BUFFER
namespace esphome::logger {
TaskLogBuffer::TaskLogBuffer(size_t total_buffer_size) {
// Store the buffer size
this->size_ = total_buffer_size;
// Allocate memory for the ring buffer using ESPHome's RAM allocator
RAMAllocator<uint8_t> allocator;
this->storage_ = allocator.allocate(this->size_);
TaskLogBuffer::TaskLogBuffer() {
// Create a static ring buffer with RINGBUF_TYPE_NOSPLIT for message integrity
this->ring_buffer_ = xRingbufferCreateStatic(this->size_, RINGBUF_TYPE_NOSPLIT, this->storage_, &this->structure_);
// Storage is a member array (embedded in Logger), no heap allocation needed
this->ring_buffer_ =
xRingbufferCreateStatic(sizeof(this->storage_), RINGBUF_TYPE_NOSPLIT, this->storage_, &this->structure_);
}
TaskLogBuffer::~TaskLogBuffer() {
if (this->ring_buffer_ != nullptr) {
// Delete the ring buffer
vRingbufferDelete(this->ring_buffer_);
this->ring_buffer_ = nullptr;
// Free the allocated memory
RAMAllocator<uint8_t> allocator;
allocator.deallocate(this->storage_, this->size_);
this->storage_ = nullptr;
}
}

View File

@@ -8,7 +8,6 @@
#ifdef USE_ESPHOME_TASK_LOG_BUFFER
#include <cstddef>
#include <cstring>
#include <memory>
#include <atomic>
#include <freertos/FreeRTOS.h>
#include <freertos/ringbuf.h>
@@ -47,8 +46,7 @@ class TaskLogBuffer {
inline const char *text_data() const { return reinterpret_cast<const char *>(this) + sizeof(LogMessage); }
};
// Constructor that takes a total buffer size
explicit TaskLogBuffer(size_t total_buffer_size);
TaskLogBuffer();
~TaskLogBuffer();
// NOT thread-safe - borrow a message from the ring buffer, only call from main loop
@@ -67,13 +65,12 @@ class TaskLogBuffer {
}
// Get the total buffer size in bytes
inline size_t size() const { return size_; }
static constexpr size_t size() { return ESPHOME_TASK_LOG_BUFFER_SIZE; }
private:
RingbufHandle_t ring_buffer_{nullptr}; // FreeRTOS ring buffer handle
StaticRingbuffer_t structure_; // Static structure for the ring buffer
uint8_t *storage_{nullptr}; // Pointer to allocated memory
size_t size_{0}; // Size of allocated memory
uint8_t storage_[ESPHOME_TASK_LOG_BUFFER_SIZE]; // Embedded in Logger (no separate heap allocation)
// Atomic counter for message tracking (only differences matter)
std::atomic<uint16_t> message_counter_{0}; // Incremented when messages are committed

View File

@@ -10,22 +10,13 @@
namespace esphome::logger {
TaskLogBuffer::TaskLogBuffer(size_t slot_count) : slot_count_(slot_count) {
// Allocate message slots
this->slots_ = std::make_unique<LogMessage[]>(slot_count);
}
TaskLogBuffer::~TaskLogBuffer() {
// unique_ptr handles cleanup automatically
}
int TaskLogBuffer::acquire_write_slot_() {
// Try to reserve a slot using compare-and-swap
size_t current_reserve = this->reserve_index_.load(std::memory_order_relaxed);
while (true) {
// Calculate next index (with wrap-around)
size_t next_reserve = (current_reserve + 1) % this->slot_count_;
size_t next_reserve = (current_reserve + 1) % ESPHOME_TASK_LOG_BUFFER_SIZE;
// Check if buffer would be full
// Buffer is full when next write position equals read position
@@ -50,7 +41,7 @@ void TaskLogBuffer::commit_write_slot_(int slot_index) {
// Try to advance the write_index if we're the next expected commit
// This ensures messages are read in order
size_t expected = slot_index;
size_t next = (slot_index + 1) % this->slot_count_;
size_t next = (slot_index + 1) % ESPHOME_TASK_LOG_BUFFER_SIZE;
// We only advance write_index if this slot is the next one expected
// This handles out-of-order commits correctly
@@ -63,7 +54,7 @@ void TaskLogBuffer::commit_write_slot_(int slot_index) {
// Successfully advanced, check if next slot is also ready
expected = next;
next = (next + 1) % this->slot_count_;
next = (next + 1) % ESPHOME_TASK_LOG_BUFFER_SIZE;
if (!this->slots_[expected].ready.load(std::memory_order_acquire)) {
break;
}
@@ -142,7 +133,7 @@ void TaskLogBuffer::release_message_main_loop() {
this->slots_[current_read].ready.store(false, std::memory_order_release);
// Advance read index
size_t next_read = (current_read + 1) % this->slot_count_;
size_t next_read = (current_read + 1) % ESPHOME_TASK_LOG_BUFFER_SIZE;
this->read_index_.store(next_read, std::memory_order_release);
}

View File

@@ -11,7 +11,6 @@
#include <cstdarg>
#include <cstddef>
#include <cstring>
#include <memory>
#include <pthread.h>
namespace esphome::logger {
@@ -50,9 +49,6 @@ namespace esphome::logger {
*/
class TaskLogBuffer {
public:
// Default number of message slots - host has plenty of memory
static constexpr size_t DEFAULT_SLOT_COUNT = 64;
// Structure for a log message (fixed size for lock-free operation)
struct LogMessage {
// Size constants
@@ -74,9 +70,7 @@ class TaskLogBuffer {
inline char *text_data() { return this->text; }
};
/// Constructor that takes the number of message slots
explicit TaskLogBuffer(size_t slot_count);
~TaskLogBuffer();
TaskLogBuffer() = default;
// NOT thread-safe - get next message from buffer, only call from main loop
// Returns true if a message was retrieved, false if buffer is empty
@@ -96,7 +90,7 @@ class TaskLogBuffer {
}
// Get the buffer size (number of slots)
inline size_t size() const { return slot_count_; }
static constexpr size_t size() { return ESPHOME_TASK_LOG_BUFFER_SIZE; }
private:
// Acquire a slot for writing (thread-safe)
@@ -106,8 +100,7 @@ class TaskLogBuffer {
// Commit a slot after writing (thread-safe)
void commit_write_slot_(int slot_index);
std::unique_ptr<LogMessage[]> slots_; // Pre-allocated message slots
size_t slot_count_; // Number of slots
LogMessage slots_[ESPHOME_TASK_LOG_BUFFER_SIZE]; // Embedded in Logger (no separate heap allocation)
// Lock-free indices using atomics
// - reserve_index_: Next slot to reserve (producers CAS this to claim slots)

View File

@@ -1,19 +1,15 @@
#ifdef USE_LIBRETINY
#include "task_log_buffer_libretiny.h"
#include "esphome/core/helpers.h"
#include "esphome/core/log.h"
#ifdef USE_ESPHOME_TASK_LOG_BUFFER
namespace esphome::logger {
TaskLogBuffer::TaskLogBuffer(size_t total_buffer_size) {
this->size_ = total_buffer_size;
// Allocate memory for the circular buffer using ESPHome's RAM allocator
RAMAllocator<uint8_t> allocator;
this->storage_ = allocator.allocate(this->size_);
TaskLogBuffer::TaskLogBuffer() {
// Create mutex for thread-safe access
// Storage is a member array (embedded in Logger), no heap allocation needed
this->mutex_ = xSemaphoreCreateMutex();
}
@@ -22,11 +18,6 @@ TaskLogBuffer::~TaskLogBuffer() {
vSemaphoreDelete(this->mutex_);
this->mutex_ = nullptr;
}
if (this->storage_ != nullptr) {
RAMAllocator<uint8_t> allocator;
allocator.deallocate(this->storage_, this->size_);
this->storage_ = nullptr;
}
}
size_t TaskLogBuffer::available_contiguous_space() const {
@@ -34,7 +25,7 @@ size_t TaskLogBuffer::available_contiguous_space() const {
// head is ahead of or equal to tail
// Available space is from head to end, plus from start to tail
// But for contiguous, just from head to end (minus 1 to avoid head==tail ambiguity)
size_t space_to_end = this->size_ - this->head_;
size_t space_to_end = ESPHOME_TASK_LOG_BUFFER_SIZE - this->head_;
if (this->tail_ == 0) {
// Can't use the last byte or head would equal tail
return space_to_end > 0 ? space_to_end - 1 : 0;
@@ -48,8 +39,8 @@ size_t TaskLogBuffer::available_contiguous_space() const {
}
bool TaskLogBuffer::borrow_message_main_loop(LogMessage *&message, uint16_t &text_length) {
// Check if buffer was initialized successfully
if (this->mutex_ == nullptr || this->storage_ == nullptr) {
// Check if mutex was initialized successfully
if (this->mutex_ == nullptr) {
return false;
}
@@ -86,7 +77,7 @@ void TaskLogBuffer::release_message_main_loop() {
this->tail_ += this->current_message_size_;
// Handle wrap-around if we've reached the end
if (this->tail_ >= this->size_) {
if (this->tail_ >= ESPHOME_TASK_LOG_BUFFER_SIZE) {
this->tail_ = 0;
}
@@ -115,9 +106,9 @@ bool TaskLogBuffer::send_message_thread_safe(uint8_t level, const char *tag, uin
// Calculate total size needed (header + text length + null terminator)
size_t total_size = message_total_size(text_length);
// Check if buffer was initialized successfully
if (this->mutex_ == nullptr || this->storage_ == nullptr) {
return false; // Buffer not initialized, fall back to direct output
// Check if mutex was initialized successfully
if (this->mutex_ == nullptr) {
return false; // Mutex not initialized, fall back to direct output
}
// Try to acquire mutex without blocking - don't block logging tasks
@@ -185,7 +176,7 @@ bool TaskLogBuffer::send_message_thread_safe(uint8_t level, const char *tag, uin
this->head_ += total_size;
// Handle wrap-around (shouldn't happen due to contiguous space check, but be safe)
if (this->head_ >= this->size_) {
if (this->head_ >= ESPHOME_TASK_LOG_BUFFER_SIZE) {
this->head_ = 0;
}

View File

@@ -59,8 +59,7 @@ class TaskLogBuffer {
// Valid log levels are 0-7, so 0xFF cannot be a real message
static constexpr uint8_t PADDING_MARKER_LEVEL = 0xFF;
// Constructor that takes a total buffer size
explicit TaskLogBuffer(size_t total_buffer_size);
TaskLogBuffer();
~TaskLogBuffer();
// NOT thread-safe - borrow a message from the buffer, only call from main loop
@@ -78,7 +77,7 @@ class TaskLogBuffer {
inline bool HOT has_messages() const { return this->message_count_ != 0; }
// Get the total buffer size in bytes
inline size_t size() const { return this->size_; }
static constexpr size_t size() { return ESPHOME_TASK_LOG_BUFFER_SIZE; }
private:
// Calculate total size needed for a message (header + text + null terminator)
@@ -87,8 +86,7 @@ class TaskLogBuffer {
// Calculate available contiguous space at write position
size_t available_contiguous_space() const;
uint8_t *storage_{nullptr}; // Pointer to allocated memory
size_t size_{0}; // Size of allocated memory
uint8_t storage_[ESPHOME_TASK_LOG_BUFFER_SIZE]; // Embedded in Logger (no separate heap allocation)
size_t head_{0}; // Write position
size_t tail_{0}; // Read position

View File

@@ -17,19 +17,16 @@ static inline uint32_t get_wlen(const mpsc_pbuf_generic *item) {
return total_size_in_32bit_words(reinterpret_cast<const TaskLogBuffer::LogMessage *>(item)->text_length);
}
TaskLogBuffer::TaskLogBuffer(size_t total_buffer_size) {
// alignment to 4 bytes
total_buffer_size = (total_buffer_size + 3) / sizeof(uint32_t);
this->mpsc_config_.buf = new uint32_t[total_buffer_size];
this->mpsc_config_.size = total_buffer_size;
TaskLogBuffer::TaskLogBuffer() {
// Storage is a member array (embedded in Logger), no heap allocation needed
this->mpsc_config_.buf = this->buf_storage_;
this->mpsc_config_.size = BUF_WORD_COUNT;
this->mpsc_config_.flags = MPSC_PBUF_MODE_OVERWRITE;
this->mpsc_config_.get_wlen = get_wlen,
this->mpsc_config_.get_wlen = get_wlen;
mpsc_pbuf_init(&this->log_buffer_, &this->mpsc_config_);
}
TaskLogBuffer::~TaskLogBuffer() { delete[] this->mpsc_config_.buf; }
bool TaskLogBuffer::send_message_thread_safe(uint8_t level, const char *tag, uint16_t line, const char *thread_name,
const char *format, va_list args) {
// First, calculate the exact length needed using a null buffer (no actual writing)

View File

@@ -33,15 +33,14 @@ class TaskLogBuffer {
// Methods for accessing message contents
inline char *text_data() { return reinterpret_cast<char *>(this) + sizeof(LogMessage); }
};
// Constructor that takes a total buffer size
explicit TaskLogBuffer(size_t total_buffer_size);
~TaskLogBuffer();
TaskLogBuffer();
~TaskLogBuffer() = default;
// Check if there are messages ready to be processed using an atomic counter for performance
inline bool HOT has_messages() { return mpsc_pbuf_is_pending(&this->log_buffer_); }
// Get the total buffer size in bytes
inline size_t size() const { return this->mpsc_config_.size * sizeof(uint32_t); }
static constexpr size_t size() { return BUF_WORD_COUNT * sizeof(uint32_t); }
// NOT thread-safe - borrow a message from the ring buffer, only call from main loop
bool borrow_message_main_loop(LogMessage *&message, uint16_t &text_length);
@@ -54,6 +53,9 @@ class TaskLogBuffer {
const char *format, va_list args);
protected:
// Round up byte size to 32-bit word count for mpsc_pbuf alignment requirement
static constexpr size_t BUF_WORD_COUNT = (ESPHOME_TASK_LOG_BUFFER_SIZE + 3) / sizeof(uint32_t);
uint32_t buf_storage_[BUF_WORD_COUNT]; // Embedded in Logger (no separate heap allocation)
mpsc_pbuf_buffer_config mpsc_config_{};
mpsc_pbuf_buffer log_buffer_{};
const mpsc_pbuf_generic *current_token_{};

View File

@@ -200,6 +200,7 @@
#define USE_ESP32_CRASH_HANDLER
#define USE_MQTT_IDF_ENQUEUE
#define USE_ESPHOME_TASK_LOG_BUFFER
#define ESPHOME_TASK_LOG_BUFFER_SIZE 768
#define USE_OTA_ROLLBACK
#define USE_ESP32_MIN_CHIP_REVISION_SET
#define USE_ESP32_SRAM1_AS_IRAM
@@ -373,18 +374,23 @@
#define USE_WEBSERVER
#define USE_WEBSERVER_AUTH
#define USE_WEBSERVER_PORT 80 // NOLINT
#define USE_ESPHOME_TASK_LOG_BUFFER
#define ESPHOME_TASK_LOG_BUFFER_SIZE 768
#endif
#ifdef USE_HOST
#define USE_HTTP_REQUEST_RESPONSE
#define USE_SOCKET_IMPL_BSD_SOCKETS
#define USE_SOCKET_SELECT_SUPPORT
#define USE_ESPHOME_TASK_LOG_BUFFER
#define ESPHOME_TASK_LOG_BUFFER_SIZE 64
#endif
#ifdef USE_NRF52
#define ESPHOME_BLE_NUS_TX_RING_BUFFER_SIZE 512
#define ESPHOME_BLE_NUS_RX_RING_BUFFER_SIZE 512
#define USE_ESPHOME_TASK_LOG_BUFFER
#define ESPHOME_TASK_LOG_BUFFER_SIZE 768
#define USE_LOGGER_EARLY_MESSAGE
#define USE_LOGGER_UART_SELECTION_USB_CDC
#define USE_LOGGER_USB_CDC

View File

@@ -26,7 +26,7 @@ void setup() {
// Log functions call global_logger->log_vprintf_() without a null check,
// so we must set up a Logger before any test that triggers logging.
static esphome::logger::Logger test_logger(0, 64);
static esphome::logger::Logger test_logger(0);
test_logger.set_log_level(ESPHOME_LOG_LEVEL);
test_logger.pre_setup();

View File

@@ -22,7 +22,7 @@ void original_setup() {
void setup() {
// Log functions call global_logger->log_vprintf_() without a null check,
// so we must set up a Logger before any test that triggers logging.
static esphome::logger::Logger test_logger(0, 64);
static esphome::logger::Logger test_logger(0);
test_logger.set_log_level(ESPHOME_LOG_LEVEL);
test_logger.pre_setup();

View File

@@ -15,7 +15,7 @@ void setup() {
static char name[] = "livingroom";
static char friendly_name[] = "LivingRoom";
App.pre_setup(name, sizeof(name) - 1, friendly_name, sizeof(friendly_name) - 1);
auto *log = new logger::Logger(115200, 512); // NOLINT
auto *log = new logger::Logger(115200); // NOLINT
log->pre_setup();
log->set_uart_selection(logger::UART_SELECTION_UART0);
App.register_component_(log);