Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 8 additions & 3 deletions aws_lambda_powertools/logging/buffer/cache.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,11 @@
from typing import Any


def _item_size(item: Any) -> int:
"""Return the UTF-8 encoded size of an item, in bytes."""
return len(str(item).encode("utf-8"))


class KeyBufferCache:
"""
A cache implementation for a single key with size tracking and eviction support.
Expand Down Expand Up @@ -39,7 +44,7 @@ def add(self, item: Any) -> None:
item : Any
The item to be stored in the cache.
"""
item_size = len(str(item))
item_size = _item_size(item)
self.cache.append(item)
self.current_size += item_size

Expand All @@ -53,7 +58,7 @@ def remove_oldest(self) -> Any:
The removed item.
"""
removed_item = self.cache.popleft()
self.current_size -= len(str(removed_item))
self.current_size -= _item_size(removed_item)
self.has_evicted = True
return removed_item

Expand Down Expand Up @@ -131,7 +136,7 @@ def add(self, key: str, item: Any) -> None:
True if item was added, False otherwise.
"""
# Check if item is larger than entire buffer
item_size = len(str(item))
item_size = _item_size(item)
if item_size > self.max_size_bytes:
raise BufferError("Cannot add item to the buffer")

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -319,6 +319,27 @@ def test_log_record_exceeding_buffer_size(stdout, service_name, monkeypatch):
assert log[0]["message"] == message


def test_log_record_exceeding_buffer_size_in_bytes(stdout, service_name, monkeypatch):
monkeypatch.setenv(constants.XRAY_TRACE_ID_ENV, "1-67c39786-5908a82a246fb67f3089263f")

# GIVEN a message that fits the buffer in characters but not in UTF-8 bytes
message = "é" * 1000

# GIVEN A logger configured with a 1600-byte buffer
logger_buffer_config = LoggerBufferConfig(max_bytes=1600, buffer_at_verbosity="DEBUG")

logger = Logger(level="DEBUG", service=service_name, stream=stdout, buffer_config=logger_buffer_config)

# WHEN Logging the message
# THEN A warning should be raised because the record is larger than max_bytes in bytes
with pytest.warns(PowertoolsUserWarning, match="Cannot add item to the buffer*"):
logger.debug(message)

# THEN the log must be flushed to avoid data loss
log = capture_multiple_logging_statements_output(stdout)
assert log[0]["message"] == message


@pytest.mark.parametrize("log_level", ["WARNING", "INFO"])
def test_logger_buffer_log_output_for_levels_above_minimum(log_level, stdout, service_name, monkeypatch):
monkeypatch.setenv(constants.XRAY_TRACE_ID_ENV, "1-67c39786-5908a82a246fb67f3089263f")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -164,3 +164,27 @@ def test_cache_size_tracking():
# THEN current size is tracked correctly
assert logger_cache.get_current_size("key1") == len("small") + len("another_item")
assert logger_cache.get_current_size("key1") <= 30


def test_cache_size_counts_utf8_bytes():
# GIVEN a new instance of LoggerBufferCache with 12 bytes
logger_cache = LoggerBufferCache(12)

# WHEN adding two non-ASCII items of 5 characters and 10 bytes each
logger_cache.add("key1", "ééééé")
logger_cache.add("key1", "ééééé")

# THEN the size is counted in bytes and the oldest item is evicted
assert logger_cache.get_current_size("key1") == 10
assert logger_cache.get("key1") == ["ééééé"]
assert logger_cache.has_items_evicted("key1") is True


def test_item_larger_than_cache_in_bytes():
# GIVEN a new instance of LoggerBufferCache with 8 bytes
logger_cache = LoggerBufferCache(8)

# WHEN adding an item of 5 characters and 10 bytes
# THEN it is rejected because its encoded size exceeds the cache
with pytest.raises(BufferError):
logger_cache.add("key1", "ééééé")