Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 24 additions & 11 deletions aws_lambda_powertools/utilities/parser/functions.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
import logging
from typing import TYPE_CHECKING, Any

from pydantic import IPvAnyNetwork, TypeAdapter
from pydantic import IPvAnyNetwork, TypeAdapter, ValidationError

from aws_lambda_powertools.shared.cache_dict import LRUDict

Expand Down Expand Up @@ -49,6 +49,28 @@ def _retrieve_or_set_model_from_cache(model: type[T]) -> TypeAdapter[T]:
return CACHE_TYPE_ADAPTER[id_model]


def _parse_string_payload(data: str, adapter: TypeAdapter[T]):
"""Validate a string payload as JSON first, then as a plain Python value.

Falls back to Python validation when Pydantic cannot validate the type from JSON
(see https://github.com/aws-powertools/powertools-lambda-python/issues/5303) and when
the string is not JSON at all, e.g. the plain-text Lambda runtime lines that CloudWatch Logs
subscriptions forward (see https://github.com/aws-powertools/powertools-lambda-python/issues/8695).
If the fallback fails too, the original JSON error is raised so callers see the same message.
"""
try:
return adapter.validate_json(data)
except NotImplementedError:
# See: https://github.com/pydantic/pydantic/issues/8890
logger.debug("Falling back to Python validation due to Pydantic implementation.")
return adapter.validate_python(json.loads(data))
except ValidationError as exc:
try:
return adapter.validate_python(data)
except ValidationError:
raise exc from None


def _parse_and_validate_event(data: dict[str, Any] | Any, adapter: TypeAdapter[T]):
"""
Parse and validate the event data using the provided adapter.
Expand All @@ -70,16 +92,7 @@ def _parse_and_validate_event(data: dict[str, Any] | Any, adapter: TypeAdapter[T

if isinstance(data, str):
logger.debug("Parsing event as string")
try:
return adapter.validate_json(data)
except NotImplementedError:
# See: https://github.com/aws-powertools/powertools-lambda-python/issues/5303
# See: https://github.com/pydantic/pydantic/issues/8890
logger.debug(
"Falling back to Python validation due to Pydantic implementation."
"See issue: https://github.com/aws-powertools/powertools-lambda-python/issues/5303",
)
data = json.loads(data)
return _parse_string_payload(data, adapter)

return adapter.validate_python(data)

Expand Down
60 changes: 60 additions & 0 deletions tests/unit/parser/_pydantic/test_cloudwatch.py
Original file line number Diff line number Diff line change
Expand Up @@ -126,3 +126,63 @@ def test_handle_invalid_event_with_envelope():
empty_dict = {}
with pytest.raises(ValidationError):
CloudWatchLogsModel(**empty_dict)


def _make_cloudwatch_event(messages: list[str]) -> dict:
inner_event_dict = {
"messageType": "DATA_MESSAGE",
"owner": "123456789012",
"logGroup": "/aws/lambda/my-function",
"logStream": "2024/10/21/[$LATEST]abcdef1234567890",
"subscriptionFilters": ["my-filter"],
"logEvents": [
{"id": f"event-{i}", "timestamp": 1729520895158 + i, "message": message}
for i, message in enumerate(messages)
],
}
compressed = zlib.compress(json.dumps(inner_event_dict).encode("utf-8"), -1)
return {"awslogs": {"data": base64.b64encode(compressed)}}


def test_cloudwatch_envelope_handles_non_json_messages():
plain_text_messages = [
"START RequestId: eadc91c8-8273-4a2d-b170-b0a7883b619e Version: $LATEST\n",
"2024-10-21T14:28:15.158Z\teadc91c8-8273-4a2d-b170-b0a7883b619e\tINFO\tHello from handler\n",
"END RequestId: eadc91c8-8273-4a2d-b170-b0a7883b619e\n",
(
"REPORT RequestId: eadc91c8-8273-4a2d-b170-b0a7883b619e\tDuration: 2.07 ms\t"
"Billed Duration: 3 ms\tMemory Size: 128 MB\tMax Memory Used: 67 MB\t\n"
),
]

parsed = parse(
event=_make_cloudwatch_event(plain_text_messages),
model=str,
envelope=envelopes.CloudWatchLogsEnvelope,
)

assert parsed == plain_text_messages


def test_cloudwatch_envelope_handles_mixed_json_and_plain_text_messages():
messages = [
json.dumps({"my_message": "hello", "user": "test"}),
"START RequestId: eadc91c8-8273-4a2d-b170-b0a7883b619e Version: $LATEST\n",
]

parsed = parse(
event=_make_cloudwatch_event(messages),
model=str,
envelope=envelopes.CloudWatchLogsEnvelope,
)

assert parsed == messages


def test_cloudwatch_envelope_plain_text_still_fails_for_object_model():
with pytest.raises(ValidationError):
parse(
event=_make_cloudwatch_event(["START RequestId: eadc91c8-8273-4a2d-b170-b0a7883b619e\n"]),
model=MyCloudWatchBusiness,
envelope=envelopes.CloudWatchLogsEnvelope,
)