Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -48,9 +48,12 @@ class CsvTransform(BaseTransform):
"""

def transform(self, input_stream: IO[bytes]) -> DictReader:
encoding = self.transform_options.pop("encoding", "utf-8")
newline = self.transform_options.pop("newline", None)
# Work on a copy: the same transform can be applied more than once (e.g. a reused instance, or when
# S3Object re-creates its transformed stream), and each call must see the original options.
options = dict(self.transform_options)
encoding = options.pop("encoding", "utf-8")
newline = options.pop("newline", None)

# csv module needs an Iterator[str], so we wrap the underlying stream into a TextIO
iterator = io.TextIOWrapper(input_stream, encoding=encoding, newline=newline)
return csv.DictReader(iterator, **self.transform_options)
return csv.DictReader(iterator, **options)
24 changes: 23 additions & 1 deletion tests/functional/streaming/_boto3/test_s3_object.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
from aws_lambda_powertools.utilities.streaming import S3Object
from aws_lambda_powertools.utilities.streaming._s3_seekable_io import _S3SeekableIO
from aws_lambda_powertools.utilities.streaming.compat import PowertoolsStreamingBody
from aws_lambda_powertools.utilities.streaming.transformations import GzipTransform, ZipTransform
from aws_lambda_powertools.utilities.streaming.transformations import CsvTransform, GzipTransform, ZipTransform


@pytest.fixture
Expand Down Expand Up @@ -114,3 +114,25 @@ def test_s3_closed_does_not_make_requests_for_zip(s3_client, s3_client_stub):
# THEN no S3 request is issued (the stubber has no responses queued, so any call would fail)
assert obj.closed is False
s3_client_stub.assert_no_pending_responses()


def _stub_get_object(s3_client_stub, payload: bytes):
streaming_body = PowertoolsStreamingBody(raw_stream=io.BytesIO(payload), content_length=len(payload))
s3_client_stub.add_response("get_object", {"Body": streaming_body})


def test_csv_transform_keeps_options_when_reused(s3_client, s3_client_stub):
# GIVEN a single CsvTransform with a non-default encoding applied to two objects
transform = CsvTransform(encoding="latin-1", delimiter=";")
payload = "name;city\ncafé;Zürich\n".encode("latin-1")

for _ in range(2):
_stub_get_object(s3_client_stub, payload)

# WHEN each object is transformed
rows = list(S3Object(bucket="bucket", key="key", boto3_client=s3_client).transform(transform))

# THEN the encoding is honoured every time, not only on the first use
assert rows == [{"name": "café", "city": "Zürich"}]

assert transform.transform_options == {"encoding": "latin-1", "delimiter": ";"}