diff --git a/UnleashClient/api_client/_async.py b/UnleashClient/api_client/_async.py index 0312463c..87d4a73c 100644 --- a/UnleashClient/api_client/_async.py +++ b/UnleashClient/api_client/_async.py @@ -10,13 +10,13 @@ from UnleashClient._retry import _send_with_retries_async from UnleashClient.api_client._shared import ( FetchResult, + _decode_body, _emit_fetch_completed, _failure_status, _fetched_status, _header, _log_resp_info, _normalized_url, - _text, ) from UnleashClient.config import PollingOptions, UnleashConfig from UnleashClient.constants import FEATURES_URL, METRICS_URL, REGISTER_URL @@ -147,7 +147,7 @@ async def _fetch_features( FetchStatus.NOT_CHANGED, ) - raw_state = _text(resp) + raw_state = _decode_body(resp) return ( FetchResult(raw_state, fetched_etag), _fetched_status(raw_state), diff --git a/UnleashClient/api_client/_shared.py b/UnleashClient/api_client/_shared.py index 35801cd0..cb29942d 100644 --- a/UnleashClient/api_client/_shared.py +++ b/UnleashClient/api_client/_shared.py @@ -3,6 +3,7 @@ classified and reported, and reading a response. Names no HTTP library. """ +import codecs import uuid from typing import NamedTuple, Optional @@ -82,11 +83,20 @@ def _header(resp: Response, name: str) -> str: return "" -def _text(resp: Response) -> str: - return resp.body.decode("utf-8", errors="replace") +def _decode_body(resp: Response) -> str: + params = {} + for param in _header(resp, "content-type").split(";")[1:]: + name, _, value = param.partition("=") + params[name.strip().lower()] = value.strip().strip('"') + charset = params.get("charset", "utf-8") + try: + codecs.lookup(charset) + except LookupError: + charset = "utf-8" + return resp.body.decode(charset) def _log_resp_info(resp: Response) -> None: LOGGER.debug("HTTP status code: %s", resp.status) LOGGER.debug("HTTP headers: %s", resp.headers) - LOGGER.debug("HTTP content: %s", _text(resp)) + LOGGER.debug("HTTP content: %s", resp.body.decode("utf-8", errors="replace")) diff --git a/UnleashClient/api_client/_sync.py b/UnleashClient/api_client/_sync.py index 0e7f93aa..241f66b2 100644 --- a/UnleashClient/api_client/_sync.py +++ b/UnleashClient/api_client/_sync.py @@ -6,13 +6,13 @@ from UnleashClient._retry import _send_with_retries from UnleashClient.api_client._shared import ( FetchResult, + _decode_body, _emit_fetch_completed, _failure_status, _fetched_status, _header, _log_resp_info, _normalized_url, - _text, ) from UnleashClient.config import PollingOptions, UnleashConfig from UnleashClient.constants import FEATURES_URL, METRICS_URL, REGISTER_URL @@ -125,7 +125,7 @@ def _fetch_features( FetchStatus.NOT_CHANGED, ) - raw_state = _text(resp) + raw_state = _decode_body(resp) return FetchResult(raw_state, fetched_etag), _fetched_status(raw_state) except Exception as exc: LOGGER.exception( diff --git a/tests/unit_tests/test_async_unleash_api_client.py b/tests/unit_tests/test_async_unleash_api_client.py index eafde511..8ce1826d 100644 --- a/tests/unit_tests/test_async_unleash_api_client.py +++ b/tests/unit_tests/test_async_unleash_api_client.py @@ -528,6 +528,29 @@ async def test_fetch_features_sends_its_request_through_the_transport(): assert request.headers["If-None-Match"] == ETAG_VALUE +STATE_WITH_ACCENT = '{"version": 1, "name": "caf\u00e9"}' + + +@mark.asyncio +async def test_a_state_that_does_not_decode_is_a_failed_fetch(): + # Latin-1 bytes sent as UTF-8: "\xe9" alone is not valid UTF-8 + transport = RecordingTransport( + Response( + status=200, + headers={"ETag": ETAG_VALUE, "Content-Type": "application/json"}, + body=STATE_WITH_ACCENT.encode("iso-8859-1"), + ) + ) + events = RecordingEvents() + + result = await api_client_over(transport, events).fetch_features( + options=PollingOptions() + ) + + assert result == (None, "", False) + assert fetch_outcomes(events) == [FetchStatus.INVALID_RESPONSE] + + @mark.asyncio async def test_register_posts_its_payload_through_the_transport(): transport = RecordingTransport(Response(status=202, headers={}, body=b"")) diff --git a/tests/unit_tests/test_unleash_api_client.py b/tests/unit_tests/test_unleash_api_client.py index ee9ffb1a..4c94681d 100644 --- a/tests/unit_tests/test_unleash_api_client.py +++ b/tests/unit_tests/test_unleash_api_client.py @@ -20,7 +20,7 @@ URL, ) from UnleashClient._headers import _HeaderFactory -from UnleashClient.api_client._shared import _header +from UnleashClient.api_client._shared import _decode_body, _header from UnleashClient.api_client._sync import _UnleashApiClient from UnleashClient.config import ( IdentityOptions, @@ -489,3 +489,57 @@ def test_a_payload_that_cannot_be_serialised_still_raises(call): ) def test_header_matches_names_case_insensitively(headers, expected): assert _header(Response(status=200, headers=headers, body=b""), "etag") == expected + + +@pytest.mark.parametrize( + "headers,expected", + ( + ({"Content-Type": "application/json; charset=iso-8859-1"}, "iso8859-1"), + ({"content-type": 'application/json; charset="ISO-8859-1"'}, "iso8859-1"), + ({"Content-Type": "application/json; charset=utf-8"}, "utf-8"), + ({"Content-Type": "application/json"}, "utf-8"), + ({}, "utf-8"), + ({"Content-Type": "application/json; charset=no-such-charset"}, "utf-8"), + ({"Content-Type": "application/json; q=1; charset=latin-1"}, "iso8859-1"), + ({"Content-Type": "application/json; CHARSET=latin-1"}, "iso8859-1"), + ({"Content-Type": "application/json ; charset = latin-1 "}, "iso8859-1"), + ({"Content-Type": "application/json; charset="}, "utf-8"), + ), + ids=( + "declared", + "quoted-uppercase", + "utf-8", + "no-charset", + "no-header", + "unknown", + "after-another-parameter", + "uppercase-name", + "spaces", + "empty", + ), +) +def test_decode_body_uses_the_declared_charset_or_utf_8(headers, expected): + body = "caf\u00e9".encode(expected) + + assert _decode_body(Response(status=200, headers=headers, body=body)) == ( + "caf\u00e9" + ) + + +STATE_WITH_ACCENT = '{"version": 1, "name": "caf\u00e9"}' + + +def test_a_state_that_does_not_decode_is_a_failed_fetch(): + transport = RecordingTransport( + Response( + status=200, + headers={"ETag": ETAG_VALUE, "Content-Type": "application/json"}, + body=STATE_WITH_ACCENT.encode("iso-8859-1"), + ) + ) + events = RecordingEvents() + + result = api_client_over(transport, events).fetch_features(options=PollingOptions()) + + assert result == (None, "", False) + assert fetch_outcomes(events) == [FetchStatus.INVALID_RESPONSE]