fix: decode the features with the reponse charset

UnleashClient/api_client/_async.py

@@ -10,13 +10,13 @@
 from UnleashClient._retry import _send_with_retries_async
 from UnleashClient.api_client._shared import (
     FetchResult,
+    _decode_body,
     _emit_fetch_completed,
     _failure_status,
     _fetched_status,
     _header,
     _log_resp_info,
     _normalized_url,
-    _text,
 )
 from UnleashClient.config import PollingOptions, UnleashConfig
 from UnleashClient.constants import FEATURES_URL, METRICS_URL, REGISTER_URL
@@ -147,7 +147,7 @@ async def _fetch_features(
                     FetchStatus.NOT_CHANGED,
                 )
 
-            raw_state = _text(resp)
+            raw_state = _decode_body(resp)
             return (
                 FetchResult(raw_state, fetched_etag),
                 _fetched_status(raw_state),

UnleashClient/api_client/_shared.py

@@ -3,6 +3,7 @@
 classified and reported, and reading a response. Names no HTTP library.
 """
 
+import codecs
 import uuid
 from typing import NamedTuple, Optional
 
@@ -82,11 +83,20 @@ def _header(resp: Response, name: str) -> str:
     return ""
 
 
-def _text(resp: Response) -> str:
-    return resp.body.decode("utf-8", errors="replace")
+def _decode_body(resp: Response) -> str:
+    params = {}
+    for param in _header(resp, "content-type").split(";")[1:]:
+        name, _, value = param.partition("=")
+        params[name.strip().lower()] = value.strip().strip('"')
+    charset = params.get("charset", "utf-8")
+    try:
+        codecs.lookup(charset)
+    except LookupError:
+        charset = "utf-8"
+    return resp.body.decode(charset)
 
 
 def _log_resp_info(resp: Response) -> None:
     LOGGER.debug("HTTP status code: %s", resp.status)
     LOGGER.debug("HTTP headers: %s", resp.headers)
-    LOGGER.debug("HTTP content: %s", _text(resp))
+    LOGGER.debug("HTTP content: %s", resp.body.decode("utf-8", errors="replace"))

UnleashClient/api_client/_sync.py

@@ -6,13 +6,13 @@
 from UnleashClient._retry import _send_with_retries
 from UnleashClient.api_client._shared import (
     FetchResult,
+    _decode_body,
     _emit_fetch_completed,
     _failure_status,
     _fetched_status,
     _header,
     _log_resp_info,
     _normalized_url,
-    _text,
 )
 from UnleashClient.config import PollingOptions, UnleashConfig
 from UnleashClient.constants import FEATURES_URL, METRICS_URL, REGISTER_URL
@@ -125,7 +125,7 @@ def _fetch_features(
                     FetchStatus.NOT_CHANGED,
                 )
 
-            raw_state = _text(resp)
+            raw_state = _decode_body(resp)
             return FetchResult(raw_state, fetched_etag), _fetched_status(raw_state)
         except Exception as exc:
             LOGGER.exception(

tests/unit_tests/test_async_unleash_api_client.py

@@ -528,6 +528,29 @@ async def test_fetch_features_sends_its_request_through_the_transport():
     assert request.headers["If-None-Match"] == ETAG_VALUE
 
 
+STATE_WITH_ACCENT = '{"version": 1, "name": "caf\u00e9"}'
+
+
+@mark.asyncio
+async def test_a_state_that_does_not_decode_is_a_failed_fetch():
+    # Latin-1 bytes sent as UTF-8: "\xe9" alone is not valid UTF-8
+    transport = RecordingTransport(
+        Response(
+            status=200,
+            headers={"ETag": ETAG_VALUE, "Content-Type": "application/json"},
+            body=STATE_WITH_ACCENT.encode("iso-8859-1"),
+        )
+    )
+    events = RecordingEvents()
+
+    result = await api_client_over(transport, events).fetch_features(
+        options=PollingOptions()
+    )
+
+    assert result == (None, "", False)
+    assert fetch_outcomes(events) == [FetchStatus.INVALID_RESPONSE]
+
+
 @mark.asyncio
 async def test_register_posts_its_payload_through_the_transport():
     transport = RecordingTransport(Response(status=202, headers={}, body=b""))

tests/unit_tests/test_unleash_api_client.py

@@ -20,7 +20,7 @@
     URL,
 )
 from UnleashClient._headers import _HeaderFactory
-from UnleashClient.api_client._shared import _header
+from UnleashClient.api_client._shared import _decode_body, _header
 from UnleashClient.api_client._sync import _UnleashApiClient
 from UnleashClient.config import (
     IdentityOptions,
@@ -489,3 +489,57 @@ def test_a_payload_that_cannot_be_serialised_still_raises(call):
 )
 def test_header_matches_names_case_insensitively(headers, expected):
     assert _header(Response(status=200, headers=headers, body=b""), "etag") == expected
+
+
+@pytest.mark.parametrize(
+    "headers,expected",
+    (
+        ({"Content-Type": "application/json; charset=iso-8859-1"}, "iso8859-1"),
+        ({"content-type": 'application/json; charset="ISO-8859-1"'}, "iso8859-1"),
+        ({"Content-Type": "application/json; charset=utf-8"}, "utf-8"),
+        ({"Content-Type": "application/json"}, "utf-8"),
+        ({}, "utf-8"),
+        ({"Content-Type": "application/json; charset=no-such-charset"}, "utf-8"),
+        ({"Content-Type": "application/json; q=1; charset=latin-1"}, "iso8859-1"),
+        ({"Content-Type": "application/json; CHARSET=latin-1"}, "iso8859-1"),
+        ({"Content-Type": "application/json ; charset = latin-1 "}, "iso8859-1"),
+        ({"Content-Type": "application/json; charset="}, "utf-8"),
+    ),
+    ids=(
+        "declared",
+        "quoted-uppercase",
+        "utf-8",
+        "no-charset",
+        "no-header",
+        "unknown",
+        "after-another-parameter",
+        "uppercase-name",
+        "spaces",
+        "empty",
+    ),
+)
+def test_decode_body_uses_the_declared_charset_or_utf_8(headers, expected):
+    body = "caf\u00e9".encode(expected)
+
+    assert _decode_body(Response(status=200, headers=headers, body=body)) == (
+        "caf\u00e9"
+    )
+
+
+STATE_WITH_ACCENT = '{"version": 1, "name": "caf\u00e9"}'
+
+
+def test_a_state_that_does_not_decode_is_a_failed_fetch():
+    transport = RecordingTransport(
+        Response(
+            status=200,
+            headers={"ETag": ETAG_VALUE, "Content-Type": "application/json"},
+            body=STATE_WITH_ACCENT.encode("iso-8859-1"),
+        )
+    )
+    events = RecordingEvents()
+
+    result = api_client_over(transport, events).fetch_features(options=PollingOptions())
+
+    assert result == (None, "", False)
+    assert fetch_outcomes(events) == [FetchStatus.INVALID_RESPONSE]