From 8abddc4b0907e80fc2b1f3c434cb6f23b8064708 Mon Sep 17 00:00:00 2001 From: Edward Nolan Date: Tue, 22 Sep 2026 11:44:22 +0000 Subject: [PATCH] Constrain code unit adaptors to same-width integral and byte sources Per SG16 feedback, replace the convertible_to constraint on as_charT with a rule that accepts a source type U if it is Char itself, or if it is std::byte or a non-bool integral type with sizeof(U) == sizeof(Char). The exposition-only functor now performs the cast with static_cast. This rejects narrowing and widening conversions, conversions between distinct charN_t types, floating-point and bool sources, enumerations other than std::byte, and class types, while accepting std::byte and same-width integer buffers. Add a design discussion section with rationale, alternatives, and a conversion table, and add unit tests covering the accept/reject matrix and the static_cast semantics for byte and signed sources. Co-Authored-By: Claude Fable 5.1 --- include/beman/utf_view/code_unit_view.hpp | 24 +++- papers/P2728.md | 70 +++++++-- tests/beman/utf_view/code_unit_view.test.cpp | 142 +++++++++++++++++++ 3 files changed, 224 insertions(+), 12 deletions(-) diff --git a/include/beman/utf_view/code_unit_view.hpp b/include/beman/utf_view/code_unit_view.hpp index fed83adb..46f76c99 100644 --- a/include/beman/utf_view/code_unit_view.hpp +++ b/include/beman/utf_view/code_unit_view.hpp @@ -20,6 +20,8 @@ import beman.utf_view; #include #if !BEMAN_UTF_VIEW_USE_MODULES() #include +#include +#include #include #include #endif @@ -33,9 +35,9 @@ namespace detail { /* PAPER */ template -struct exposition_only_implicit_cast_to { +struct exposition_only_cast_to { constexpr T operator()(auto x) const noexcept { - return x; + return static_cast(x); } }; @@ -45,11 +47,25 @@ struct exposition_only_implicit_cast_to { namespace detail { + // A source type U may be cast to the code unit type Char if it is Char + // itself, or if it is std::byte or a non-bool integral type of the same + // width as Char. This rules out narrowing, widening (e.g. accidentally + // treating UTF-8 code units as code points), floating-point sources, bool, + // enumerations other than std::byte, and class types such as proxy + // references. + template + concept code_unit_castable = + std::same_as || + ((std::same_as || + (std::is_integral_v && !std::same_as)) && + sizeof(U) == sizeof(Char)); + template struct as_code_unit_impl : std::ranges::range_adaptor_closure> { template - requires std::convertible_to, Char> && + requires code_unit_castable< + std::remove_cvref_t>, Char> && (!std::is_array_v>) constexpr auto operator()(R&& r) const { using T = std::remove_cvref_t; @@ -57,7 +73,7 @@ namespace detail { return std::ranges::empty_view{}; } else { return beman::transform_view::transform_view( - std::forward(r), exposition_only_implicit_cast_to{}); + std::forward(r), exposition_only_cast_to{}); } } }; diff --git a/papers/P2728.md b/papers/P2728.md index e9f8d246..b50b3386 100644 --- a/papers/P2728.md +++ b/papers/P2728.md @@ -300,7 +300,9 @@ for casting between character types: `as_char`, `as_wchar_t`, `as_char8_t`, `as_char16_t`, and `as_char32_t`. These are syntactic sugar for producing a `std::ranges::transform_view` with -an exposition-only transformation functor that performs the needed cast. +an exposition-only transformation functor that performs the needed cast. The +rule governing which source types may be cast is discussed in +[Code Unit Adaptor Conversion Rule](#code-unit-adaptor-conversion-rule). # Additional Examples @@ -1143,8 +1145,8 @@ Add the following subclause to [range.adaptors]{.sref}: ```cpp template -struct @*implicit-cast-to*@ { // @*exposition only*@/ - constexpr T operator()(auto x) const noexcept { return x; } +struct @*cast-to*@ { // @*exposition only*@ + constexpr T operator()(auto x) const noexcept { return static_cast(x); } }; ``` @@ -1152,15 +1154,18 @@ The names `as_char`, `as_wchar_t`, `as_char8_t`, `as_char16_t`, and `as_char32_t` denote range adaptor objects ([range.adaptor.object]). Let `as_charT` denote any one of `as_char`, `as_wchar_t`, `as_char8_t`, `as_char16_t`, and `as_char32_t`. Let `Char` be the corresponding character -type for `as_charT`, let `E` be an expression and let `T` be -`remove_cvref_t`. If `ranges::range_reference_t` does not -model `convertible_to`, or if `T` is an array, `as_charT(E)` is -ill-formed. Otherwise, the expression `as_charT(E)` is expression-equivalent to: +type for `as_charT`, let `E` be an expression, let `T` be +`remove_cvref_t`, and let `U` be +`remove_cvref_t>`. `as_charT(E)` is ill-formed if +`T` is an array, or if `U` is not `Char` and `U` is not a type `X` such that +`sizeof(X) == sizeof(Char)` and either `X` is `byte` or `X` is an integral type +other than `bool`. Otherwise, the expression `as_charT(E)` is +expression-equivalent to: - If `T` is a specialization of `empty_view` ([range.empty.view]), then `empty_view{}`. -- Otherwise, `ranges::transform_view(std::views::all(E), @*implicit-cast-to*@{})`. +- Otherwise, `ranges::transform_view(std::views::all(E), @*cast-to*@{})`. \[Example 1: ```c++ @@ -1365,6 +1370,53 @@ But we voted in favor of providing it for forward ranges: > > A: I think it's probably useful, but I'm not certain, so let's wait. +## Code Unit Adaptor Conversion Rule + +For the `as_charT` casting views, the rule we use to determine whether the element type +`U` of a view's source range makes it eligible for conversion to the target type `Char` +is: + +- `U` is `Char`, or +- `U` is `std::byte` or a non-`bool` integral type, and `sizeof(U) == sizeof(Char)`. + +This accomplishes the following design goals: + +- Rules out casts which are either illogical (floats, bools) or which are likely to be + mistakes (class types, enums other than `std::byte`) +- Rules out narrowing and widening conversions by the width requirement, including between + `charN_t` types +- Allows conversions from same-width integral types and `std::byte` to `charN_t` types, + such that `int` can be used to store `char32_t` (as is done in ICU) and `unsigned short` + can be used for `char16_t` (assuming those types are appropriately sized), and `wchar_t` + can either be cast to `char16_t` or `char32_t` depending on its width. + +This is consistent with [@P3695R3]'s deprecation of most implicit conversions between +`charN_t` types. Despite the fact that that paper only drops implicit casts, and the +`as_charT` views can be likened to an explicit cast, I think it's appropriate here to only +bless casts that we think usually make sense, and make the user work a bit harder to spell +out a `transform_view` if, for example, they want to cast between `char16_t` and +`char32_t` (whose implicit conversions [@P3695R3] preserved, but which I don't think +merits an explicit carveout here since there's no justification based on backwards +compatibility). + +| Source | `as_char` | `as_char8_t` | `as_char16_t` | `as_char32_t` | `as_wchar_t` | +|---|---|---|---|---|---| +| `char` | yes | yes | no | no | no | +| `signed char`, `unsigned char` | yes | yes | no | no | no | +| `std::byte` | yes | yes | no | no | no | +| `char8_t` | yes | yes | no | no | no | +| `char16_t` | no | no | yes | no | same width | +| `char32_t` | no | no | no | yes | same width | +| `wchar_t` | no | no | same width | same width | yes | +| `short`, `unsigned short` | no | no | yes | no | same width | +| `int`, `unsigned int` | no | no | no | same width | same width | +| `long`, `unsigned long` | no | no | no | same width | same width | +| `long long`, `unsigned long long` | no | no | no | no | no | +| `bool` | no | no | no | no | no | +| floating-point types | no | no | no | no | no | +| enumerations other than `std::byte` | no | no | no | no | no | +| class types | no | no | no | no | no | + # Changelog ## Changes since R14 @@ -1374,6 +1426,8 @@ But we voted in favor of providing it for forward ranges: - Implement `it.base_code_units()` for forward ranges per SG9 feedback - Add random access support for UTF-32-to-UTF-32 self-transcode per SG9 feedback +- Constrain the code unit adaptors to same-width integral and `std::byte` + sources per SG16 feedback, and add a design discussion of the rule ## Changes since R13 diff --git a/tests/beman/utf_view/code_unit_view.test.cpp b/tests/beman/utf_view/code_unit_view.test.cpp index 6c601c62..0c094209 100644 --- a/tests/beman/utf_view/code_unit_view.test.cpp +++ b/tests/beman/utf_view/code_unit_view.test.cpp @@ -13,9 +13,14 @@ #if BEMAN_UTF_VIEW_USE_MODULES() import std; #else +#include +#include #include #include +#include +#include #include +#include #endif namespace beman::utf_view::tests { @@ -101,6 +106,87 @@ static_assert( test_random_access_iterator, test_random_access_iterator>>() | as_char8_t)>>); +// Conversion rule: a source type is accepted if it is the target type, or if it +// is std::byte or a non-bool integral type of the same width as the target. +template +constexpr bool adaptable = requires { std::declval() | Adaptor; }; + +enum unscoped_byte_enum : unsigned char { unscoped_byte_enum_value }; +enum class scoped_byte_enum : unsigned char { value }; +struct convertible_to_char8_t { + constexpr operator char8_t() const { return u8'x'; } +}; + +// as_char8_t +static_assert(adaptable); +static_assert(adaptable); +static_assert(adaptable, as_char8_t>); +static_assert(adaptable, as_char8_t>); +static_assert(adaptable, as_char8_t>); +static_assert(adaptable, as_char8_t>); +static_assert(adaptable, as_char8_t>); +static_assert(!adaptable); +static_assert(!adaptable); +static_assert(!adaptable); +static_assert(!adaptable, as_char8_t>); +static_assert(!adaptable, as_char8_t>); +static_assert(!adaptable, as_char8_t>); +static_assert(!adaptable&, as_char8_t>); +static_assert(!adaptable, as_char8_t>); +static_assert(!adaptable, as_char8_t>); +static_assert(!adaptable, as_char8_t>); +static_assert(!adaptable, as_char8_t>); +static_assert(!adaptable, as_char8_t>); +static_assert(!adaptable); + +// as_char +static_assert(adaptable); +static_assert(adaptable); +static_assert(adaptable, as_char>); +static_assert(adaptable, as_char>); +static_assert(!adaptable); +static_assert(!adaptable); + +// as_char16_t +static_assert(adaptable); +static_assert(adaptable, as_char16_t>); +static_assert(adaptable, as_char16_t>); +static_assert(adaptable, as_char16_t>); +static_assert(adaptable == + (sizeof(wchar_t) == sizeof(char16_t))); +static_assert(!adaptable); +static_assert(!adaptable); +static_assert(!adaptable); +static_assert(!adaptable, as_char16_t>); + +// as_char32_t +static_assert(adaptable); +static_assert(adaptable, as_char32_t>); +static_assert(adaptable, as_char32_t>); +static_assert(adaptable, as_char32_t> == + (sizeof(int) == sizeof(char32_t))); +static_assert(adaptable, as_char32_t> == + (sizeof(unsigned) == sizeof(char32_t))); +static_assert(adaptable, as_char32_t> == + (sizeof(long) == sizeof(char32_t))); +static_assert(adaptable == + (sizeof(wchar_t) == sizeof(char32_t))); +static_assert(!adaptable); +static_assert(!adaptable); +static_assert(!adaptable, as_char32_t>); +static_assert(!adaptable, as_char32_t>); + +// as_wchar_t +static_assert(adaptable); +static_assert(adaptable == + (sizeof(wchar_t) == sizeof(char16_t))); +static_assert(adaptable == + (sizeof(wchar_t) == sizeof(char32_t))); +static_assert(adaptable, as_wchar_t> == + (sizeof(wchar_t) == sizeof(unsigned short))); +static_assert(!adaptable); +static_assert(!adaptable); + constexpr bool smoke_test() { std::string_view foo{"foo"}; auto bar{foo | as_char8_t}; @@ -120,6 +206,50 @@ constexpr bool special_case_test() { return true; } +constexpr bool same_type_test() { + std::u8string_view foo{u8"foo"}; + auto bar{foo | as_char8_t}; + if ((bar | std::ranges::to()) != foo) { + return false; + } + return true; +} + +constexpr bool byte_test() { + std::array const euro{std::byte{0xE2}, std::byte{0x82}, + std::byte{0xAC}}; + auto const as_utf8{euro | as_char8_t | std::ranges::to()}; + if (as_utf8 != std::u8string_view{u8"\u20AC"}) { + return false; + } + return true; +} + +// The cast is as if by static_cast, so negative values in a signed source +// preserve their bit pattern. +constexpr bool signed_source_test() { + std::array const euro{static_cast(-30), + static_cast(-126), + static_cast(-84)}; + auto const as_utf8{euro | as_char8_t | std::ranges::to()}; + if (as_utf8 != std::u8string_view{u8"\u20AC"}) { + return false; + } + return true; +} + +constexpr bool int_source_test() { + if constexpr (sizeof(int) == sizeof(char32_t)) { + std::array const code_points{0x1F574, 0xFFFD}; + auto const as_utf32{code_points | as_char32_t | + std::ranges::to()}; + if (as_utf32 != std::u32string_view{U"\U0001F574\uFFFD"}) { + return false; + } + } + return true; +} + constexpr bool value_category_test() { std::string foo{"foo"}; (void)(foo | as_char8_t); @@ -136,6 +266,18 @@ CONSTEXPR_UNLESS_MSVC bool code_unit_view_test() { if (!special_case_test()) { return false; } + if (!same_type_test()) { + return false; + } + if (!byte_test()) { + return false; + } + if (!signed_source_test()) { + return false; + } + if (!int_source_test()) { + return false; + } if (!value_category_test()) { return false; }