diff --git a/include/beman/utf_view/to_utf_view.hpp b/include/beman/utf_view/to_utf_view.hpp index a614094..156541a 100644 --- a/include/beman/utf_view/to_utf_view.hpp +++ b/include/beman/utf_view/to_utf_view.hpp @@ -362,7 +362,15 @@ struct to_utf_view::exposition_only_iterator : detail::iter_catego return std::move(exposition_only_current_); } - /* PAPER: constexpr value_type operator*() const; */ + constexpr auto base_code_units() const + requires std::ranges::forward_range + { + return std::ranges::subrange( + exposition_only_current_, + std::ranges::next(exposition_only_current_, exposition_only_to_increment_)); + } + + /* PAPER: constexpr value_type operator*() const; */ /* !PAPER */ constexpr value_type operator*() const { if constexpr (E == to_utf_view_kind::expected) { @@ -490,8 +498,11 @@ struct to_utf_view::exposition_only_iterator : detail::iter_catego } if (exposition_only_current_ != exposition_only_end()) { exposition_only_read(); - } else if constexpr (!std::ranges::forward_range) { - exposition_only_buf_index_ = -1; + } else { + exposition_only_to_increment_ = 0; + if constexpr (!std::ranges::forward_range) { + exposition_only_buf_index_ = -1; + } } } } diff --git a/papers/P2728.md b/papers/P2728.md index fba6645..6a0720c 100644 --- a/papers/P2728.md +++ b/papers/P2728.md @@ -904,8 +904,11 @@ private: } if (@*current_*@ != @*end*@()) { @*read*@(); - } else if constexpr (!forward_range<@*Base*@>) { - @*buf_index_*@ = -1; + } else { + @*to_increment_*@ = 0; + if constexpr (!forward_range<@*Base*@>) { + @*buf_index_*@ = -1; + } } } } @@ -1257,385 +1260,44 @@ Strong consensus against. ## `.base_code_units()` -### Proposal - -I received feedback that it could be useful to provide a `.base_code_units()` member -function on the transcoding iterator which would give out a range of iterators from the -underlying range delimiting the code units that make up the current code point. - -Since we can't give out iterators to the underlying range if it's a (non-forward) input -range, it's also been suggested that in this case, `.base_code_units()` would still be -available, but would give out iterators to a special cache that's stored in the iterator. +Based on feedback that this was a desired feature, and implementation +experience in similar APIs provided by ICU, this paper introduces a new member +function for the transcoding iterator: `.base_code_units()`. No existing +standard library view provides an equivalent API, so this is novel. It provides +a range of iterators from the underlying range delimiting the code units that +make up the current code point. + +This member function is only provided for forward ranges. Previously, there had +been discussion about providing this member function for input ranges as +well. However, there were no good options for making this work. We could give +out iterators to a special cache that's stored in the iterator, but then the +result of `.base_code_units()` would be invalidated after incrementing the +iterator. We could also change the return type of `.base_code_units()` to +something like `std::inplace_vector`, but then separate invocations +of `it.base_code_units().begin()` and `it.base_code_units().end()` on the same +iterator would point to different objects. + +Based on these concerns, SG9 voted against providing `.base_code_units()` for +input ranges in Brno: + +> **Poll: We want base_code_units() for input ranges (requires a cache and has potential lifetime issues).** +> +> No objection to unanimous dissent. +> Attendance: 7 -To quote from a reflector email discussing this suggestion: +But we voted in favor of providing it for forward ranges: -> I think it would be useful to differentiate access to the (complete) underlying range vs -> access to the input code unit sequence for the current character. Obviously, access to the -> complete underlying range isn't possible for input iterators, but access to the current -> input code unit sequence is (with the caching approach described above is). The iterators -> could expose this interface: +> **Poll: We want base_code_units() for forward ranges (pure syntactic sugar for `subrange(it.base(), next(it).base())`)** > -> ```c++ -> // Forward+ iterators only; returns an iterator into the underlying range. -> constexpr const iterator_t& base() const & noexcept requires forward_range { ... } -> constexpr iterator_t base() && requires forward_range { ... } +> SF| F | N | A | SA +> ---|---|---|---|--- +> 1| 4 | 0 | 1 | 0 > -> // Input+ iterators; returns a subrange containing the input code units for the current character. -> // References the input code unit sequence cache for input iterators. -> // References the underlying range otherwise. -> constexpr subrange<...> base_code_units() const noexcept { ... } -> ``` +> Attendance: 7 +> Author: F +> Strong consensus. > -> Unlike `base()`, `base_code_units()` would not necessarily contain iterators for the -> underlying range (e.g., in the case of a caching input iterator). - -Note that the choice to provide `.base_code_units()` for input ranges affects ABI since -the size of the transcoding iterator depends on whether it contains the cache. - -### Precedent - -[@P0244R2] provides transcoding iterators with a `.base_range()` member function that -provide this range, although its input iterator functionality is implemented using special -caching iterators that have shared ownership of a cache, instead of by storing the cached -range in the iterator itself. - -ICU provides multiple analogous APIs. The most directly comparable one is the -`.stringView()` [member -function](https://github.com/unicode-org/icu/blob/649262a75ecddb15a0e58d71f637a8a32eaabd43/icu4c/source/common/unicode/utfiterator.h#L416) -on the `UnsafeCodeUnits` transcoding iterator, which provides a `std::basic_string_view` -containing the underlying code units for the current code point. `UnsafeCodeUnits` also -provides `.begin()` and `.end()` member functions which give out the same range. Unlike -the proposed `.base_code_units()` member function, neither of these APIs provide support -for input iterators; `.stringView()` is only enabled when the base range is contiguous, -and `.begin()` and `.end()` are only enabled if it's a forward range. - -### Lifetime Issues - -Here's an example of a function where the use of `.base_code_units()` subtly introduces UB -when the function is passed an input range. - -This is a run-length-encoder that prints a count of the number of consecutive times it's -seen a code point, followed by the code units making up that code point: - -```cpp -void print_runs(std::ranges::range auto text) { - auto utf_view = text | std::views::to_utf32; - auto it = utf_view.begin(); - while (it != utf_view.end()) { - auto units = it.base_code_units(); - char32_t code_point = *it; - int count = 1; - ++it; - while (it != utf_view.end() && *it == code_point) { - ++count; - ++it; - } - std::print( - "{}x{::#x} ", count, - units | std::views::transform([](char8_t c) { return (std::uint8_t)c; } )); - } - std::println(""); -} -``` - -When invoked with `u8"ⒶⒶⒶⒷⒸ"sv`, it prints: - -``` -3x[0xe2, 0x92, 0xb6] 1x[0xe2, 0x92, 0xb7] 1x[0xe2, 0x92, 0xb8] -``` - -When invoked with `u8"ⒶⒶⒶⒷⒸ"sv | std::views::as_input`, it invokes library undefined -behavior and prints corrupted output. Worse, the UB here isn't caught by AddressSanitizer -or UndefinedBehaviorSanitizer because the invalidated `auto units` range points into the -same, valid, transcoding iterator, whose cache simply contains the values for the -subsequent code point, so the corrupted output is not automatically diagnosable. - -With ICU's APIs, on the other hand, this would fail to compile, because ICU only provides -them for forward ranges. - -I think this footgun would show up frequently. - -In response, it was suggested that the above lifetime issue could be addressed by changing -the return type of `.base_code_units()` to something like `std::inplace_vector`. - -That creates a different lifetime problem. Consider this example. The Unicode Tags block -is intended for use in flag emojis but has been used for LLM prompt injections. Say a user -writes the following function, which divides the stream of characters into Tags and -non-Tags, and also imagine that they have a custom sink type that accepts iterator pairs -rather than ranges: - -```cpp -constexpr bool is_tag(char32_t c) { return (c & ~0x7F) == 0xE0000; } - -void partition_tags(std::ranges::range auto text, sink non_tags, sink tags) { - auto utf_view = text | std::views::to_utf32; - for (auto it = utf_view.begin(); it != utf_view.end(); ++it) { - (is_tag(*it) ? tags : non_tags).consume( - it.base_code_units().begin(), it.base_code_units().end()); - } -} -``` - -Again, this works perfectly well when `partition_tags` is passed a forward range, but then -when it's passed an input range, because each call to `.base_code_units()` returns a -separate temporary `std::inplace_vector`, `it.base_code_units().begin()` and -`it.base_code_units().end()` now point to different objects, so the function invokes UB. - -### Survey of Range Adaptors that Downgrade to Input - -Some range adaptors downgrade forward ranges into input ranges: these are, to my -understanding, `views::as_input`, `views::cache_latest`, `views::join`, and -`views::join_with`. - -`[range.as.input.overview]` states, "This is useful to avoid overhead that can be -necessary to provide support for the operations needed for greater iterator strength." -This use case is potentially relevant for transcoding views, since the size of the -iterator may be greater with a stronger iterator category. For example, bidirectional -transcoding iterators need to store the begin iterator from the underlying range to avoid -overrunning the beginning when transcoding backwards, but forward iterators don't need it. - -But implementing `.base_code_units()` for input views would actually cause -`views::as_input` to *increase* the transcoding iterator's overhead relative to its -forward-iterator implementation, because the iterator would need to contain an additional -code unit cache. - -`views::as_input` was introduced by [@P3725R3], "Filter View Extensions for Safer Use," -and, rather than avoiding overhead, its main motivation was composition with -`std::views::filter` in order to avoid pitfalls related to mutating through a filter. - -This is potentially relevant to transcoding, in that someone might write a filter-view -pipeline on characters. Say a user wants to print the UTF-8 code units for all the -non-ASCII code points in a range. That would look like this: - -```cpp -void print_nonascii_code_points_and_code_units(std::ranges::range auto text) { - auto print_code_point{ - [](char32_t code_point, auto code_unit_range) { - std::println( - "{:#x} = {::#x}", static_cast(code_point), - code_unit_range | std::views::transform([](char8_t c) { return (std::uint8_t)c; })); - }}; - auto code_points = text - | std::views::filter([](char8_t c) { return c >= 0x80; }) - | std::views::to_utf32; - for (auto it = code_points.begin(); it != code_points.end(); ++it) { - print_code_point(*it, it.base_code_units()); - } -} -``` - -A user following the [@P3725R3] guidance might insert a `views::as_input` adaptor into the -pipeline before `std::views::filter`, which would continue to compile and work if we -provided `.base_code_units()` for input ranges, but which would cause -`print_nonascii_code_points_and_code_units` to fail to compile if we didn't. - -But `views::as_input` isn't strictly necessary here. And we already need to teach users -that inserting `views::as_input` before `std::views::filter` will, in rare cases, cause -some uses of `.base()` to fail to compile. To demonstrate why this isn't a novelty, -consider the following example: - -```cpp -struct Task { int priority; }; - -bool submit_batch(std::ranges::range auto batch); - -// Submit the high-priority tasks in batches; on a transient failure, hand the -// remaining high-priority tasks to the retry queue. -void submit_high_priority_tasks(std::vector& tasks) { - auto high = tasks | std::views::filter([](Task const& t) { return t.priority > 100; }); - auto batches = high | std::views::chunk(BATCH_SIZE); - for (auto it = batches.begin(); it != batches.end(); ++it) { - if (!submit_batch(*it)) { - requeue(std::ranges::subrange(it.base(), high.end())); - return; - } - } -} -``` - -This works as written, but if `views::as_input` is inserted in front of `views::filter`, -the call to `it.base()` fails to compile because `std::ranges::chunk_view`'s iterator -doesn't provide `.base()` for input views. But `views::as_input` is unnecessary here as -well. - -Furthermore, it's worth noting that the list of plausible reasons to apply a filter_view -on code *units* as opposed to code *points* is extremely short; ordinarily, doing so risks -corrupting the output. - -Moving on to `views::cache_latest`: that one is an adaptor with a niche use case and no -special relevance to transcoding. - -`views::join` is directly relevant, since it's common for users to want to reassemble a -text string that had previously been broken up into separate parts before transcoding -it. `views::join_with` is also potentially relevant, since users may want to transcode -text after having used `views::join_with` to add separators to it. - -It's important to note that in the common case, `views::join` and `views::join_with` do -not downgrade from forward to input. They only do so if the range of ranges it's given is -a range of *prvalue* ranges. - -For example, the `views::join` adaptor in the following example does not downgrade: - -```cpp -void print_errors(std::ranges::range auto packets) { - auto print_code_units = - [](std::ranges::range auto code_unit) { - std::println("{::#x}", - code_unit - | std::views::transform([](char8_t c) { return (std::uint8_t)c; })); - }; - auto utf_view = packets - | std::views::join - | std::views::to_utf32_or_error; - for (auto it = utf_view.begin(); it != utf_view.end(); ++it) { - if (!(*it).has_value()) { - print_code_units(it.base_code_units()); - } - } -} -``` - -But, if the packets need to be decrypted before transcoding, and the user alters the -pipeline like so: - -```diff -+ std::u8string decrypt(std::u8string_view packet) { -+ return packet -+ | std::views::transform( -+ [](char8_t c) { -+ return static_cast(c ^ 0x55); -+ }) -+ | std::ranges::to(); -+ } - -void print_errors(std::ranges::range auto packets) { - auto print_code_units = - [](std::ranges::range auto code_unit) { - std::println("{::#x}", - code_unit - | std::views::transform([](char8_t c) { return (std::uint8_t)c; })); - }; - auto utf_view = packets -+ | std::views::transform(decrypt) - | std::views::join - | std::views::to_utf32_or_error; - for (auto it = utf_view.begin(); it != utf_view.end(); ++it) { - if (!(*it).has_value()) { - print_code_units(it.base_code_units()); - } - } -} -``` - -Then it downgrades. - -### Alternatives to `.base_code_units()` for Users - -#### Forward Ranges - -For forward ranges, `it.base_code_units()` is equivalent to -`std::ranges::subrange(it.base(), std::ranges::next(it).base())`. - -The expression above raised concerns about the fact that its use in a loop would mean -performing two `operator++` operations on every loop iteration, but that can be mitigated -by simply caching the previous iterator while iterating forwards: - -```cpp -auto prev_base = it.base(); -++it; -auto code_units = std::ranges::subrange(prev_base, it.base()); -``` - -#### Input Ranges - -For input ranges, since the transcoding view doesn't provide `.base()`, the workaround -involves making a copy of the input range in order to get a forward range. - -##### Copying the entire range - -If it's viable to copy the entire range, you can simply insert a -`std::ranges::to()` into the range pipeline. - -```cpp -void print_utf8_code_points_and_code_units(std::ranges::range auto text) { - auto print_code_point{ - [](char32_t code_point, auto code_unit_range) { - std::println( - "{:#x} = {::#x}", static_cast(code_point), - code_unit_range | std::views::transform([](char8_t c) { return (std::uint8_t)c; })); - }}; - auto code_points = text - | std::ranges::to() - | std::views::to_utf32; - for (auto it = code_points.begin(); it != code_points.end(); ++it) { - print_code_point(*it, std::ranges::subrange(it.base(), std::ranges::next(it).base())); - } -} -``` - -When invoked with `u8"AΩ€😀b"sv | std::views::as_input`, this prints: - -``` -0x41 = [0x41] -0x3a9 = [0xce, 0xa9] -0x20ac = [0xe2, 0x82, 0xac] -0x1f600 = [0xf0, 0x9f, 0x98, 0x80] -0x62 = [0x62] -``` - -##### Avoiding copying - -Although it requires rolling your own segmentation, it is possible to iterate over an -input view's code unit subsequences with additional refactoring: - -```cpp -constexpr bool is_utf8_continuation(char8_t c) { return (c & 0xC0) == 0x80; } - -void print_utf16_and_utf8_code_units_per_code_point(std::ranges::range auto text) { - auto print_code_point{ - [](auto u16_view, auto u8_view) { - std::println( - "{::#x} = {::#x}", - u16_view | std::views::transform([](char16_t c) { return (std::uint16_t)c; }), - u8_view | std::views::transform([](char8_t c) { return (std::uint8_t)c; })); - }}; - auto it = text.begin(); - std::u8string code_point; - while (it != text.end()) { - code_point.clear(); - code_point.push_back(*it); - ++it; - it = std::ranges::find_if(std::move(it), text.end(), [&](char8_t c) { - if (!is_utf8_continuation(c)) return true; - code_point.push_back(c); - return false; - }); - print_code_point(code_point | std::views::to_utf16, code_point); - } -} -``` - -When invoked with `u8"AΩ€😀b"sv | std::views::as_input`, this prints: - -``` -[0x41] = [0x41] -[0x3a9] = [0xce, 0xa9] -[0x20ac] = [0xe2, 0x82, 0xac] -[0xd83d, 0xde00] = [0xf0, 0x9f, 0x98, 0x80] -[0x62] = [0x62] -``` - -Clearly, this isn't an ideal user experience, but it only applies to users who have an -input range that is too large to copy, and also need to access the underlying code unit -sequence for a code point. In my opinion, preserving the ergonomics of that use case is -not worth the tradeoff of introducing the safety footgun demonstrated by the RLE example -above. - -### Implementation - -An experimental implementation of `.base_code_units()` is available on the -`enolan_basecodeunits2` branch of `beman.utf_view`. +> A: I think it's probably useful, but I'm not certain, so let's wait. # Changelog @@ -1643,6 +1305,7 @@ An experimental implementation of `.base_code_units()` is available on the - Revert SIMD support and add design discussion of why it was rejected - Rename `to_utf_view_error_kind` to `to_utf_view_kind` per SG9 discussion +- Implement `it.base_code_units()` for forward ranges per SG9 feedback ## Changes since R13 diff --git a/tests/beman/utf_view/to_utf_view.test.cpp b/tests/beman/utf_view/to_utf_view.test.cpp index c49e100..97052fb 100644 --- a/tests/beman/utf_view/to_utf_view.test.cpp +++ b/tests/beman/utf_view/to_utf_view.test.cpp @@ -1457,6 +1457,282 @@ constexpr bool input_range_equality_test() { return true; } +constexpr bool base_code_units_test() { + auto const subrange_equal{ + [](Subrange const& s1, Subrange const& s2) { + return s1.begin() == s2.begin() && s1.end() == s2.end(); + }}; + { + std::initializer_list const arr{ + static_cast('\x51'), static_cast('\xCF'), + static_cast('\x95'), static_cast('\xE5'), + static_cast('\xAD'), static_cast('\xA6'), + static_cast('\xF0'), static_cast('\xA1'), + static_cast('\xAA'), static_cast('\x87')}; + { + test_forward_iterator const forward_it(arr); + test_forward_iterator const second_code_point(std::ranges::next(forward_it, 1)); + test_forward_iterator const third_code_point(std::ranges::next(second_code_point, 2)); + test_forward_iterator const fourth_code_point(std::ranges::next(third_code_point, 3)); + test_forward_iterator const end(std::ranges::next(fourth_code_point, 4)); + auto u32v{std::ranges::subrange{forward_it, std::default_sentinel} | to_utf32}; + auto it{u32v.begin()}; + if (!subrange_equal( + it.base_code_units(), std::ranges::subrange(forward_it, second_code_point))) { + return false; + } + ++it; + if (!subrange_equal( + it.base_code_units(), std::ranges::subrange(second_code_point, third_code_point))) { + return false; + } + ++it; + if (!subrange_equal( + it.base_code_units(), std::ranges::subrange(third_code_point, fourth_code_point))) { + return false; + } + ++it; + if (!subrange_equal( + it.base_code_units(), std::ranges::subrange(fourth_code_point, end))) { + return false; + } + ++it; + if (!subrange_equal( + it.base_code_units(), std::ranges::subrange(end, end))) { + return false; + } + } + { + test_bidi_iterator const bidi_it(arr); + test_bidi_iterator const second_code_point(std::ranges::next(bidi_it, 1)); + test_bidi_iterator const third_code_point(std::ranges::next(second_code_point, 2)); + test_bidi_iterator const fourth_code_point(std::ranges::next(third_code_point, 3)); + test_bidi_iterator const end(std::ranges::next(fourth_code_point, 4)); + auto u32v{std::ranges::subrange{bidi_it, std::default_sentinel} | to_utf32}; + auto it{u32v.begin()}; + std::ranges::advance(it, 4); + if (!subrange_equal( + it.base_code_units(), std::ranges::subrange(end, end))) { + return false; + } + --it; + if (!subrange_equal( + it.base_code_units(), std::ranges::subrange(fourth_code_point, end))) { + return false; + } + --it; + if (!subrange_equal( + it.base_code_units(), std::ranges::subrange(third_code_point, fourth_code_point))) { + return false; + } + --it; + if (!subrange_equal( + it.base_code_units(), std::ranges::subrange(second_code_point, third_code_point))) { + return false; + } + --it; + if (!subrange_equal( + it.base_code_units(), std::ranges::subrange(bidi_it, second_code_point))) { + return false; + } + } + } + { + std::initializer_list arr{U'\u0051', U'\u03D5', U'\u5B66', U'\x00021A87'}; + { + test_forward_iterator const forward_it(arr); + test_forward_iterator const second_code_point(std::ranges::next(forward_it)); + test_forward_iterator const third_code_point(std::ranges::next(second_code_point)); + test_forward_iterator const fourth_code_point(std::ranges::next(third_code_point)); + test_forward_iterator const end(std::ranges::next(fourth_code_point)); + auto u32v{std::ranges::subrange{forward_it, std::default_sentinel} | to_utf8}; + auto it{u32v.begin()}; + if (!subrange_equal( + it.base_code_units(), std::ranges::subrange(forward_it, second_code_point))) { + return false; + } + ++it; + if (!subrange_equal( + it.base_code_units(), std::ranges::subrange(second_code_point, third_code_point))) { + return false; + } + ++it; + if (!subrange_equal( + it.base_code_units(), std::ranges::subrange(second_code_point, third_code_point))) { + return false; + } + ++it; + if (!subrange_equal( + it.base_code_units(), std::ranges::subrange(third_code_point, fourth_code_point))) { + return false; + } + ++it; + if (!subrange_equal( + it.base_code_units(), std::ranges::subrange(third_code_point, fourth_code_point))) { + return false; + } + ++it; + if (!subrange_equal( + it.base_code_units(), std::ranges::subrange(third_code_point, fourth_code_point))) { + return false; + } + ++it; + if (!subrange_equal( + it.base_code_units(), std::ranges::subrange(fourth_code_point, end))) { + return false; + } + ++it; + if (!subrange_equal( + it.base_code_units(), std::ranges::subrange(fourth_code_point, end))) { + return false; + } + ++it; + if (!subrange_equal( + it.base_code_units(), std::ranges::subrange(fourth_code_point, end))) { + return false; + } + ++it; + if (!subrange_equal( + it.base_code_units(), std::ranges::subrange(fourth_code_point, end))) { + return false; + } + ++it; + if (!subrange_equal( + it.base_code_units(), std::ranges::subrange(end, end))) { + return false; + } + } + { + test_bidi_iterator const bidi_it(arr); + test_bidi_iterator const second_code_point(std::ranges::next(bidi_it)); + test_bidi_iterator const third_code_point(std::ranges::next(second_code_point)); + test_bidi_iterator const fourth_code_point(std::ranges::next(third_code_point)); + test_bidi_iterator const end(std::ranges::next(fourth_code_point)); + auto u32v{std::ranges::subrange{bidi_it, std::default_sentinel} | to_utf8}; + auto it{u32v.begin()}; + std::ranges::advance(it, 10); + if (!subrange_equal( + it.base_code_units(), std::ranges::subrange(end, end))) { + return false; + } + --it; + if (!subrange_equal( + it.base_code_units(), std::ranges::subrange(fourth_code_point, end))) { + return false; + } + --it; + if (!subrange_equal( + it.base_code_units(), std::ranges::subrange(fourth_code_point, end))) { + return false; + } + --it; + if (!subrange_equal( + it.base_code_units(), std::ranges::subrange(fourth_code_point, end))) { + return false; + } + --it; + if (!subrange_equal( + it.base_code_units(), std::ranges::subrange(fourth_code_point, end))) { + return false; + } + --it; + if (!subrange_equal( + it.base_code_units(), std::ranges::subrange(third_code_point, fourth_code_point))) { + return false; + } + --it; + if (!subrange_equal( + it.base_code_units(), std::ranges::subrange(third_code_point, fourth_code_point))) { + return false; + } + --it; + if (!subrange_equal( + it.base_code_units(), std::ranges::subrange(third_code_point, fourth_code_point))) { + return false; + } + --it; + if (!subrange_equal( + it.base_code_units(), std::ranges::subrange(second_code_point, third_code_point))) { + return false; + } + --it; + if (!subrange_equal( + it.base_code_units(), std::ranges::subrange(second_code_point, third_code_point))) { + return false; + } + --it; + if (!subrange_equal( + it.base_code_units(), std::ranges::subrange(bidi_it, second_code_point))) { + return false; + } + } + } + { + // invalid UTF + std::initializer_list const arr{ + static_cast('\xf0'), static_cast('\x9f'), + static_cast('\x99'), static_cast('\xff'), + static_cast('\xF0'), static_cast('\xA1'), + static_cast('\xAA'), static_cast('\x87')}; + { + test_forward_iterator const forward_it(arr); + test_forward_iterator const second_code_point(std::ranges::next(forward_it, 3)); + test_forward_iterator const third_code_point(std::ranges::next(second_code_point, 1)); + test_forward_iterator const end(std::ranges::next(third_code_point, 4)); + auto u32v{std::ranges::subrange{forward_it, std::default_sentinel} | to_utf32}; + auto it{u32v.begin()}; + if (!subrange_equal( + it.base_code_units(), std::ranges::subrange(forward_it, second_code_point))) { + return false; + } + ++it; + if (!subrange_equal( + it.base_code_units(), std::ranges::subrange(second_code_point, third_code_point))) { + return false; + } + ++it; + if (!subrange_equal( + it.base_code_units(), std::ranges::subrange(third_code_point, end))) { + return false; + } + ++it; + if (!subrange_equal( + it.base_code_units(), std::ranges::subrange(end, end))) { + return false; + } + } + { + test_bidi_iterator const bidi_it(arr); + test_bidi_iterator const second_code_point(std::ranges::next(bidi_it, 3)); + test_bidi_iterator const third_code_point(std::ranges::next(second_code_point, 1)); + test_bidi_iterator const end(std::ranges::next(third_code_point, 4)); + auto u32v{std::ranges::subrange{bidi_it, std::default_sentinel} | to_utf32}; + auto it{u32v.begin()}; + std::ranges::advance(it, 3); + if (!subrange_equal( + it.base_code_units(), std::ranges::subrange(end, end))) { + return false; + } + --it; + if (!subrange_equal( + it.base_code_units(), std::ranges::subrange(third_code_point, end))) { + return false; + } + --it; + if (!subrange_equal( + it.base_code_units(), std::ranges::subrange(second_code_point, third_code_point))) { + return false; + } + --it; + if (!subrange_equal( + it.base_code_units(), std::ranges::subrange(bidi_it, second_code_point))) { + return false; + } + } + } + return true; +} + CONSTEXPR_UNLESS_MSVC bool utf_view_test() { if (!input_iterator_test(std::initializer_list{u8'x'})) { return false; @@ -1608,6 +1884,9 @@ CONSTEXPR_UNLESS_MSVC bool utf_view_test() { if (!input_range_equality_test()) { return false; } + if (!base_code_units_test()) { + return false; + } return true; }