BDE 4.39.x Production Release
Loading...
Searching...
No Matches
bdlde_utf8util.h
Go to the documentation of this file.
1/// @file bdlde_utf8util.h
2///
3/// The content of this file has been pre-processed for Doxygen.
4///
5
6
7// bdlde_utf8util.h -*-C++-*-
8#ifndef INCLUDED_BDLDE_UTF8UTIL
9#define INCLUDED_BDLDE_UTF8UTIL
10
11#include <bsls_ident.h>
12BSLS_IDENT("$Id: $")
13
14/// @defgroup bdlde_utf8util bdlde_utf8util
15/// @brief Provide basic utilities for UTF-8 encodings.
16/// @addtogroup bdl
17/// @{
18/// @addtogroup bdlde
19/// @{
20/// @addtogroup bdlde_utf8util
21/// @{
22///
23/// <h1> Outline </h1>
24/// * <a href="#bdlde_utf8util-purpose"> Purpose</a>
25/// * <a href="#bdlde_utf8util-classes"> Classes </a>
26/// * <a href="#bdlde_utf8util-description"> Description </a>
27/// * <a href="#bdlde_utf8util-empty-input-strings"> Empty Input Strings </a>
28/// * <a href="#bdlde_utf8util-usage"> Usage </a>
29/// * <a href="#bdlde_utf8util-example-1-validating-strings-and-counting-unicode-code-points"> Example 1: Validating Strings and Counting Unicode Code Points </a>
30/// * <a href="#bdlde_utf8util-example-2-advancing-over-a-given-number-of-code-points"> Example 2: Advancing Over a Given Number of Code Points </a>
31/// * <a href="#bdlde_utf8util-example-3-validating-utf-8-read-from-a-bsl-streambuf"> Example 3: Validating UTF-8 Read from a bsl::streambuf </a>
32///
33/// # Purpose {#bdlde_utf8util-purpose}
34/// Provide basic utilities for UTF-8 encodings.
35///
36/// # Classes {#bdlde_utf8util-classes}
37///
38/// - bdlde::Utf8Util: namespace for utilities for UTF-8 encodings
39///
40/// # Description {#bdlde_utf8util-description}
41/// This component provides, within the `bdlde::Utf8Util` `struct`,
42/// a suite of static functions supporting UTF-8 encoded strings. Two
43/// interfaces are provided for each function, one where the length of the
44/// string (in *bytes*) is passed as a separate argument, and one where the
45/// string is passed as a null-terminated C-style string.
46///
47/// A string is deemed to contain valid UTF-8 if it is compliant with RFC 3629,
48/// meaning that only 1-, 2-, 3-, and 4-byte sequences are allowed. Values
49/// above `U+10ffff` are also not allowed.
50///
51/// Seven types of functions are provided:
52///
53/// * `isValid`, which checks for validity, per RFC 3629, of a (candidate)
54/// UTF-8 string. "Overlong values", that is, values encoded in more bytes
55/// than necessary, are not tolerated; nor are "surrogate values", which are
56/// values in the range `[U+d800 .. U+dfff]`.
57/// * `advanceIfValid` and `advanceRaw`, which advance some number of Unicode
58/// code points, each of which may be encoded in multiple bytes in a UTF-8
59/// string. `advanceRaw` assumes the string is valid UTF-8, while
60/// `advanceIfValid` checks the input for validity and stops advancing if a
61/// sequence is encountered that is not valid UTF-8.
62/// * `numCodePointsIfValid` and `numCodePointsRaw`, which return the number of
63/// Unicode code points in a UTF-8 string. Note that `numCodePointsIfValid`
64/// both validates a (candidate) UTF-8 string and counts the number of
65/// Unicode code points that it contains.
66/// * `numBytesIfValid`, which returns the number of bytes a specified number
67/// of Unicode code points occupy in a UTF-8 string.
68/// * `getByteSize`, which returns the length of a single UTF-8 encoded
69/// character.
70/// * `CodePointValue`, which returns the integral value of a single UTF-8
71/// encoded character.
72/// * `appendUtf8Character`, which appends a single Unicode code point to a
73/// UTF-8 string.
74///
75/// Embedded null bytes are allowed in strings that are accompanied by an
76/// explicit length argument. Naturally, null-terminated C-style strings cannot
77/// contain embedded null code points.
78///
79/// The UTF-8 format is described in the RFC 3629 document at:
80/// @code
81/// http://tools.ietf.org/html/rfc3629
82/// @endcode
83/// and in Wikipedia at:
84/// @code
85/// http://en.wikipedia.org/wiki/Utf-8
86/// @endcode
87///
88/// ## Empty Input Strings {#bdlde_utf8util-empty-input-strings}
89///
90///
91/// The utility functions provided by this component consider the empty string
92/// to be valid UTF-8. For those functions that take input as a
93/// `(pointer, length)` pair, if `0 == pointer` and `0 == length`, then the
94/// input is interpreted as a valid, empty string. However, if `0 == pointer`
95/// and `0 != length`, the behavior is undefined. All such functions have a
96/// counterpart that takes a lone pointer to a null-terminated (C-style) string.
97/// The behavior is always undefined if 0 is supplied for that lone pointer.
98///
99/// ## Usage {#bdlde_utf8util-usage}
100///
101///
102/// This section illustrates intended use of this component.
103///
104/// ### Example 1: Validating Strings and Counting Unicode Code Points {#bdlde_utf8util-example-1-validating-strings-and-counting-unicode-code-points}
105///
106///
107/// In this usage example, we will encode some Unicode code points in UTF-8
108/// strings and demonstrate those that are valid and those that are not.
109///
110/// First, we build an unquestionably valid UTF-8 string:
111/// @code
112/// bsl::string string;
113/// bdlde::Utf8Util::appendUtf8CodePoint(&string, 0xff00);
114/// bdlde::Utf8Util::appendUtf8CodePoint(&string, 0x856);
115/// bdlde::Utf8Util::appendUtf8CodePoint(&string, 'a');
116/// bdlde::Utf8Util::appendUtf8CodePoint(&string, 0x1008aa);
117/// bdlde::Utf8Util::appendUtf8CodePoint(&string, 0xfff);
118/// bdlde::Utf8Util::appendUtf8CodePoint(&string, 'w');
119/// bdlde::Utf8Util::appendUtf8CodePoint(&string, 0x1abcd);
120/// bdlde::Utf8Util::appendUtf8CodePoint(&string, '.');
121/// bdlde::Utf8Util::appendUtf8CodePoint(&string, '\n');
122/// @endcode
123/// Then, we check its validity and measure its length:
124/// @code
125/// assert(true == bdlde::Utf8Util::isValid(string.data(), string.length()));
126/// assert(true == bdlde::Utf8Util::isValid(string.c_str()));
127///
128/// assert( 9 == bdlde::Utf8Util::numCodePointsRaw(string.data(),
129/// string.length()));
130/// assert( 9 == bdlde::Utf8Util::numCodePointsRaw(string.c_str()));
131/// @endcode
132/// Next, we encode a lone surrogate value, `0xd8ab`, that we encode as the raw
133/// 3-byte sequence "\xed\xa2\xab" to avoid validation:
134/// @code
135/// bsl::string stringWithSurrogate = string + "\xed\xa2\xab";
136///
137/// assert(false == bdlde::Utf8Util::isValid(stringWithSurrogate.data(),
138/// stringWithSurrogate.length()));
139/// assert(false == bdlde::Utf8Util::isValid(stringWithSurrogate.c_str()));
140/// @endcode
141/// Then, we cannot use `numCodePointsRaw` to count the code points in
142/// `stringWithSurrogate`, since the behavior of that method is undefined unless
143/// the string is valid. Instead, the `numCodePointsIfValid` method can be used
144/// on strings whose validity we are uncertain of:
145/// @code
146/// const char *invalidPosition = 0;
147///
148/// bsls::Types::IntPtr rc;
149/// rc = bdlde::Utf8Util::numCodePointsIfValid(&invalidPosition,
150/// stringWithSurrogate.data(),
151/// stringWithSurrogate.length());
152/// assert(rc < 0);
153/// assert(bdlde::Utf8Util::k_SURROGATE == rc);
154/// assert(invalidPosition == stringWithSurrogate.data() + string.length());
155///
156/// invalidPosition = 0; // reset
157///
158/// rc = bdlde::Utf8Util::numCodePointsIfValid(&invalidPosition,
159/// stringWithSurrogate.c_str());
160/// assert(rc < 0);
161/// assert(bdlde::Utf8Util::k_SURROGATE == rc);
162/// assert(invalidPosition == stringWithSurrogate.data() + string.length());
163/// @endcode
164/// Now, we encode 0, which is allowed. However, note that we cannot use any
165/// interfaces that take a null-terminated string for this case:
166/// @code
167/// bsl::string stringWithNull = string;
168/// stringWithNull += '\0';
169///
170/// assert(true == bdlde::Utf8Util::isValid(stringWithNull.data(),
171/// stringWithNull.length()));
172///
173/// assert( 10 == bdlde::Utf8Util::numCodePointsRaw(stringWithNull.data(),
174/// stringWithNull.length()));
175/// @endcode
176/// Finally, we encode `0x3a` (`:`) as an overlong value using 2 bytes, which is
177/// not valid UTF-8 (since `:` can be "encoded" in 1 byte):
178/// @code
179/// bsl::string stringWithOverlong = string;
180/// stringWithOverlong += static_cast<char>(0xc0); // start of 2-byte
181/// // sequence
182/// stringWithOverlong += static_cast<char>(0x80 | ':'); // continuation byte
183///
184/// assert(false == bdlde::Utf8Util::isValid(stringWithOverlong.data(),
185/// stringWithOverlong.length()));
186/// assert(false == bdlde::Utf8Util::isValid(stringWithOverlong.c_str()));
187///
188/// rc = bdlde::Utf8Util::numCodePointsIfValid(&invalidPosition,
189/// stringWithOverlong.data(),
190/// stringWithOverlong.length());
191/// assert(rc < 0);
192/// assert(bdlde::Utf8Util::k_OVERLONG_ENCODING == rc);
193/// assert(invalidPosition == stringWithOverlong.data() + string.length());
194///
195/// rc = bdlde::Utf8Util::numCodePointsIfValid(&invalidPosition,
196/// stringWithOverlong.c_str());
197/// assert(rc < 0);
198/// assert(bdlde::Utf8Util::k_OVERLONG_ENCODING == rc);
199/// assert(invalidPosition == stringWithOverlong.data() + string.length());
200/// @endcode
201///
202/// ### Example 2: Advancing Over a Given Number of Code Points {#bdlde_utf8util-example-2-advancing-over-a-given-number-of-code-points}
203///
204///
205/// In this example, we will use the various `advance` functions to advance
206/// through a UTF-8 string.
207///
208/// First, build the string using `appendUtf8CodePoint`, keeping track of how
209/// many bytes are in each Unicode code point:
210/// @code
211/// bsl::string string;
212/// bdlde::Utf8Util::appendUtf8CodePoint(&string, 0xff00); // 3 bytes
213/// bdlde::Utf8Util::appendUtf8CodePoint(&string, 0x1ff); // 2 bytes
214/// bdlde::Utf8Util::appendUtf8CodePoint(&string, 'a'); // 1 byte
215/// bdlde::Utf8Util::appendUtf8CodePoint(&string, 0x1008aa); // 4 bytes
216/// bdlde::Utf8Util::appendUtf8CodePoint(&string, 0x1abcd); // 4 bytes
217/// string += "\xe3\x8f\xfe"; // 3 bytes (invalid 3-byte sequence,
218/// // the first 2 bytes are valid but the
219/// // last continuation byte is invalid)
220/// bdlde::Utf8Util::appendUtf8CodePoint(&string, 'w'); // 1 byte
221/// bdlde::Utf8Util::appendUtf8CodePoint(&string, '\n'); // 1 byte
222/// @endcode
223/// Then, declare a few variables we'll need:
224/// @code
225/// bsls::Types::IntPtr rc;
226/// int status;
227/// const char *result;
228/// const char *const start = string.c_str();
229/// @endcode
230/// Next, try advancing 2 code points, then 3, then 4, observing that the value
231/// returned is the number of Unicode code points advanced. Note that since
232/// we're only advancing over valid UTF-8, we can use either `advanceRaw` or
233/// `advanceIfValid`:
234/// @code
235/// rc = bdlde::Utf8Util::advanceRaw( &result, start, 2);
236/// assert(2 == rc);
237/// assert(3 + 2 == result - start);
238///
239/// rc = bdlde::Utf8Util::advanceIfValid(&status, &result, start, 2);
240/// assert(0 == status);
241/// assert(2 == rc);
242/// assert(3 + 2 == result - start);
243///
244/// rc = bdlde::Utf8Util::advanceRaw( &result, start, 3);
245/// assert(3 == rc);
246/// assert(3 + 2 + 1 == result - start);
247///
248/// rc = bdlde::Utf8Util::advanceIfValid(&status, &result, start, 3);
249/// assert(0 == status);
250/// assert(3 == rc);
251/// assert(3 + 2 + 1 == result - start);
252///
253/// rc = bdlde::Utf8Util::advanceRaw( &result, start, 4);
254/// assert(4 == rc);
255/// assert(3 + 2 + 1 + 4 == result - start);
256///
257/// rc = bdlde::Utf8Util::advanceIfValid(&status, &result, start, 4);
258/// assert(0 == status);
259/// assert(4 == rc);
260/// assert(3 + 2 + 1 + 4 == result - start);
261/// @endcode
262/// Then, try advancing by more code points than are present using
263/// `advanceIfValid`, and wind up stopping when we encounter invalid input. The
264/// behavior of `advanceRaw` is undefined if it is used on invalid input, so we
265/// cannot use it here. Also note that we will stop at the beginning of the
266/// invalid Unicode code point, and not at the first incorrect byte, which is
267/// two bytes later:
268/// @code
269/// rc = bdlde::Utf8Util::advanceIfValid(&status, &result, start, INT_MAX);
270/// assert(0 != status);
271/// assert(5 == rc);
272/// assert(3 + 2 + 1 + 4 + 4 == result - start);
273/// assert(static_cast<int>(string.length()) > result - start);
274/// @endcode
275/// Now, doctor the string to replace the invalid code point with a valid one,
276/// so the string is entirely correct UTF-8:
277/// @code
278/// string[3 + 2 + 1 + 4 + 4 + 2] = static_cast<char>(0x8a);
279/// @endcode
280/// Finally, advance using both functions by more code points than are in the
281/// string and in both cases wind up at the end of the string. Note that
282/// `advanceIfValid` does not return an error (non-zero) value to `status` when
283/// it encounters the end of the string:
284/// @code
285/// rc = bdlde::Utf8Util::advanceRaw( &result, start, INT_MAX);
286/// assert(8 == rc);
287/// assert(3 + 2 + 1 + 4 + 4 + 3 + 1 + 1 == result - start);
288/// assert(static_cast<int>(string.length()) == result - start);
289///
290/// rc = bdlde::Utf8Util::advanceIfValid(&status, &result, start, INT_MAX);
291/// assert(0 == status);
292/// assert(8 == rc);
293/// assert(3 + 2 + 1 + 4 + 4 + 3 + 1 + 1 == result - start);
294/// assert(static_cast<int>(string.length()) == result - start);
295/// @endcode
296///
297/// ### Example 3: Validating UTF-8 Read from a bsl::streambuf {#bdlde_utf8util-example-3-validating-utf-8-read-from-a-bsl-streambuf}
298///
299///
300/// In this usage example, we will demonstrate reading and validating UTF-8
301/// from a stream.
302///
303/// We write a function to read valid UTF-8 to a `bsl::string`. We don't know
304/// how long the input will be, so we don't know how long to make the string
305/// before we start. We will grow the string in small, 32-byte increments.
306/// @code
307/// /// Read valid UTF-8 from the specified streambuf `sb` to the specified
308/// /// `output`. Return 0 if the input was exhausted without encountering
309/// /// any invalid UTF-8, and a non-zero value otherwise. If invalid UTF-8
310/// /// is encountered, log a message describing the problem after loading
311/// /// all the valid UTF-8 preceding it into `output`. Note that after the
312/// /// call, in no case will `output` contain any invalid UTF-8.
313/// int utf8StreambufToString(bsl::string *output,
314/// bsl::streambuf *sb)
315/// {
316/// enum { k_READ_LENGTH = 32 };
317///
318/// output->clear();
319/// while (true) {
320/// bsl::size_t len = output->length();
321/// output->resize(len + k_READ_LENGTH);
322/// int status;
323/// IntPtr numBytes = bdlde::Utf8Util::readIfValid(&status,
324/// &(*output)[len],
325/// k_READ_LENGTH,
326/// sb);
327/// BSLS_ASSERT(0 <= numBytes);
328/// BSLS_ASSERT(numBytes <= k_READ_LENGTH);
329///
330/// output->resize(len + numBytes);
331/// if (0 < status) {
332/// // Buffer was full before the end of input was encountered.
333/// // Note that `numBytes` may be up to 3 bytes less than
334/// // `k_READ_LENGTH`.
335///
336/// BSLS_ASSERT(k_READ_LENGTH - 4 < numBytes);
337///
338/// // Go on to grow the string and get more input.
339///
340/// continue;
341/// }
342/// else if (0 == status) {
343/// // Success! We've reached the end of input without
344/// // encountering any invalid UTF-8.
345///
346/// return 0; // RETURN
347/// }
348/// else {
349/// // Invalid UTF-8 encountered; the value of `status` indicates
350/// // the exact nature of the problem. `numBytes` returned from
351/// // the above call indicated the number of valid UTF-8 bytes
352/// // read before encountering the invalid UTF-8.
353///
354/// BSLS_LOG_ERROR("Invalid UTF-8 error %s at position %u.\n",
355/// bdlde::Utf8Util::toAscii(status),
356/// static_cast<unsigned>(output->length()));
357///
358/// return -1; // RETURN
359/// }
360/// }
361/// }
362/// @endcode
363/// @}
364/** @} */
365/** @} */
366
367/** @addtogroup bdl
368 * @{
369 */
370/** @addtogroup bdlde
371 * @{
372 */
373/** @addtogroup bdlde_utf8util
374 * @{
375 */
376
377#include <bdlscm_version.h>
378
379#include <bsls_assert.h>
380#include <bsls_libraryfeatures.h>
381#include <bsls_review.h>
382#include <bsls_types.h>
383
384#include <bsl_cstddef.h>
385#include <bsl_cstdlib.h>
386#include <bsl_iosfwd.h>
387#include <bsl_streambuf.h>
388#include <bsl_string.h>
389
390#include <string>
391
392
393
394namespace bdlde {
395 // ===============
396 // struct Utf8Util
397 // ===============
398
399/// This struct provides a namespace for static methods used for validating
400/// UTF-8 strings, for counting the number of Unicode code points in them,
401/// for advancing pointers through UTF-8 strings by a specified number of
402/// Unicode code points, for counting the number of bytes a UTF-8 leading
403/// substring occupies, for counting the number of bytes in a UTF-8
404/// character, and for appending a Unicode character to a UTF-8 string.
405///
406/// See @ref bdlde_utf8util
407struct Utf8Util {
408
409 // PUBLIC TYPES
414
415 /// Default code point to be substituted for errors.
416 enum { k_ERROR_CODE_POINT = 0xfffd };
417
418 /// Enumerate the error status values that are returned (possibly
419 /// through an out parameter) from some methods in this utility.
420 ///
421 /// \note Note that some of the functions in this `struct` have a return value
422 /// that is non-negative on success, and one of these values when an
423 /// error occurs, so all of these values must be negative to distinguish
424 /// them from a "success" value.
426
428 // The end of input was reached partway through a multibyte UTF-8
429 // sequence.
430
432 // A continuation byte was encountered when not within a multibyte
433 // sequence.
434
436 // A non-continuation byte was encountered where a continuation byte
437 // was expected.
438
440 // The encoded Unicode value could have been encoded in a sequence
441 // of fewer bytes.
442
444 // A sequence began with an octet with its 5 highest-order bits all
445 // set, which is always invalid in UTF-8.
446
448 // A value larger than 0x10FFFF was encoded.
449
450 k_SURROGATE = -7
451 // Illegal occurrence of Unicode code point reserved for surrogate
452 // values in UTF-16. Note that all surrogate values are illegal as
453 // Unicode code points.
454 };
455
456 // CLASS METHODS
457
458 /// Advance past 0 or more consecutive *valid* Unicode code points at
459 /// the beginning of the specified `string`, until either the specified
460 /// `numCodePoints` have been traversed, or the terminating null byte or
461 /// invalid UTF-8 is encountered (whichever occurs first), and return
462 /// the number of Unicode code points traversed. Set the specified
463 /// `*status` to 0 if no invalid UTF-8 is encountered, and to a value
464 /// from the `ErrorStatus` `enum` otherwise. Set the specified
465 /// `*result` to the address of the byte immediately following the last
466 /// valid code point traversed, or to `string` if `string` is empty or
467 /// `numCodePoints` is 0. `string` is necessarily null-terminated, so
468 /// it cannot contain embedded null bytes.
469 ///
470 /// \pre The behavior is undefined unless `0 <= numCodePoints`.
471 /// \note Note that the value returned will be
472 /// in the range `[0 .. numCodePoints]`. Also note that `string` may
473 /// contain less than `bsl::strlen(string)` Unicode code points.
474 static IntPtr advanceIfValid(int *status,
475 const char **result,
476 const char *string,
477 IntPtr numCodePoints);
478
479 /// Advance past 0 or more consecutive *valid* Unicode code points at
480 /// the beginning of the specified `string` having the specified
481 /// `length` (in bytes), until either the specified `numCodePoints` or
482 /// `length` bytes have been traversed, or invalid UTF-8 is encountered
483 /// (whichever occurs first), and return the number of Unicode code
484 /// points traversed. Set the specified `*status` to 0 if no invalid
485 /// UTF-8 is encountered, and to a value from the `ErrorStatus` `enum`
486 /// otherwise. Set the specified `*result` to the address of the byte
487 /// immediately following the last valid code point traversed, or to
488 /// `string` if `length` or `numCodePoints` is 0. `string` need not be
489 /// null-terminated and can contain embedded null bytes, and `string`
490 /// may be null if `0 == length` (see {Empty Input Strings}).
491 ///
492 /// \pre The behavior is undefined unless `0 <= numCodePoints`.
493 /// \note Note that the
494 /// value returned will be in the range `[0 .. numCodePoints]`. Also
495 /// note that `string` may contain less than `length` Unicode code
496 /// points.
497 static IntPtr advanceIfValid(int *status,
498 const char **result,
499 const char *string,
500 size_type length,
501 IntPtr numCodePoints);
502
503 /// Advance past 0 or more consecutive *valid* Unicode code points at
504 /// the beginning of the specified `string`, until either the specified
505 /// `numCodePoints` bytes or the whole `string` have been traversed, or
506 /// invalid UTF-8 is encountered (whichever occurs first), and return
507 /// the number of Unicode code points traversed. Set the specified
508 /// `*status` to 0 if no invalid UTF-8 is encountered, and to a value
509 /// from the `ErrorStatus` `enum` otherwise. Set the specified
510 /// `*result` to the address of the byte immediately following the last
511 /// valid code point traversed, or to `string` if its length or
512 /// `numCodePoints` is 0. `string` need not be null-terminated and can contain embedded null bytes.
513 ///
514 /// \pre The behavior is undefined unless `0 <= numCodePoints`.
515 ///
516 /// \note Note that the value returned will be in the
517 /// range `[0 .. numCodePoints]`. Also note that `string` may contain
518 /// less than `string.length()` Unicode code points.
519 static IntPtr advanceIfValid(int *status,
520 const char **result,
521 const bsl::string_view& string,
522 IntPtr numCodePoints);
523
524 /// Advance past 0 or more consecutive Unicode code points at the
525 /// beginning of the specified `string`, until either the specified
526 /// `numCodePoints` bytes have been traversed or the terminating null
527 /// byte is encountered (whichever occurs first), and return the number
528 /// of Unicode code points traversed. Set the specified `*result` to
529 /// the address of the byte immediately following the last code point
530 /// traversed, or to `string` if `string` is empty or `numCodePoints` is
531 /// 0. `string` is necessarily null-terminated, so it cannot contain embedded null bytes.
532 ///
533 /// \pre The behavior is undefined unless `string` contains valid UTF-8 and `0 <= numCodePoints`.
534 ///
535 /// \note Note that the value
536 /// returned will be in the range `[0 .. numCodePoints]`. Also note
537 /// that `string` may contain less than `bsl::strlen(string)` Unicode
538 /// code points.
539 static IntPtr advanceRaw(const char **result,
540 const char *string,
541 IntPtr numCodePoints);
542
543 /// Advance past 0 or more consecutive Unicode code points at the
544 /// beginning of the specified `string` having the specified `length`
545 /// (in bytes), until either the specified `numCodePoints` or `length`
546 /// bytes have been traversed (whichever occurs first), and return the
547 /// number of Unicode code points traversed. Set the specified
548 /// `*result` to the address of the byte immediately following the last
549 /// code point traversed, or to `string` if `length` or `numCodePoints`
550 /// is 0. `string` need not be null-terminated and can contain embedded
551 /// null bytes, and `string` may be null if `0 == length` (see {Empty Input Strings}).
552 ///
553 /// \pre The behavior is undefined unless the initial
554 /// `length` bytes of `string` contain valid UTF-8 and `0 <= numCodePoints`.
555 ///
556 /// \note Note that the value returned will be in the
557 /// range `[0 .. numCodePoints]`. Also note that `string` may contain
558 /// less than `length` Unicode code points.
559 static IntPtr advanceRaw(const char **result,
560 const char *string,
561 size_type length,
562 IntPtr numCodePoints);
563
564 /// Advance past 0 or more consecutive Unicode code points at the
565 /// beginning of the specified `string`, until either the specified
566 /// `numCodePoints` bytes or the whole string have been traversed
567 /// (whichever occurs first), and return the number of Unicode code
568 /// points traversed. Set the specified `*result` to the address of the
569 /// byte immediately following the last code point traversed, or to
570 /// `string` if `length` or `numCodePoints` is 0. `string` need not be
571 /// null-terminated and can contain embedded null bytes.
572 ///
573 /// \pre The behavior is undefined unless `string` contains only valid UTF-8 characters and `0 <= numCodePoints`.
574 ///
575 /// \note Note that the value returned will be in
576 /// the range `[0 .. numCodePoints]`. Also note that `string` may
577 /// contain less than `length` Unicode code points.
578 static IntPtr advanceRaw(const char **result,
579 const bsl::string_view& string,
580 IntPtr numCodePoints);
581
582 /// Append the UTF-8 encoding of the specified Unicode `codePoint` to
583 /// the specified `output` string. Return 0 on success, and a non-zero
584 /// value otherwise.
585 ///
586 /// @deprecated Use @ref appendUtf8CodePoint instead.
587 static int appendUtf8Character(bsl::string *output,
588 unsigned int codePoint);
589
590 /// Append the UTF-8 encoding of the specified Unicode `codePoint` to
591 /// the specified `output` string. Return 0 on success, and a non-zero
592 /// value otherwise.
594 unsigned int codePoint);
595 static int appendUtf8CodePoint(std::string *output,
596 unsigned int codePoint);
597#ifdef BSLS_LIBRARYFEATURES_HAS_CPP17_PMR_STRING
598 static int appendUtf8CodePoint(std::pmr::string *output,
599 unsigned int codePoint);
600#endif
601
602 /// Return the numeric value of the UTF-8-encoded code point beginning at the specified `codePoint`.
603 ///
604 /// \pre The behavior is undefined unless
605 /// `codePoint` is the address of the first byte of a valid UTF-8
606 /// encoded character.
607 static int codePointValue(const char *codePoint);
608
609 /// Return the length (in bytes) of the UTF-8-encoded code point
610 /// beginning at the specified `codePoint`.
611 ///
612 /// \pre The behavior is undefined unless `codePoint` is the address of the first byte of a valid UTF-8 encoded character.
613 ///
614 /// \note Note that the value returned will be in the
615 /// range `[1 .. 4]`. Also note that 1 is returned if `0 == *codePoint`
616 /// since '\0' is a valid 1-byte encoding.
617 ///
618 /// @deprecated Use @ref numBytesInCodePoint instead.
619 static int getByteSize(const char *codePoint);
620
621 /// Return the length (in bytes) of the UTF-8-encoded code point
622 /// beginning at the specified `codePoint`.
623 ///
624 /// \pre The behavior is undefined unless `codePoint` is the address of the first byte of a valid UTF-8 encoded character.
625 ///
626 /// \note Note that the value returned will be in the
627 /// range `[1 .. 4]`. Also note that 1 is returned if `0 == *codePoint`
628 /// since '\0' is a valid 1-byte encoding.
629 static int numBytesInCodePoint(const char *codePoint);
630
631 /// For the specified `byteOffset` in the specified `input`, load the
632 /// offset's line number into the specified `lineNumber`, the column
633 /// number into the specified `utf8Column`, and the byte offset for the
634 /// start of the line into `startOfLineByteOffset`. Optionally specify
635 /// `lineDelimeter` used to the determine line separator. If
636 /// `lineDelimeter` is not supplied, lines are delimited using `\n`.
637 /// Return 0 on success, or a non-zero value if `location` cannot be
638 /// found in `input` or if `input` contains non-UTF-8 characters. The
639 /// `utf8Column` is the number of UTF-8 code points between
640 /// `startOfLineByteOffset` and `byteOffset`.
641 static int getLineAndColumnNumber(Uint64 *lineNumber,
642 Uint64 *utf8Column,
643 Uint64 *startOfLineByteOffset,
644 bsl::streambuf *input,
645 Uint64 byteOffset);
646 static int getLineAndColumnNumber(Uint64 *lineNumber,
647 Uint64 *utf8Column,
648 Uint64 *startOfLineByteOffset,
649 bsl::streambuf *input,
650 Uint64 byteOffset,
651 char lineDelimeter);
652
653 /// Return `true` if the specified `string` contains valid UTF-8, and
654 /// `false` otherwise. `string` is necessarily null-terminated, so it
655 /// cannot contain embedded null bytes.
656 static bool isValid(const char *string);
657
658 /// Return `true` if the specified `string` having the specified
659 /// `length` (in bytes) contains valid UTF-8, and `false` otherwise.
660 /// `string` need not be null-terminated and can contain embedded null
661 /// bytes, and `string` may be null if `0 == length` (see {Empty Input
662 /// Strings}).
663 static bool isValid(const char *string, size_type length);
664
665 /// Return `true` if the specified `string` contains valid UTF-8, and
666 /// `false` otherwise. `string` need not be null-terminated and can
667 /// contain embedded null bytes.
668 static bool isValid(const bsl::string_view& string);
669
670 /// Return `true` if the specified `string` contains valid UTF-8, and
671 /// `false` otherwise. If `string` contains invalid UTF-8, load into
672 /// the specified `invalidString` the address of the beginning of the
673 /// first invalid UTF-8 sequence encountered; `invalidString` is
674 /// unaffected if `string` contains only valid UTF-8. `string` is
675 /// necessarily null-terminated, so it cannot contain embedded null
676 /// bytes.
677 static bool isValid(const char **invalidString, const char *string);
678
679 /// Return `true` if the specified `string` having the specified
680 /// `length` (in bytes) contains valid UTF-8, and `false` otherwise. If
681 /// `string` contains invalid UTF-8, load into the specified
682 /// `invalidString` the address of the byte after the last valid code
683 /// point traversed; `invalidString` is unaffected if `string` contains
684 /// only valid UTF-8. `string` need not be null-terminated and can
685 /// contain embedded null bytes, and `string` may be null if
686 /// `0 == length` (see {Empty Input Strings}).
687 static bool isValid(const char **invalidString,
688 const char *string,
689 size_type length);
690
691 /// Return `true` if the specified `string` contains only valid UTF-8
692 /// characters, and `false` otherwise. If `string` contains invalid
693 /// UTF-8, load into the specified `invalidString` the address of the
694 /// byte after the last valid code point traversed; `invalidString` is
695 /// unaffected if `string` contains only valid UTF-8. `string` need not
696 /// be null-terminated and can contain embedded null bytes.
697 static bool isValid(const char **invalidString,
698 const bsl::string_view& string);
699
700 /// If the specified `codePoint` (having at least the specified
701 /// `numBytes`) refers to a valid UTF-8 code point then return `true`
702 /// and load the specified `status` with the number of bytes in the
703 /// code-point; otherwise, if `codePoint` is not a valid code-point,
704 /// return `false` and load `status` with one of the (negative) `ErrorStatus` constants.
705 ///
706 /// \pre The behavior is undefined unless
707 /// `numBytes > 0`.
708 static bool isValidCodePoint(int *status,
709 const char *codePoint,
710 size_type numBytes);
711
712 /// Return the length (in bytes) of the specified `numCodePoints` UTF-8
713 /// encodings in the specified `string`, or a value less than 0 if
714 /// `string` contains less than `numCodePoints` encodings.
715 ///
716 /// \pre The behavior is undefined unless `string` refers to valid UTF-8.
717 ///
718 /// \note Note that `string` may contain more than `numCodePoints` encodings in which
719 /// case the trailing ones are ignored.
720 ///
721 /// @deprecated Use @ref numBytesRaw instead.
722 static IntPtr numBytesIfValid(const bsl::string_view& string,
723 IntPtr numCodePoints);
724
725 /// Return the length (in bytes) of the specified `numCodePoints` UTF-8
726 /// encodings in the specified `string`, or a value less than 0 if
727 /// `string` contains less than `numCodePoints` encodings.
728 ///
729 /// \pre The behavior is undefined unless `string` refers to valid UTF-8.
730 ///
731 /// \note Note that `string` may contain more than `numCodePoints` encodings in which
732 /// case the trailing ones are ignored.
733 static IntPtr numBytesRaw(const bsl::string_view& string,
734 IntPtr numCodePoints);
735
736 /// Return the number of Unicode code points in the specified `string`.
737 /// `string` is necessarily null-terminated, so it cannot contain embedded null bytes.
738 ///
739 /// \pre The behavior is undefined unless `string` contains valid UTF-8.
740 ///
741 /// \note Note that `string` may contain less than
742 /// `bsl::strlen(string)` Unicode code points.
743 ///
744 /// @deprecated Use @ref numCodePointsRaw instead.
745 static IntPtr numCharacters(const char *string);
746
747 /// Return the number of Unicode code points in the specified `string`
748 /// having the specified `length` (in bytes). `string` need not be
749 /// null-terminated and can contain embedded null bytes, and `string`
750 /// may be null if `0 == length` (see {Empty Input Strings}).
751 ///
752 /// \pre The behavior is undefined unless `string` contains valid UTF-8.
753 ///
754 /// \note Note that `string` may contain less than `length` Unicode code points.
755 ///
756 /// @deprecated Use @ref numCodePointsRaw instead.
757 static IntPtr numCharacters(const char *string, size_type length);
758
759 /// Return the number of Unicode code points in the specified `string`
760 /// if it contains valid UTF-8, with no effect on the specified
761 /// `invalidString`. Otherwise, return a negative value and load into
762 /// `invalidString` the address of the byte after the last valid Unicode
763 /// code point traversed. `string` is necessarily null-terminated, so it cannot contain embedded null bytes.
764 ///
765 /// \note Note that `string` may
766 /// contain less than `bsl::strlen(string)` Unicode code points.
767 ///
768 /// @deprecated Use @ref numCodePointsIfValid instead.
769 static IntPtr numCharactersIfValid(const char **invalidString,
770 const char *string);
771
772 /// Return the number of Unicode code points in the specified `string`
773 /// having the specified `length` (in bytes) if `string` contains valid
774 /// UTF-8, with no effect on the specified `invalidString`. Otherwise,
775 /// return a negative value and load into `invalidString` the address of
776 /// the byte after the last valid Unicode code point traversed.
777 /// `string` need not be null-terminated and may contain embedded null
778 /// bytes, and `string` may be null if `0 == length` (see {Empty Input Strings}).
779 ///
780 /// \note Note that `string` may contain less than `length`
781 /// Unicode code points.
782 ///
783 /// @deprecated Use @ref numCodePointsIfValid instead.
784 static IntPtr numCharactersIfValid(const char **invalidString,
785 const char *string,
786 size_type length);
787
788 /// Return the number of Unicode code points in the specified `string`.
789 /// `string` is necessarily null-terminated, so it cannot contain embedded null bytes.
790 ///
791 /// \pre The behavior is undefined unless `string` contains valid UTF-8.
792 ///
793 /// \note Note that `string` may contain less than
794 /// `bsl::strlen(string)` Unicode code points.
795 ///
796 /// @deprecated Use @ref numCodePointsRaw instead.
797 static IntPtr numCharactersRaw(const char *string);
798
799 /// Return the number of Unicode code points in the specified `string`
800 /// having the specified `length` (in bytes). `string` need not be
801 /// null-terminated and can contain embedded null bytes, and `string`
802 /// may be null if `0 == length` (see {Empty Input Strings}).
803 ///
804 /// \pre The behavior is undefined `string` contains valid UTF-8.
805 ///
806 /// \note Note that `string` may contain less than `length` Unicode code points.
807 ///
808 /// @deprecated Use @ref numCodePointsRaw instead.
809 static IntPtr numCharactersRaw(const char *string, size_type length);
810
811 /// Return the number of Unicode code points in the specified `string`
812 /// if it contains valid UTF-8, with no effect on the specified
813 /// `invalidString`. Otherwise, return a value from the `ErrorStatus`
814 /// `enum` (which are all negative) and load into `invalidString` the
815 /// address of the byte after the last valid Unicode code point
816 /// traversed. `string` is necessarily null-terminated, so it cannot contain embedded null bytes.
817 ///
818 /// \note Note that `string` may contain less
819 /// than `bsl::strlen(string)` Unicode code points.
820 static IntPtr numCodePointsIfValid(const char **invalidString,
821 const char *string);
822
823 /// Return the number of Unicode code points in the specified `string`
824 /// having the specified `length` (in bytes) if `string` contains valid
825 /// UTF-8, with no effect on the specified `invalidString`. Otherwise,
826 /// return a value from the `ErrorStatus` `enum` (which are all
827 /// negative) and load into `invalidString` the address of the byte
828 /// after the last valid Unicode code point traversed. `string` need
829 /// not be null-terminated and may contain embedded null bytes, and
830 /// `string` may be null if `0 == length` (see {Empty Input Strings}).
831 ///
832 /// \note Note that `string` may contain less than `length` Unicode code
833 /// points.
834 static IntPtr numCodePointsIfValid(const char **invalidString,
835 const char *string,
836 size_type length);
837
838 /// Return the number of Unicode code points in the specified `string`
839 /// if `string` contains valid UTF-8, with no effect on the specified
840 /// `invalidString`. Otherwise, return a value from the `ErrorStatus`
841 /// `enum` (which are all negative) and load into `invalidString` the
842 /// address of the byte after the last valid Unicode code point
843 /// traversed. `string` need not be null-terminated and may contain
844 /// embedded null bytes.
845 static IntPtr numCodePointsIfValid(const char **invalidString,
846 const bsl::string_view& string);
847
848 /// Return the number of Unicode code points in the specified `string`.
849 /// `string` is necessarily null-terminated, so it cannot contain embedded null bytes.
850 ///
851 /// \pre The behavior is undefined unless `string` contains valid UTF-8.
852 ///
853 /// \note Note that `string` may contain less than
854 /// `bsl::strlen(string)` Unicode code points.
855 static IntPtr numCodePointsRaw(const char *string);
856
857 /// Return the number of Unicode code points in the specified `string`
858 /// having the specified `length` (in bytes). `string` need not be
859 /// null-terminated and can contain embedded null bytes, and `string`
860 /// may be null if `0 == length` (see {Empty Input Strings}).
861 ///
862 /// \pre The behavior is undefined unless `string` contains valid UTF-8.
863 ///
864 /// \note Note that `string` may contain less than `length` Unicode code points.
865 static IntPtr numCodePointsRaw(const char *string, size_type length);
866
867 /// Return the number of Unicode code points in the specified `string`.
868 /// `string` need not be null-terminated and can contain embedded null bytes.
869 ///
870 /// \pre The behavior is undefined unless `string` contains valid
871 /// UTF-8.
872 static IntPtr numCodePointsRaw(const bsl::string_view& string);
873
874 /// Read from the specified `input` and copy *valid* UTF-8 (only) to the
875 /// specified `outputBuffer` having the specified `outputBufferLength`
876 /// (in bytes). Load the specified `status` with:
877 /// * 0 if `input` reached `eof` without encountering any invalid UTF-8
878 /// or prematurely exhausting `outputBuffer`.
879 /// * A positive value if `input` was not completely read due to
880 /// `outputBuffer` being filled (or nearly filled) without
881 /// encountering any invalid UTF-8.
882 /// * A negative value from `ErrorStatus` if invalid UTF-8 was
883 /// encountered (without having written the invalid sequence to
884 /// `outputBuffer`).
885 /// Return the number of bytes of valid UTF-8 written to 'outputBuffer.
886 /// If no invalid UTF-8 is encountered, or if `input` supports
887 /// `sputbackc` with a putback buffer capacity of at least 4 bytes,
888 /// `input` will be left positioned at the end of the valid UTF-8 read,
889 /// otherwise, `input` will be left in an unspecified state.
890 ///
891 /// \pre The behavior is undefined unless `4 <= outputBufferLength`.
892 ///
893 /// \note Note that this function will stop reading `input` when less than 4 bytes of
894 /// space remain in `outputBuffer` to prevent the possibility of a
895 /// 4-byte UTF-8 sequence being truncated partway through.
896 static size_type readIfValid(int *status,
897 char *outputBuffer,
898 size_type outputBufferLength,
899 bsl::streambuf *input);
900
901 /// Translate the specified `input`, which is UTF-8, replacing error
902 /// sequences in `input` with `utf32ErrorCode` translated to UTF-8, and
903 /// return the number of errors found, putting the result in `*output`.
904 /// Any previous contents of `*output` are discarded. If `utf32ErrorCode`
905 /// is 0, error sequences will be simply deleted.
906 ///
907 /// \pre The behavior is undefined unless `utf32ErrorCode` is a valid unicode value for UTF-8.
909 bsl::string *output,
910 const bsl::string_view& input,
911 unsigned int utf32ErrorCode = k_ERROR_CODE_POINT);
913 std::string *output,
914 const bsl::string_view& input,
915 unsigned int utf32ErrorCode = k_ERROR_CODE_POINT);
916#ifdef BSLS_LIBRARYFEATURES_HAS_CPP17_PMR_STRING
918 std::pmr::string *output,
919 const bsl::string_view& input,
920 unsigned int utf32ErrorCode = k_ERROR_CODE_POINT);
921#endif
922
923 /// Return the non-modifiable string representation of the `ErrorStatus`
924 /// enumerator matching the specified `value`, if it exists, and "(*
925 /// unrecognized value *)" otherwise. The string representation of an
926 /// enumerator that matches `value` is the enumerator name with the "k_" prefix elided.
927 ///
928 /// \note Note that this method may be used to aid in
929 /// interpreting status values that are returned from some methods in
930 /// this utility. See `ErrorStatus`.
931 static const char *toAscii(IntPtr value);
932};
933
934 // =======================
935 // struct Utf8Util_ImpUtil
936 // =======================
937
938/// [**PRIVATE**] This struct provides a namespace for static methods used to implement `Utf8Util`.
939///
940/// \note Note that the functions are not typically useful
941/// for clients, and are primarily exposed to allow for more thorough
942/// testing.
943///
944/// See @ref bdlde_utf8util
946 // CLASS METHODS
947
950
951 /// Given a string referred to by the specified `input`, return the number
952 /// of bytes in first code point in the string, whether the code point is valid or invalid.
953 ///
954 /// \note Note that if the string begins with an invalid
955 /// header octet `(0xf8 == (*input & 0xf8))` only that octet is skipped,
956 /// any following continuation octets are seen as separate errors.
957 ///
958 /// \pre The behavior is undefined if `input.empty()`.
960 const bsl::string_view& input);
961
962 /// For the specified `byteOffset` in the specified `input`, load the
963 /// byte offset's line number into the specified `lineNumber`, the
964 /// column number into the specified `utf8Column`, and the byte offset
965 /// for the start of the line into the specified
966 /// `startOfLineByteOffset`, using the specified `lineDelimeter` as the
967 /// line separator, and using the specified `temporaryReadBuffer` (of
968 /// the specified length `temporaryReadBufferNumBytes`) as a temporary
969 /// buffer for reading. Return 0 on success, or a non-zero value if
970 /// `location` cannot be found in `input` or if `input` contains
971 /// non-UTF-8 characters. The `utf8Column` is the number of UTF-8 code
972 /// points between `startOfLineByteOffset` and `byteOffset`.
973 ///
974 /// \pre The behavior is undefined unless `temporaryReadBuffer` refers to a valid
975 /// buffer of at least `temporaryReadBufferNumBytes` bytes, and
976 /// `temporaryReadBufferNumBytes` is greater than or equal to 4.
978 Uint64 *lineNumber,
979 Uint64 *utf8Column,
980 Uint64 *startOfLineByteOffset,
981 bsl::streambuf *input,
982 Uint64 byteOffset,
983 char lineDelimeter,
984 char *temporaryReadBuffer,
985 int temporaryReadBufferNumBytes);
986};
987
988// ============================================================================
989// INLINE DEFINITIONS
990// ============================================================================
991
992 // ---------------
993 // struct Utf8Util
994 // ---------------
995
996// CLASS METHODS
997inline
999 int *status,
1000 const char **result,
1001 const bsl::string_view& string,
1002 IntPtr numCodePoints)
1003{
1004
1005 return advanceIfValid(status,
1006 result,
1007 string.data(),
1008 string.length(),
1009 numCodePoints);
1010}
1011
1012inline
1014 const bsl::string_view& string,
1015 IntPtr numCodePoints)
1016{
1017 return advanceRaw(result, string.data(), string.length(), numCodePoints);
1018}
1019
1020inline
1022 unsigned int codePoint)
1023{
1024 return appendUtf8CodePoint(output, codePoint);
1025}
1026
1027inline
1028int Utf8Util::getByteSize(const char *codePoint)
1029{
1030 return numBytesInCodePoint(codePoint);
1031}
1032
1033inline
1035 Uint64 *utf8Column,
1036 Uint64 *startOfLineByteOffset,
1037 bsl::streambuf *input,
1038 Uint64 byteOffset)
1039{
1040 return getLineAndColumnNumber(lineNumber,
1041 utf8Column,
1042 startOfLineByteOffset,
1043 input,
1044 byteOffset,
1045 '\n');
1046}
1047
1048inline
1050 Uint64 *utf8Column,
1051 Uint64 *startOfLineByteOffset,
1052 bsl::streambuf *input,
1053 Uint64 byteOffset,
1054 char lineDelimeter)
1055{
1056 enum { k_BUFFER_SIZE = 2048 };
1057 char buffer[k_BUFFER_SIZE];
1059 utf8Column,
1060 startOfLineByteOffset,
1061 input,
1062 byteOffset,
1063 lineDelimeter,
1064 buffer,
1065 k_BUFFER_SIZE);
1066}
1067
1068inline
1069bool Utf8Util::isValid(const char *string)
1070{
1071 BSLS_ASSERT(string);
1072
1073 const char *dummy = 0;
1074 return isValid(&dummy, string);
1075}
1076
1077inline
1078bool Utf8Util::isValid(const char *string, size_type length)
1079{
1080 BSLS_ASSERT(string || 0 == length);
1081
1082 const char *dummy = 0;
1083 return isValid(&dummy, string, length);
1084}
1085
1086inline
1088{
1089 const char *dummy = 0;
1090 return isValid(&dummy, string);
1091}
1092
1093inline
1095 const bsl::string_view& string,
1096 IntPtr numCodePoints)
1097{
1098 return numBytesRaw(string, numCodePoints);
1099}
1100
1101inline
1103{
1104 return numCodePointsRaw(string);
1105}
1106
1107inline
1109{
1110 return numCodePointsRaw(string, length);
1111}
1112
1113inline
1115 const char *string)
1116{
1117 return numCodePointsIfValid(invalidString, string);
1118}
1119
1120inline
1122 const char *string,
1123 size_type length)
1124{
1125 return numCodePointsIfValid(invalidString, string, length);
1126}
1127
1128inline
1130{
1131 return numCodePointsRaw(string);
1132}
1133
1134inline
1136 size_type length)
1137{
1138 return numCodePointsRaw(string, length);
1139}
1140
1141inline
1143{
1144 return numCodePointsRaw(string.data(), string.length());
1145}
1146
1147} // close package namespace
1148
1149
1150#endif
1151
1152// ----------------------------------------------------------------------------
1153// Copyright 2015 Bloomberg Finance L.P.
1154//
1155// Licensed under the Apache License, Version 2.0 (the "License");
1156// you may not use this file except in compliance with the License.
1157// You may obtain a copy of the License at
1158//
1159// http://www.apache.org/licenses/LICENSE-2.0
1160//
1161// Unless required by applicable law or agreed to in writing, software
1162// distributed under the License is distributed on an "AS IS" BASIS,
1163// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1164// See the License for the specific language governing permissions and
1165// limitations under the License.
1166// ----------------------------- END-OF-FILE ----------------------------------
1167
1168/** @} */
1169/** @} */
1170/** @} */
Definition bslstl_stringview.h:471
Definition bslstl_string.h:1252
#define BSLS_ASSERT(X)
Definition bsls_assert.h:1976
#define BSLS_IDENT(str)
BSLS_IDENT() - insert string into .comment binary segment (if supported)
Definition bsls_ident.h:238
Definition bdlde_base64alphabet.h:118
Definition bdlde_utf8util.h:945
static size_type advancePastValidOrInvalidCodePoint(const bsl::string_view &input)
static int getLineAndColumnNumber(Uint64 *lineNumber, Uint64 *utf8Column, Uint64 *startOfLineByteOffset, bsl::streambuf *input, Uint64 byteOffset, char lineDelimeter, char *temporaryReadBuffer, int temporaryReadBufferNumBytes)
bsls::Types::size_type size_type
Definition bdlde_utf8util.h:949
bsls::Types::Uint64 Uint64
Definition bdlde_utf8util.h:948
Definition bdlde_utf8util.h:407
static size_type readIfValid(int *status, char *outputBuffer, size_type outputBufferLength, bsl::streambuf *input)
static IntPtr numCodePointsIfValid(const char **invalidString, const char *string)
static IntPtr advanceRaw(const char **result, const char *string, size_type length, IntPtr numCodePoints)
bsls::Types::UintPtr UintPtr
Definition bdlde_utf8util.h:412
bsls::Types::Uint64 Uint64
Definition bdlde_utf8util.h:413
static const char * toAscii(IntPtr value)
static size_type replaceErrors(bsl::string *output, const bsl::string_view &input, unsigned int utf32ErrorCode=k_ERROR_CODE_POINT)
static IntPtr advanceIfValid(int *status, const char **result, const char *string, size_type length, IntPtr numCodePoints)
static IntPtr numCharactersRaw(const char *string)
Definition bdlde_utf8util.h:1129
static int numBytesInCodePoint(const char *codePoint)
static bool isValid(const char *string)
Definition bdlde_utf8util.h:1069
static int codePointValue(const char *codePoint)
static int appendUtf8CodePoint(bsl::string *output, unsigned int codePoint)
@ k_ERROR_CODE_POINT
Definition bdlde_utf8util.h:416
static size_type replaceErrors(std::string *output, const bsl::string_view &input, unsigned int utf32ErrorCode=k_ERROR_CODE_POINT)
static IntPtr numCodePointsRaw(const char *string, size_type length)
static IntPtr numCharactersIfValid(const char **invalidString, const char *string)
Definition bdlde_utf8util.h:1114
static IntPtr advanceRaw(const char **result, const char *string, IntPtr numCodePoints)
static bool isValid(const char **invalidString, const char *string, size_type length)
static IntPtr numCodePointsRaw(const char *string)
bsls::Types::IntPtr IntPtr
Definition bdlde_utf8util.h:411
static bool isValid(const char **invalidString, const char *string)
static bool isValidCodePoint(int *status, const char *codePoint, size_type numBytes)
static int appendUtf8Character(bsl::string *output, unsigned int codePoint)
Definition bdlde_utf8util.h:1021
static IntPtr numBytesIfValid(const bsl::string_view &string, IntPtr numCodePoints)
Definition bdlde_utf8util.h:1094
static bool isValid(const char **invalidString, const bsl::string_view &string)
static IntPtr numBytesRaw(const bsl::string_view &string, IntPtr numCodePoints)
static IntPtr numCodePointsIfValid(const char **invalidString, const bsl::string_view &string)
static int getLineAndColumnNumber(Uint64 *lineNumber, Uint64 *utf8Column, Uint64 *startOfLineByteOffset, bsl::streambuf *input, Uint64 byteOffset)
Definition bdlde_utf8util.h:1034
static IntPtr advanceIfValid(int *status, const char **result, const char *string, IntPtr numCodePoints)
static int getByteSize(const char *codePoint)
Definition bdlde_utf8util.h:1028
static IntPtr numCodePointsIfValid(const char **invalidString, const char *string, size_type length)
static int appendUtf8CodePoint(std::string *output, unsigned int codePoint)
ErrorStatus
Definition bdlde_utf8util.h:425
@ k_UNEXPECTED_CONTINUATION_OCTET
Definition bdlde_utf8util.h:431
@ k_NON_CONTINUATION_OCTET
Definition bdlde_utf8util.h:435
@ k_OVERLONG_ENCODING
Definition bdlde_utf8util.h:439
@ k_SURROGATE
Definition bdlde_utf8util.h:450
@ k_VALUE_LARGER_THAN_0X10FFFF
Definition bdlde_utf8util.h:447
@ k_END_OF_INPUT_TRUNCATION
Definition bdlde_utf8util.h:427
@ k_INVALID_INITIAL_OCTET
Definition bdlde_utf8util.h:443
static IntPtr numCharacters(const char *string)
Definition bdlde_utf8util.h:1102
bsls::Types::size_type size_type
Definition bdlde_utf8util.h:410
std::size_t UintPtr
Definition bsls_types.h:128
std::size_t size_type
Definition bsls_types.h:126
unsigned long long Uint64
Definition bsls_types.h:139
std::ptrdiff_t IntPtr
Definition bsls_types.h:132