1 /* $NetBSD: c8rtomb.c,v 1.9 2024/10/12 16:44:44 rillig Exp $ */ 2 3 /*- 4 * Copyright (c) 2024 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 17 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 18 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 20 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26 * POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 /* 30 * c8rtomb(s, c8, ps) 31 * 32 * Encode the Unicode UTF-8 code unit c8 into the multibyte buffer 33 * s under the current locale, using multibyte encoding state ps. 34 * 35 * If c8 is not the last byte of a UTF-8 scalar value sequence, no 36 * output will be produced, but c8 will be remembered; this must 37 * be followed by another call passing the following bytes. 38 * 39 * Return the number of bytes stored on success, or (size_t)-1 on 40 * error with errno set to EILSEQ. 41 * 42 * At most MB_CUR_MAX bytes will be stored. 43 * 44 * References: 45 * 46 * The Unicode Standard, Version 15.0 -- Core Specification, The 47 * Unicode Consortium, Sec. 3.9 `Unicode Encoding Forms': UTF-8, 48 * p. 124. 49 * https://www.unicode.org/versions/Unicode15.0.0/UnicodeStandard-15.0.pdf#page=150 50 * https://web.archive.org/web/20240718101254/https://www.unicode.org/versions/Unicode15.0.0/UnicodeStandard-15.0.pdf#page=150 51 * 52 * F. Yergeau, `UTF-8, a transformation format of ISO 10646', 53 * RFC 3629, Internet Engineering Task Force, November 2003. 54 * https://datatracker.ietf.org/doc/html/rfc3629 55 */ 56 57 #include <sys/cdefs.h> 58 __RCSID("$NetBSD: c8rtomb.c,v 1.9 2024/10/12 16:44:44 rillig Exp $"); 59 60 #include "namespace.h" 61 62 #include <assert.h> 63 #include <errno.h> 64 #include <limits.h> 65 #include <locale.h> 66 #include <stdalign.h> 67 #include <stddef.h> 68 #include <stdint.h> 69 #include <uchar.h> 70 71 #include "c32rtomb.h" 72 #include "setlocale_local.h" 73 74 struct c8rtombstate { 75 char32_t state_c32; /* 8-bit state and 24-bit buffer */ 76 mbstate_t mbs; 77 }; 78 __CTASSERT(offsetof(struct c8rtombstate, mbs) <= sizeof(mbstate_t)); 79 __CTASSERT(sizeof(struct c32rtombstate) <= sizeof(mbstate_t) - 80 offsetof(struct c8rtombstate, mbs)); 81 __CTASSERT(alignof(struct c8rtombstate) <= alignof(mbstate_t)); 82 83 /* 84 * UTF-8 validation, inspired by Bjoern Hoermann's UTF-8 decoder at 85 * <http://bjoern.hoehrmann.de/utf-8/decoder/dfa/>, but reimplemented 86 * from scratch. 87 */ 88 89 #define UTF8_ACCEPT 0 90 #define UTF8_REJECT 96 91 92 typedef uint8_t utf8_class_t; 93 typedef uint8_t utf8_state_t; 94 95 static const uint8_t utf8_classtab[] = { 96 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 97 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 98 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 99 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 100 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, 101 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 102 8,8,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 103 11,3,3,3,3,3,3,3,3,3,3,3,3,4,3,3, 7,6,6,6,5,8,8,8,8,8,8,8,8,8,8,8, 104 }; 105 106 static const uint8_t utf8_statetab[] = { 107 0,96,12,36,48,84,72,60,96,96,96,24, 96, 0,96,96,96,96,96,96, 0, 0,96,96, 108 96,12,96,96,96,96,96,96,96,96,96,96, 96,12,96,96,96,96,96,96,12,12,96,96, 109 96,96,96,96,96,96,96,96,12,12,96,96, 96,36,96,96,96,96,96,96,96,36,96,96, 110 96,36,96,96,96,96,96,96,36,36,96,96, 96,96,96,96,96,96,96,96,36,96,96,96, 111 96,96,96,96,96,96,96,96,96,96,96,96, 112 }; 113 114 static utf8_state_t 115 utf8_decode_step(utf8_state_t state, char8_t c8, char32_t *pc32) 116 { 117 const utf8_class_t class = utf8_classtab[c8]; 118 119 *pc32 = (state == UTF8_ACCEPT 120 ? (c8 & (0xff >> class)) 121 : ((c8 & 0x3f) | (*pc32 << 6))); 122 123 return utf8_statetab[state + class]; 124 } 125 126 #ifdef __weak_alias 127 __weak_alias(c8rtomb_l,_c8rtomb_l) 128 #endif 129 130 size_t 131 c8rtomb(char *restrict s, char8_t c8, mbstate_t *restrict ps) 132 { 133 134 return c8rtomb_l(s, c8, ps, _current_locale()); 135 } 136 137 size_t 138 c8rtomb_l(char *restrict s, char8_t c8, mbstate_t *restrict ps, locale_t loc) 139 { 140 static mbstate_t psbuf; 141 char buf[MB_LEN_MAX]; 142 struct c8rtombstate *S; 143 utf8_state_t state; 144 char32_t c32; 145 146 /* 147 * `If ps is a null pointer, each function uses its own 148 * internal mbstate_t object instead, which is initialized at 149 * program startup to the initial conversion state; the 150 * functions are not required to avoid data races with other 151 * calls to the same function in this case. The 152 * implementation behaves as if no library function calls 153 * these functions with a null pointer for ps.' 154 */ 155 if (ps == NULL) 156 ps = &psbuf; 157 158 /* 159 * `If s is a null pointer, the c8rtomb function is equivalent 160 * to the call 161 * 162 * c8rtomb(buf, u8'\0', ps) 163 * 164 * where buf is an internal buffer. 165 */ 166 if (s == NULL) { 167 s = buf; 168 c8 = 0; /* XXX u8'\0' */ 169 } 170 171 /* 172 * Open the private UTF-8 decoding state. 173 */ 174 S = (struct c8rtombstate *)(void *)ps; 175 176 /* 177 * `If c8 is a null character, a null byte is stored, preceded 178 * by any shift sequence needed to restore the initial shift 179 * state; the resulting state described is the initial 180 * conversion state.' 181 * 182 * So if c8 is null, discard any buffered input -- there's 183 * nothing we can legitimately do with it -- and convert a null 184 * scalar value, which by definition of c32rtomb writes out any 185 * shift sequence reset followed by a null byte. 186 */ 187 if (c8 == '\0') { 188 c32 = 0; 189 goto accept; 190 } 191 192 /* 193 * Get the current state and buffer. 194 */ 195 __CTASSERT(UTF8_ACCEPT == 0); /* initial conversion state */ 196 state = __SHIFTOUT(S->state_c32, __BITS(31,24)); 197 c32 = __SHIFTOUT(S->state_c32, __BITS(23,0)); 198 199 /* 200 * Feed the byte into the state machine to update the state. 201 */ 202 state = utf8_decode_step(state, c8, &c32); 203 switch (state) { 204 case UTF8_REJECT: 205 /* 206 * Invalid UTF-8. Fail with EILSEQ. 207 */ 208 errno = EILSEQ; 209 return (size_t)-1; 210 default: 211 /* 212 * Valid UTF-8 so far but incomplete. Update state and 213 * output nothing. 214 */ 215 S->state_c32 = 216 __SHIFTIN(state, __BITS(31,24)) | 217 __SHIFTIN(c32, __BITS(23,0)); 218 return 0; 219 case UTF8_ACCEPT: 220 accept: 221 /* 222 * We have a scalar value. Clear the state and output 223 * the scalar value. 224 */ 225 S->state_c32 = 0; 226 return c32rtomb_l(s, c32, &S->mbs, loc); 227 } 228 } 229
Indexes created Tue Oct 28 12:10:06 GMT 2025