c8rtomb.c revision 1.2
1 /* $NetBSD: c8rtomb.c,v 1.2 2024/08/15 22:23:17 riastradh Exp $ */ 2 3 /*- 4 * Copyright (c) 2024 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 17 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 18 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 20 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26 * POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 /* 30 * c8rtomb(s, c8, ps) 31 * 32 * Encode the Unicode UTF-8 code unit c8 into the multibyte buffer 33 * s under the current locale, using multibyte encoding state ps. 34 * 35 * If c8 is not the last byte of a UTF-8 scalar value sequence, no 36 * output will be produced, but c8 will be remembered; this must 37 * be followed by another call passing the following bytes. 38 * 39 * Return the number of bytes stored on success, or (size_t)-1 on 40 * error with errno set to EILSEQ. 41 * 42 * At most MB_CUR_MAX bytes will be stored. 43 * 44 * References: 45 * 46 * The Unicode Standard, Version 15.0 -- Core Specification, The 47 * Unicode Consortium, Sec. 3.9 `Unicode Encoding Forms': UTF-8, 48 * p. 124. 49 * https://www.unicode.org/versions/Unicode15.0.0/UnicodeStandard-15.0.pdf#page=150 50 * https://web.archive.org/web/20240718101254/https://www.unicode.org/versions/Unicode15.0.0/UnicodeStandard-15.0.pdf#page=150 51 * 52 * F. Yergeau, `UTF-8, a transformation format of ISO 10646', 53 * RFC 3629, Internet Engineering Task Force, November 2003. 54 * https://datatracker.ietf.org/doc/html/rfc3629 55 */ 56 57 #include <sys/cdefs.h> 58 __RCSID("$NetBSD: c8rtomb.c,v 1.2 2024/08/15 22:23:17 riastradh Exp $"); 59 60 #include "namespace.h" 61 62 #include <assert.h> 63 #include <errno.h> 64 #include <limits.h> 65 #include <stdalign.h> 66 #include <stddef.h> 67 #include <stdint.h> 68 #include <uchar.h> 69 70 #include "c32rtomb.h" 71 72 struct c8rtombstate { 73 char32_t state_c32; /* 8-bit state and 24-bit buffer */ 74 mbstate_t mbs; 75 }; 76 __CTASSERT(offsetof(struct c8rtombstate, mbs) <= sizeof(mbstate_t)); 77 __CTASSERT(sizeof(struct c32rtombstate) <= sizeof(mbstate_t) - 78 offsetof(struct c8rtombstate, mbs)); 79 __CTASSERT(alignof(struct c8rtombstate) <= alignof(mbstate_t)); 80 81 /* 82 * UTF-8 validation, inspired by Bjoern Hoermann's UTF-8 decoder at 83 * <http://bjoern.hoehrmann.de/utf-8/decoder/dfa/>, but reimplemented 84 * from scratch. 85 */ 86 87 #define UTF8_ACCEPT 0 88 #define UTF8_REJECT 96 89 90 typedef uint_fast8_t utf8_class_t; 91 typedef uint_fast8_t utf8_state_t; 92 93 static uint8_t utf8_classtab[] = { 94 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 95 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 96 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 97 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 98 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, 99 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 100 8,8,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 101 11,3,3,3,3,3,3,3,3,3,3,3,3,4,3,3, 7,6,6,6,5,8,8,8,8,8,8,8,8,8,8,8, 102 }; 103 104 static uint8_t utf8_statetab[] = { 105 0,96,12,36,48,84,72,60,96,96,96,24, 96, 0,96,96,96,96,96,96, 0, 0,96,96, 106 96,12,96,96,96,96,96,96,96,96,96,96, 96,12,96,96,96,96,96,96,12,12,96,96, 107 96,96,96,96,96,96,96,96,12,12,96,96, 96,36,96,96,96,96,96,96,96,36,96,96, 108 96,36,96,96,96,96,96,96,36,36,96,96, 96,96,96,96,96,96,96,96,36,96,96,96, 109 96,96,96,96,96,96,96,96,96,96,96,96, 110 }; 111 112 static utf8_state_t 113 utf8_decode_step(utf8_state_t state, char8_t c8, char32_t *pc32) 114 { 115 const utf8_class_t class = utf8_classtab[c8]; 116 117 *pc32 = (state == UTF8_ACCEPT 118 ? (c8 & (0xff >> class)) 119 : ((c8 & 0x3f) | (*pc32 << 6))); 120 121 return utf8_statetab[state + class]; 122 } 123 124 size_t 125 c8rtomb(char *restrict s, char8_t c8, mbstate_t *restrict ps) 126 { 127 static mbstate_t psbuf; 128 char buf[MB_LEN_MAX]; 129 struct c8rtombstate *S; 130 utf8_state_t state; 131 char32_t c32; 132 133 /* 134 * `If ps is a null pointer, each function uses its own 135 * internal mbstate_t object instead, which is initialized at 136 * program startup to the initial conversion state; the 137 * functions are not required to avoid data races with other 138 * calls to the same function in this case. The 139 * implementation behaves as if no library function calls 140 * these functions with a null pointer for ps.' 141 */ 142 if (ps == NULL) 143 ps = &psbuf; 144 145 /* 146 * `If s is a null pointer, the c8rtomb function is equivalent 147 * to the call 148 * 149 * c8rtomb(buf, u8'\0', ps) 150 * 151 * where buf is an internal buffer. 152 */ 153 if (s == NULL) { 154 s = buf; 155 c8 = 0; /* XXX u8'\0' */ 156 } 157 158 /* 159 * Open the private UTF-8 decoding state. 160 */ 161 S = (struct c8rtombstate *)ps; 162 163 #if 0 164 /* 165 * `If c8 is a null character, a null byte is stored, preceded 166 * by any shift sequence needed to restore the initial shift 167 * state; the resulting state described is the initial 168 * conversion state.' 169 * 170 * XXX But what else gets stored? Do we just discard any 171 * pending sequence, or do we convert it to something else, or 172 * what? 173 */ 174 if (c8 == u8'\0') { 175 memset(S->buf, 0, sizeof(S->buf)); 176 S->n = 0; 177 } 178 #endif 179 180 /* 181 * Get the current state and buffer. 182 */ 183 __CTASSERT(UTF8_ACCEPT == 0); /* initial conversion state */ 184 state = __SHIFTOUT(S->state_c32, __BITS(31,24)); 185 c32 = __SHIFTOUT(S->state_c32, __BITS(23,0)); 186 187 /* 188 * Feed the byte into the state machine to update the state. 189 */ 190 state = utf8_decode_step(state, c8, &c32); 191 switch (state) { 192 case UTF8_REJECT: 193 /* 194 * Invalid UTF-8. Fail with EILSEQ. 195 */ 196 errno = EILSEQ; 197 return (size_t)-1; 198 default: 199 /* 200 * Valid UTF-8 so far but incomplete. Update state and 201 * output nothing. 202 */ 203 S->state_c32 = __SHIFTIN(state, __BITS(31,24)) | 204 __SHIFTIN(c32, __BITS(23,0)); 205 return 0; 206 case UTF8_ACCEPT: 207 /* 208 * We have a scalar value. Clear the state and output 209 * the scalar value. 210 */ 211 __CTASSERT(UTF8_ACCEPT == 0); 212 S->state_c32 = 0; 213 return c32rtomb(s, c32, &S->mbs); 214 } 215 } 216
Indexes created Fri Oct 31 11:10:07 GMT 2025