c8rtomb.c revision 1.1
1 /* $NetBSD: c8rtomb.c,v 1.1 2024/08/15 21:19:45 riastradh Exp $ */ 2 3 /*- 4 * Copyright (c) 2024 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 17 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 18 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 20 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26 * POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 /* 30 * c8rtomb(s, c8, ps) 31 * 32 * Encode the Unicode UTF-8 code unit c8 into the multibyte buffer 33 * s under the current locale, using multibyte encoding state ps. 34 * 35 * If c8 is not the last byte of a UTF-8 scalar value sequence, no 36 * output will be produced, but c8 will be remembered; this must 37 * be followed by another call passing the following bytes. 38 * 39 * Return the number of bytes stored on success, or (size_t)-1 on 40 * error with errno set to EILSEQ. 41 * 42 * At most MB_CUR_MAX bytes will be stored. 43 * 44 * References: 45 * 46 * The Unicode Standard, Version 15.0 -- Core Specification, The 47 * Unicode Consortium, Sec. 3.9 `Unicode Encoding Forms': UTF-8, 48 * p. 124. 49 * https://www.unicode.org/versions/Unicode15.0.0/UnicodeStandard-15.0.pdf#page=150 50 * https://web.archive.org/web/20240718101254/https://www.unicode.org/versions/Unicode15.0.0/UnicodeStandard-15.0.pdf#page=150 51 * 52 * F. Yergeau, `UTF-8, a transformation format of ISO 10646', 53 * RFC 3629, Internet Engineering Task Force, November 2003. 54 * https://datatracker.ietf.org/doc/html/rfc3629 55 */ 56 57 #include <sys/cdefs.h> 58 __RCSID("$NetBSD: c8rtomb.c,v 1.1 2024/08/15 21:19:45 riastradh Exp $"); 59 60 #include <assert.h> 61 #include <errno.h> 62 #include <limits.h> 63 #include <stdalign.h> 64 #include <stddef.h> 65 #include <stdint.h> 66 #include <uchar.h> 67 68 #include "c32rtomb.h" 69 70 struct c8rtombstate { 71 char32_t state_c32; /* 8-bit state and 24-bit buffer */ 72 mbstate_t mbs; 73 }; 74 __CTASSERT(offsetof(struct c8rtombstate, mbs) <= sizeof(mbstate_t)); 75 __CTASSERT(sizeof(struct c32rtombstate) <= sizeof(mbstate_t) - 76 offsetof(struct c8rtombstate, mbs)); 77 __CTASSERT(alignof(struct c8rtombstate) <= alignof(mbstate_t)); 78 79 /* 80 * UTF-8 validation, inspired by Bjoern Hoermann's UTF-8 decoder at 81 * <http://bjoern.hoehrmann.de/utf-8/decoder/dfa/>, but reimplemented 82 * from scratch. 83 */ 84 85 #define UTF8_ACCEPT 0 86 #define UTF8_REJECT 96 87 88 typedef uint_fast8_t utf8_class_t; 89 typedef uint_fast8_t utf8_state_t; 90 91 static uint8_t utf8_classtab[] = { 92 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 93 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 94 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 95 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 96 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, 97 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 98 8,8,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 99 11,3,3,3,3,3,3,3,3,3,3,3,3,4,3,3, 7,6,6,6,5,8,8,8,8,8,8,8,8,8,8,8, 100 }; 101 102 static uint8_t utf8_statetab[] = { 103 0,96,12,36,48,84,72,60,96,96,96,24, 96, 0,96,96,96,96,96,96, 0, 0,96,96, 104 96,12,96,96,96,96,96,96,96,96,96,96, 96,12,96,96,96,96,96,96,12,12,96,96, 105 96,96,96,96,96,96,96,96,12,12,96,96, 96,36,96,96,96,96,96,96,96,36,96,96, 106 96,36,96,96,96,96,96,96,36,36,96,96, 96,96,96,96,96,96,96,96,36,96,96,96, 107 96,96,96,96,96,96,96,96,96,96,96,96, 108 }; 109 110 static utf8_state_t 111 utf8_decode_step(utf8_state_t state, char8_t c8, char32_t *pc32) 112 { 113 const utf8_class_t class = utf8_classtab[c8]; 114 115 *pc32 = (state == UTF8_ACCEPT 116 ? (c8 & (0xff >> class)) 117 : ((c8 & 0x3f) | (*pc32 << 6))); 118 119 return utf8_statetab[state + class]; 120 } 121 122 size_t 123 c8rtomb(char *restrict s, char8_t c8, mbstate_t *restrict ps) 124 { 125 static mbstate_t psbuf; 126 char buf[MB_LEN_MAX]; 127 struct c8rtombstate *S; 128 utf8_state_t state; 129 char32_t c32; 130 131 /* 132 * `If ps is a null pointer, each function uses its own 133 * internal mbstate_t object instead, which is initialized at 134 * program startup to the initial conversion state; the 135 * functions are not required to avoid data races with other 136 * calls to the same function in this case. The 137 * implementation behaves as if no library function calls 138 * these functions with a null pointer for ps.' 139 */ 140 if (ps == NULL) 141 ps = &psbuf; 142 143 /* 144 * `If s is a null pointer, the c8rtomb function is equivalent 145 * to the call 146 * 147 * c8rtomb(buf, u8'\0', ps) 148 * 149 * where buf is an internal buffer. 150 */ 151 if (s == NULL) { 152 s = buf; 153 c8 = 0; /* XXX u8'\0' */ 154 } 155 156 /* 157 * Open the private UTF-8 decoding state. 158 */ 159 S = (struct c8rtombstate *)ps; 160 161 #if 0 162 /* 163 * `If c8 is a null character, a null byte is stored, preceded 164 * by any shift sequence needed to restore the initial shift 165 * state; the resulting state described is the initial 166 * conversion state.' 167 * 168 * XXX But what else gets stored? Do we just discard any 169 * pending sequence, or do we convert it to something else, or 170 * what? 171 */ 172 if (c8 == u8'\0') { 173 memset(S->buf, 0, sizeof(S->buf)); 174 S->n = 0; 175 } 176 #endif 177 178 /* 179 * Get the current state and buffer. 180 */ 181 __CTASSERT(UTF8_ACCEPT == 0); /* initial conversion state */ 182 state = __SHIFTOUT(S->state_c32, __BITS(31,24)); 183 c32 = __SHIFTOUT(S->state_c32, __BITS(23,0)); 184 185 /* 186 * Feed the byte into the state machine to update the state. 187 */ 188 state = utf8_decode_step(state, c8, &c32); 189 switch (state) { 190 case UTF8_REJECT: 191 /* 192 * Invalid UTF-8. Fail with EILSEQ. 193 */ 194 errno = EILSEQ; 195 return (size_t)-1; 196 default: 197 /* 198 * Valid UTF-8 so far but incomplete. Update state and 199 * output nothing. 200 */ 201 S->state_c32 = __SHIFTIN(state, __BITS(31,24)) | 202 __SHIFTIN(c32, __BITS(23,0)); 203 return 0; 204 case UTF8_ACCEPT: 205 /* 206 * We have a scalar value. Clear the state and output 207 * the scalar value. 208 */ 209 __CTASSERT(UTF8_ACCEPT == 0); 210 S->state_c32 = 0; 211 return c32rtomb(s, c32, &S->mbs); 212 } 213 } 214
Indexes created Thu Oct 30 03:10:01 GMT 2025