ReactOS  0.4.14-dev-115-g4576127
utf8.c
Go to the documentation of this file.
1 /*
2  * UTF-8 support routines
3  *
4  * Copyright 2000 Alexandre Julliard
5  *
6  * This library is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * This library is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with this library; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA
19  */
20 
21 #include <string.h>
22 
23 #include "wine/unicode.h"
24 
25 extern WCHAR wine_compose( const WCHAR *str ) DECLSPEC_HIDDEN;
26 
27 /* number of following bytes in sequence based on first byte value (for bytes above 0x7f) */
28 static const char utf8_length[128] =
29 {
30  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, /* 0x80-0x8f */
31  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, /* 0x90-0x9f */
32  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, /* 0xa0-0xaf */
33  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, /* 0xb0-0xbf */
34  0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1, /* 0xc0-0xcf */
35  1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, /* 0xd0-0xdf */
36  2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, /* 0xe0-0xef */
37  3,3,3,3,3,0,0,0,0,0,0,0,0,0,0,0 /* 0xf0-0xff */
38 };
39 
40 /* first byte mask depending on UTF-8 sequence length */
41 static const unsigned char utf8_mask[4] = { 0x7f, 0x1f, 0x0f, 0x07 };
42 
43 /* minimum Unicode value depending on UTF-8 sequence length */
44 static const unsigned int utf8_minval[4] = { 0x0, 0x80, 0x800, 0x10000 };
45 
46 
47 /* get the next char value taking surrogates into account */
48 static inline unsigned int get_surrogate_value( const WCHAR *src, unsigned int srclen )
49 {
50  if (src[0] >= 0xd800 && src[0] <= 0xdfff) /* surrogate pair */
51  {
52  if (src[0] > 0xdbff || /* invalid high surrogate */
53  srclen <= 1 || /* missing low surrogate */
54  src[1] < 0xdc00 || src[1] > 0xdfff) /* invalid low surrogate */
55  return 0;
56  return 0x10000 + ((src[0] & 0x3ff) << 10) + (src[1] & 0x3ff);
57  }
58  return src[0];
59 }
60 
61 /* query necessary dst length for src string */
62 static inline int get_length_wcs_utf8( int flags, const WCHAR *src, unsigned int srclen )
63 {
64  int len;
65  unsigned int val;
66 
67  for (len = 0; srclen; srclen--, src++)
68  {
69  if (*src < 0x80) /* 0x00-0x7f: 1 byte */
70  {
71  len++;
72  continue;
73  }
74  if (*src < 0x800) /* 0x80-0x7ff: 2 bytes */
75  {
76  len += 2;
77  continue;
78  }
79  if (!(val = get_surrogate_value( src, srclen )))
80  {
81  if (flags & WC_ERR_INVALID_CHARS) return -2;
82  continue;
83  }
84  if (val < 0x10000) /* 0x800-0xffff: 3 bytes */
85  len += 3;
86  else /* 0x10000-0x10ffff: 4 bytes */
87  {
88  len += 4;
89  src++;
90  srclen--;
91  }
92  }
93  return len;
94 }
95 
96 /* wide char to UTF-8 string conversion */
97 /* return -1 on dst buffer overflow, -2 on invalid input char */
98 int wine_utf8_wcstombs( int flags, const WCHAR *src, int srclen, char *dst, int dstlen )
99 {
100  int len;
101 
102  if (!dstlen) return get_length_wcs_utf8( flags, src, srclen );
103 
104  for (len = dstlen; srclen; srclen--, src++)
105  {
106  WCHAR ch = *src;
107  unsigned int val;
108 
109  if (ch < 0x80) /* 0x00-0x7f: 1 byte */
110  {
111  if (!len--) return -1; /* overflow */
112  *dst++ = ch;
113  continue;
114  }
115 
116  if (ch < 0x800) /* 0x80-0x7ff: 2 bytes */
117  {
118  if ((len -= 2) < 0) return -1; /* overflow */
119  dst[1] = 0x80 | (ch & 0x3f);
120  ch >>= 6;
121  dst[0] = 0xc0 | ch;
122  dst += 2;
123  continue;
124  }
125 
126  if (!(val = get_surrogate_value( src, srclen )))
127  {
128  if (flags & WC_ERR_INVALID_CHARS) return -2;
129  continue;
130  }
131 
132  if (val < 0x10000) /* 0x800-0xffff: 3 bytes */
133  {
134  if ((len -= 3) < 0) return -1; /* overflow */
135  dst[2] = 0x80 | (val & 0x3f);
136  val >>= 6;
137  dst[1] = 0x80 | (val & 0x3f);
138  val >>= 6;
139  dst[0] = 0xe0 | val;
140  dst += 3;
141  }
142  else /* 0x10000-0x10ffff: 4 bytes */
143  {
144  if ((len -= 4) < 0) return -1; /* overflow */
145  dst[3] = 0x80 | (val & 0x3f);
146  val >>= 6;
147  dst[2] = 0x80 | (val & 0x3f);
148  val >>= 6;
149  dst[1] = 0x80 | (val & 0x3f);
150  val >>= 6;
151  dst[0] = 0xf0 | val;
152  dst += 4;
153  src++;
154  srclen--;
155  }
156  }
157  return dstlen - len;
158 }
159 
160 /* helper for the various utf8 mbstowcs functions */
161 static inline unsigned int decode_utf8_char( unsigned char ch, const char **str, const char *strend )
162 {
163  unsigned int len = utf8_length[ch-0x80];
164  unsigned int res = ch & utf8_mask[len];
165  const char *end = *str + len;
166 
167  if (end > strend) return ~0;
168  switch(len)
169  {
170  case 3:
171  if ((ch = end[-3] ^ 0x80) >= 0x40) break;
172  res = (res << 6) | ch;
173  (*str)++;
174  case 2:
175  if ((ch = end[-2] ^ 0x80) >= 0x40) break;
176  res = (res << 6) | ch;
177  (*str)++;
178  case 1:
179  if ((ch = end[-1] ^ 0x80) >= 0x40) break;
180  res = (res << 6) | ch;
181  (*str)++;
182  if (res < utf8_minval[len]) break;
183  return res;
184  }
185  return ~0;
186 }
187 
188 /* query necessary dst length for src string with composition */
189 static inline int get_length_mbs_utf8_compose( int flags, const char *src, int srclen )
190 {
191  int ret = 0;
192  unsigned int res;
193  WCHAR composed[2];
194  const char *srcend = src + srclen;
195 
196  composed[0] = 0;
197  while (src < srcend)
198  {
199  unsigned char ch = *src++;
200  if (ch < 0x80) /* special fast case for 7-bit ASCII */
201  {
202  composed[0] = ch;
203  ret++;
204  continue;
205  }
206  if ((res = decode_utf8_char( ch, &src, srcend )) <= 0xffff)
207  {
208  if (composed[0])
209  {
210  composed[1] = res;
211  if ((composed[0] = wine_compose( composed ))) continue;
212  }
213  composed[0] = res;
214  ret++;
215  }
216  else if (res <= 0x10ffff)
217  {
218  ret += 2;
219  composed[0] = 0; /* no composition for surrogates */
220  }
221  else if (flags & MB_ERR_INVALID_CHARS) return -2; /* bad char */
222  /* otherwise ignore it */
223  }
224  return ret;
225 }
226 
227 /* UTF-8 to wide char string conversion with composition */
228 /* return -1 on dst buffer overflow, -2 on invalid input char */
229 static int utf8_mbstowcs_compose( int flags, const char *src, int srclen, WCHAR *dst, int dstlen )
230 {
231  unsigned int res;
232  const char *srcend = src + srclen;
233  WCHAR composed[2];
234  WCHAR *dstend = dst + dstlen;
235 
237 
238  composed[0] = 0;
239  while (src < srcend)
240  {
241  unsigned char ch = *src++;
242  if (ch < 0x80) /* special fast case for 7-bit ASCII */
243  {
244  if (dst >= dstend) return -1; /* overflow */
245  *dst++ = composed[0] = ch;
246  continue;
247  }
248  if ((res = decode_utf8_char( ch, &src, srcend )) <= 0xffff)
249  {
250  if (composed[0])
251  {
252  composed[1] = res;
253  if ((composed[0] = wine_compose( composed )))
254  {
255  dst[-1] = composed[0];
256  continue;
257  }
258  }
259  if (dst >= dstend) return -1; /* overflow */
260  *dst++ = composed[0] = res;
261  }
262  else if (res <= 0x10ffff) /* we need surrogates */
263  {
264  if (dst >= dstend - 1) return -1; /* overflow */
265  res -= 0x10000;
266  *dst++ = 0xd800 | (res >> 10);
267  *dst++ = 0xdc00 | (res & 0x3ff);
268  composed[0] = 0; /* no composition for surrogates */
269  }
270  else if (flags & MB_ERR_INVALID_CHARS) return -2; /* bad char */
271  /* otherwise ignore it */
272  }
273  return dstlen - (dstend - dst);
274 }
275 
276 /* query necessary dst length for src string */
277 static inline int get_length_mbs_utf8( int flags, const char *src, int srclen )
278 {
279  int ret = 0;
280  unsigned int res;
281  const char *srcend = src + srclen;
282 
283  while (src < srcend)
284  {
285  unsigned char ch = *src++;
286  if (ch < 0x80) /* special fast case for 7-bit ASCII */
287  {
288  ret++;
289  continue;
290  }
291  if ((res = decode_utf8_char( ch, &src, srcend )) <= 0x10ffff)
292  {
293  if (res > 0xffff) ret++;
294  ret++;
295  }
296  else if (flags & MB_ERR_INVALID_CHARS) return -2; /* bad char */
297  /* otherwise ignore it */
298  }
299  return ret;
300 }
301 
302 /* UTF-8 to wide char string conversion */
303 /* return -1 on dst buffer overflow, -2 on invalid input char */
304 int wine_utf8_mbstowcs( int flags, const char *src, int srclen, WCHAR *dst, int dstlen )
305 {
306  unsigned int res;
307  const char *srcend = src + srclen;
308  WCHAR *dstend = dst + dstlen;
309 
311 
312  if (!dstlen) return get_length_mbs_utf8( flags, src, srclen );
313 
314  while ((dst < dstend) && (src < srcend))
315  {
316  unsigned char ch = *src++;
317  if (ch < 0x80) /* special fast case for 7-bit ASCII */
318  {
319  *dst++ = ch;
320  continue;
321  }
322  if ((res = decode_utf8_char( ch, &src, srcend )) <= 0xffff)
323  {
324  *dst++ = res;
325  }
326  else if (res <= 0x10ffff) /* we need surrogates */
327  {
328  if (dst == dstend - 1) return -1; /* overflow */
329  res -= 0x10000;
330  *dst++ = 0xd800 | (res >> 10);
331  *dst++ = 0xdc00 | (res & 0x3ff);
332  }
333  else if (flags & MB_ERR_INVALID_CHARS) return -2; /* bad char */
334  /* otherwise ignore it */
335  }
336  if (src < srcend) return -1; /* overflow */
337  return dstlen - (dstend - dst);
338 }
#define MB_ERR_INVALID_CHARS
Definition: unicode.h:41
WCHAR wine_compose(const WCHAR *str) DECLSPEC_HIDDEN
Definition: compose.c:395
static unsigned int get_surrogate_value(const WCHAR *src, unsigned int srclen)
Definition: utf8.c:48
#define MB_COMPOSITE
Definition: unicode.h:40
GLuint GLuint end
Definition: gl.h:1545
static const unsigned char utf8_mask[4]
Definition: utf8.c:41
static const char utf8_length[128]
Definition: utf8.c:28
static int get_length_mbs_utf8(int flags, const char *src, int srclen)
Definition: utf8.c:277
const WCHAR * str
int wine_utf8_mbstowcs(int flags, const char *src, int srclen, WCHAR *dst, int dstlen)
Definition: utf8.c:304
GLuint GLfloat * val
Definition: glext.h:7180
static int get_length_wcs_utf8(int flags, const WCHAR *src, unsigned int srclen)
Definition: utf8.c:62
__wchar_t WCHAR
Definition: xmlstorage.h:180
static DWORD LPDWORD LPCSTR DWORD srclen
Definition: directory.c:51
#define WC_ERR_INVALID_CHARS
Definition: unicode.h:47
static int get_length_mbs_utf8_compose(int flags, const char *src, int srclen)
Definition: utf8.c:189
GLbitfield flags
Definition: glext.h:7161
int ret
GLenum GLsizei len
Definition: glext.h:6722
GLenum src
Definition: glext.h:6340
static DWORD dstlen
Definition: directory.c:51
#define DECLSPEC_HIDDEN
Definition: precomp.h:8
GLenum GLenum dst
Definition: glext.h:6340
GLuint res
Definition: glext.h:9613
static unsigned int decode_utf8_char(unsigned char ch, const char **str, const char *strend)
Definition: utf8.c:161
static const unsigned int utf8_minval[4]
Definition: utf8.c:44
int wine_utf8_wcstombs(int flags, const WCHAR *src, int srclen, char *dst, int dstlen)
Definition: utf8.c:98
static int utf8_mbstowcs_compose(int flags, const char *src, int srclen, WCHAR *dst, int dstlen)
Definition: utf8.c:229