Mercurial > code
comparison tools/mkunicode/src/utf.h @ 352:7fe8d4094983
Utf8:
- Fix invalid decoding from UTF-8 to UTF-32
- Add all files
author | David Demelier <markand@malikania.fr> |
---|---|
date | Wed, 08 Apr 2015 12:33:45 +0200 |
parents | |
children |
comparison
equal
deleted
inserted
replaced
351:47a206e724f2 | 352:7fe8d4094983 |
---|---|
1 /* | |
2 * The authors of this software are Rob Pike and Ken Thompson. | |
3 * Copyright (c) 1998-2002 by Lucent Technologies. | |
4 * Portions Copyright (c) 2009 The Go Authors. All rights reserved. | |
5 * Permission to use, copy, modify, and distribute this software for any | |
6 * purpose without fee is hereby granted, provided that this entire notice | |
7 * is included in all copies of any software which is or includes a copy | |
8 * or modification of this software and in all copies of the supporting | |
9 * documentation for such software. | |
10 * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR IMPLIED | |
11 * WARRANTY. IN PARTICULAR, NEITHER THE AUTHORS NOR LUCENT TECHNOLOGIES MAKE ANY | |
12 * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE MERCHANTABILITY | |
13 * OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR PURPOSE. | |
14 */ | |
15 | |
16 #ifndef _UTFH_ | |
17 #define _UTFH_ 1 | |
18 | |
19 typedef unsigned int Rune; /* Code-point values in Unicode 4.0 are 21 bits wide.*/ | |
20 | |
21 enum | |
22 { | |
23 UTFmax = 4, /* maximum bytes per rune */ | |
24 Runesync = 0x80, /* cannot represent part of a UTF sequence (<) */ | |
25 Runeself = 0x80, /* rune and UTF sequences are the same (<) */ | |
26 Runeerror = 0xFFFD, /* decoding error in UTF */ | |
27 Runemax = 0x10FFFF, /* maximum rune value */ | |
28 }; | |
29 | |
30 #ifdef __cplusplus | |
31 extern "C" { | |
32 #endif | |
33 | |
34 /* | |
35 * rune routines | |
36 */ | |
37 | |
38 /* | |
39 * These routines were written by Rob Pike and Ken Thompson | |
40 * and first appeared in Plan 9. | |
41 * SEE ALSO | |
42 * utf (7) | |
43 * tcs (1) | |
44 */ | |
45 | |
46 // runetochar copies (encodes) one rune, pointed to by r, to at most | |
47 // UTFmax bytes starting at s and returns the number of bytes generated. | |
48 | |
49 int runetochar(char* s, const Rune* r); | |
50 | |
51 | |
52 // chartorune copies (decodes) at most UTFmax bytes starting at s to | |
53 // one rune, pointed to by r, and returns the number of bytes consumed. | |
54 // If the input is not exactly in UTF format, chartorune will set *r | |
55 // to Runeerror and return 1. | |
56 // | |
57 // Note: There is no special case for a "null-terminated" string. A | |
58 // string whose first byte has the value 0 is the UTF8 encoding of the | |
59 // Unicode value 0 (i.e., ASCII NULL). A byte value of 0 is illegal | |
60 // anywhere else in a UTF sequence. | |
61 | |
62 int chartorune(Rune* r, const char* s); | |
63 | |
64 | |
65 // charntorune is like chartorune, except that it will access at most | |
66 // n bytes of s. If the UTF sequence is incomplete within n bytes, | |
67 // charntorune will set *r to Runeerror and return 0. If it is complete | |
68 // but not in UTF format, it will set *r to Runeerror and return 1. | |
69 // | |
70 // Added 2004-09-24 by Wei-Hwa Huang | |
71 | |
72 int charntorune(Rune* r, const char* s, int n); | |
73 | |
74 // isvalidcharntorune(str, n, r, consumed) | |
75 // is a convenience function that calls "*consumed = charntorune(r, str, n)" | |
76 // and returns an int (logically boolean) indicating whether the first | |
77 // n bytes of str was a valid and complete UTF sequence. | |
78 | |
79 int isvalidcharntorune(const char* str, int n, Rune* r, int* consumed); | |
80 | |
81 // runelen returns the number of bytes required to convert r into UTF. | |
82 | |
83 int runelen(Rune r); | |
84 | |
85 | |
86 // runenlen returns the number of bytes required to convert the n | |
87 // runes pointed to by r into UTF. | |
88 | |
89 int runenlen(const Rune* r, int n); | |
90 | |
91 | |
92 // fullrune returns 1 if the string s of length n is long enough to be | |
93 // decoded by chartorune, and 0 otherwise. This does not guarantee | |
94 // that the string contains a legal UTF encoding. This routine is used | |
95 // by programs that obtain input one byte at a time and need to know | |
96 // when a full rune has arrived. | |
97 | |
98 int fullrune(const char* s, int n); | |
99 | |
100 // The following routines are analogous to the corresponding string | |
101 // routines with "utf" substituted for "str", and "rune" substituted | |
102 // for "chr". | |
103 | |
104 // utflen returns the number of runes that are represented by the UTF | |
105 // string s. (cf. strlen) | |
106 | |
107 int utflen(const char* s); | |
108 | |
109 | |
110 // utfnlen returns the number of complete runes that are represented | |
111 // by the first n bytes of the UTF string s. If the last few bytes of | |
112 // the string contain an incompletely coded rune, utfnlen will not | |
113 // count them; in this way, it differs from utflen, which includes | |
114 // every byte of the string. (cf. strnlen) | |
115 | |
116 int utfnlen(const char* s, long n); | |
117 | |
118 | |
119 // utfrune returns a pointer to the first occurrence of rune r in the | |
120 // UTF string s, or 0 if r does not occur in the string. The NULL | |
121 // byte terminating a string is considered to be part of the string s. | |
122 // (cf. strchr) | |
123 | |
124 /*const*/ char* utfrune(const char* s, Rune r); | |
125 | |
126 | |
127 // utfrrune returns a pointer to the last occurrence of rune r in the | |
128 // UTF string s, or 0 if r does not occur in the string. The NULL | |
129 // byte terminating a string is considered to be part of the string s. | |
130 // (cf. strrchr) | |
131 | |
132 /*const*/ char* utfrrune(const char* s, Rune r); | |
133 | |
134 | |
135 // utfutf returns a pointer to the first occurrence of the UTF string | |
136 // s2 as a UTF substring of s1, or 0 if there is none. If s2 is the | |
137 // null string, utfutf returns s1. (cf. strstr) | |
138 | |
139 const char* utfutf(const char* s1, const char* s2); | |
140 | |
141 | |
142 // utfecpy copies UTF sequences until a null sequence has been copied, | |
143 // but writes no sequences beyond es1. If any sequences are copied, | |
144 // s1 is terminated by a null sequence, and a pointer to that sequence | |
145 // is returned. Otherwise, the original s1 is returned. (cf. strecpy) | |
146 | |
147 char* utfecpy(char *s1, char *es1, const char *s2); | |
148 | |
149 | |
150 | |
151 // These functions are rune-string analogues of the corresponding | |
152 // functions in strcat (3). | |
153 // | |
154 // These routines first appeared in Plan 9. | |
155 // SEE ALSO | |
156 // memmove (3) | |
157 // rune (3) | |
158 // strcat (2) | |
159 // | |
160 // BUGS: The outcome of overlapping moves varies among implementations. | |
161 | |
162 Rune* runestrcat(Rune* s1, const Rune* s2); | |
163 Rune* runestrncat(Rune* s1, const Rune* s2, long n); | |
164 | |
165 const Rune* runestrchr(const Rune* s, Rune c); | |
166 | |
167 int runestrcmp(const Rune* s1, const Rune* s2); | |
168 int runestrncmp(const Rune* s1, const Rune* s2, long n); | |
169 | |
170 Rune* runestrcpy(Rune* s1, const Rune* s2); | |
171 Rune* runestrncpy(Rune* s1, const Rune* s2, long n); | |
172 Rune* runestrecpy(Rune* s1, Rune* es1, const Rune* s2); | |
173 | |
174 Rune* runestrdup(const Rune* s); | |
175 | |
176 const Rune* runestrrchr(const Rune* s, Rune c); | |
177 long runestrlen(const Rune* s); | |
178 const Rune* runestrstr(const Rune* s1, const Rune* s2); | |
179 | |
180 | |
181 | |
182 // The following routines test types and modify cases for Unicode | |
183 // characters. Unicode defines some characters as letters and | |
184 // specifies three cases: upper, lower, and title. Mappings among the | |
185 // cases are also defined, although they are not exhaustive: some | |
186 // upper case letters have no lower case mapping, and so on. Unicode | |
187 // also defines several character properties, a subset of which are | |
188 // checked by these routines. These routines are based on Unicode | |
189 // version 3.0.0. | |
190 // | |
191 // NOTE: The routines are implemented in C, so the boolean functions | |
192 // (e.g., isupperrune) return 0 for false and 1 for true. | |
193 // | |
194 // | |
195 // toupperrune, tolowerrune, and totitlerune are the Unicode case | |
196 // mappings. These routines return the character unchanged if it has | |
197 // no defined mapping. | |
198 | |
199 Rune toupperrune(Rune r); | |
200 Rune tolowerrune(Rune r); | |
201 Rune totitlerune(Rune r); | |
202 | |
203 | |
204 // isupperrune tests for upper case characters, including Unicode | |
205 // upper case letters and targets of the toupper mapping. islowerrune | |
206 // and istitlerune are defined analogously. | |
207 | |
208 int isupperrune(Rune r); | |
209 int islowerrune(Rune r); | |
210 int istitlerune(Rune r); | |
211 | |
212 | |
213 // isalpharune tests for Unicode letters; this includes ideographs in | |
214 // addition to alphabetic characters. | |
215 | |
216 int isalpharune(Rune r); | |
217 | |
218 | |
219 // isdigitrune tests for digits. Non-digit numbers, such as Roman | |
220 // numerals, are not included. | |
221 | |
222 int isdigitrune(Rune r); | |
223 | |
224 | |
225 // isspacerune tests for whitespace characters, including "C" locale | |
226 // whitespace, Unicode defined whitespace, and the "zero-width | |
227 // non-break space" character. | |
228 | |
229 int isspacerune(Rune r); | |
230 | |
231 | |
232 // (The comments in this file were copied from the manpage files rune.3, | |
233 // isalpharune.3, and runestrcat.3. Some formatting changes were also made | |
234 // to conform to Google style. /JRM 11/11/05) | |
235 | |
236 #ifdef __cplusplus | |
237 } | |
238 #endif | |
239 | |
240 #endif |