3 * @version $Id: bad.php,v 1.2 2006/02/26 13:20:44 harryf Exp $
4 * Tools for locating / replacing bad bytes in UTF-8 strings
5 * The Original Code is Mozilla Communicator client code.
6 * The Initial Developer of the Original Code is
7 * Netscape Communications Corporation.
8 * Portions created by the Initial Developer are Copyright (C) 1998
9 * the Initial Developer. All Rights Reserved.
10 * Ported to PHP by Henri Sivonen (http://hsivonen.iki.fi)
11 * Slight modifications to fit with phputf8 library by Harry Fuecks (hfuecks gmail com)
12 * @see http://lxr.mozilla.org/seamonkey/source/intl/uconv/src/nsUTF8ToUnicode.cpp
13 * @see http://lxr.mozilla.org/seamonkey/source/intl/uconv/src/nsUnicodeToUTF8.cpp
14 * @see http://hsivonen.iki.fi/php-utf8/
20 //--------------------------------------------------------------------
22 * Locates the first bad byte in a UTF-8 string returning it's
23 * byte index in the string
24 * PCRE Pattern to locate bad bytes in a UTF-8 string
25 * Comes from W3 FAQ: Multilingual Forms
26 * Note: modified to include full ASCII range including control chars
27 * @see http://www.w3.org/International/questions/qa-forms-utf-8
29 * @return mixed integer byte index or FALSE if no bad found
33 function utf8_bad_find($str) {
35 '([\x00-\x7F]'. # ASCII (including control chars)
36 '|[\xC2-\xDF][\x80-\xBF]'. # non-overlong 2-byte
37 '|\xE0[\xA0-\xBF][\x80-\xBF]'. # excluding overlongs
38 '|[\xE1-\xEC\xEE\xEF][\x80-\xBF]{2}'. # straight 3-byte
39 '|\xED[\x80-\x9F][\x80-\xBF]'. # excluding surrogates
40 '|\xF0[\x90-\xBF][\x80-\xBF]{2}'. # planes 1-3
41 '|[\xF1-\xF3][\x80-\xBF]{3}'. # planes 4-15
42 '|\xF4[\x80-\x8F][\x80-\xBF]{2}'. # plane 16
43 '|(.{1}))'; # invalid byte
46 while (preg_match('/'.$UTF8_BAD.'/S', $str, $matches)) {
47 $bytes = strlen($matches[0]);
48 if ( isset($matches[2])) {
52 $str = substr($str,$bytes);
57 //--------------------------------------------------------------------
59 * Locates all bad bytes in a UTF-8 string and returns a list of their
60 * byte index in the string
61 * PCRE Pattern to locate bad bytes in a UTF-8 string
62 * Comes from W3 FAQ: Multilingual Forms
63 * Note: modified to include full ASCII range including control chars
64 * @see http://www.w3.org/International/questions/qa-forms-utf-8
66 * @return mixed array of integers or FALSE if no bad found
70 function utf8_bad_findall($str) {
72 '([\x00-\x7F]'. # ASCII (including control chars)
73 '|[\xC2-\xDF][\x80-\xBF]'. # non-overlong 2-byte
74 '|\xE0[\xA0-\xBF][\x80-\xBF]'. # excluding overlongs
75 '|[\xE1-\xEC\xEE\xEF][\x80-\xBF]{2}'. # straight 3-byte
76 '|\xED[\x80-\x9F][\x80-\xBF]'. # excluding surrogates
77 '|\xF0[\x90-\xBF][\x80-\xBF]{2}'. # planes 1-3
78 '|[\xF1-\xF3][\x80-\xBF]{3}'. # planes 4-15
79 '|\xF4[\x80-\x8F][\x80-\xBF]{2}'. # plane 16
80 '|(.{1}))'; # invalid byte
83 while (preg_match('/'.$UTF8_BAD.'/S', $str, $matches)) {
84 $bytes = strlen($matches[0]);
85 if ( isset($matches[2])) {
89 $str = substr($str,$bytes);
91 if ( count($badList) > 0 ) {
97 //--------------------------------------------------------------------
99 * Strips out any bad bytes from a UTF-8 string and returns the rest
100 * PCRE Pattern to locate bad bytes in a UTF-8 string
101 * Comes from W3 FAQ: Multilingual Forms
102 * Note: modified to include full ASCII range including control chars
103 * @see http://www.w3.org/International/questions/qa-forms-utf-8
109 function utf8_bad_strip($str) {
111 '([\x00-\x7F]'. # ASCII (including control chars)
112 '|[\xC2-\xDF][\x80-\xBF]'. # non-overlong 2-byte
113 '|\xE0[\xA0-\xBF][\x80-\xBF]'. # excluding overlongs
114 '|[\xE1-\xEC\xEE\xEF][\x80-\xBF]{2}'. # straight 3-byte
115 '|\xED[\x80-\x9F][\x80-\xBF]'. # excluding surrogates
116 '|\xF0[\x90-\xBF][\x80-\xBF]{2}'. # planes 1-3
117 '|[\xF1-\xF3][\x80-\xBF]{3}'. # planes 4-15
118 '|\xF4[\x80-\x8F][\x80-\xBF]{2}'. # plane 16
119 '|(.{1}))'; # invalid byte
121 while (preg_match('/'.$UTF8_BAD.'/S', $str, $matches)) {
122 if ( !isset($matches[2])) {
125 $str = substr($str,strlen($matches[0]));
127 $result = ob_get_contents();
132 //--------------------------------------------------------------------
134 * Replace bad bytes with an alternative character - ASCII character
135 * recommended is replacement char
136 * PCRE Pattern to locate bad bytes in a UTF-8 string
137 * Comes from W3 FAQ: Multilingual Forms
138 * Note: modified to include full ASCII range including control chars
139 * @see http://www.w3.org/International/questions/qa-forms-utf-8
140 * @param string to search
141 * @param string to replace bad bytes with (defaults to '?') - use ASCII
146 function utf8_bad_replace($str, $replace = '?') {
148 '([\x00-\x7F]'. # ASCII (including control chars)
149 '|[\xC2-\xDF][\x80-\xBF]'. # non-overlong 2-byte
150 '|\xE0[\xA0-\xBF][\x80-\xBF]'. # excluding overlongs
151 '|[\xE1-\xEC\xEE\xEF][\x80-\xBF]{2}'. # straight 3-byte
152 '|\xED[\x80-\x9F][\x80-\xBF]'. # excluding surrogates
153 '|\xF0[\x90-\xBF][\x80-\xBF]{2}'. # planes 1-3
154 '|[\xF1-\xF3][\x80-\xBF]{3}'. # planes 4-15
155 '|\xF4[\x80-\x8F][\x80-\xBF]{2}'. # plane 16
156 '|(.{1}))'; # invalid byte
158 while (preg_match('/'.$UTF8_BAD.'/S', $str, $matches)) {
159 if ( !isset($matches[2])) {
164 $str = substr($str,strlen($matches[0]));
166 $result = ob_get_contents();
171 //--------------------------------------------------------------------
173 * Return code from utf8_bad_identify() when a five octet sequence is detected.
174 * Note: 5 octets sequences are valid UTF-8 but are not supported by Unicode so
175 * do not represent a useful character
176 * @see utf8_bad_identify
180 define('UTF8_BAD_5OCTET',1);
183 * Return code from utf8_bad_identify() when a six octet sequence is detected.
184 * Note: 6 octets sequences are valid UTF-8 but are not supported by Unicode so
185 * do not represent a useful character
186 * @see utf8_bad_identify
190 define('UTF8_BAD_6OCTET',2);
193 * Return code from utf8_bad_identify().
194 * Invalid octet for use as start of multi-byte UTF-8 sequence
195 * @see utf8_bad_identify
199 define('UTF8_BAD_SEQID',3);
202 * Return code from utf8_bad_identify().
203 * From Unicode 3.1, non-shortest form is illegal
204 * @see utf8_bad_identify
208 define('UTF8_BAD_NONSHORT',4);
211 * Return code from utf8_bad_identify().
212 * From Unicode 3.2, surrogate characters are illegal
213 * @see utf8_bad_identify
217 define('UTF8_BAD_SURROGATE',5);
220 * Return code from utf8_bad_identify().
221 * Codepoints outside the Unicode range are illegal
222 * @see utf8_bad_identify
226 define('UTF8_BAD_UNIOUTRANGE',6);
229 * Return code from utf8_bad_identify().
230 * Incomplete multi-octet sequence
231 * Note: this is kind of a "catch-all"
232 * @see utf8_bad_identify
236 define('UTF8_BAD_SEQINCOMPLETE',7);
238 //--------------------------------------------------------------------
240 * Reports on the type of bad byte found in a UTF-8 string. Returns a
241 * status code on the first bad byte found
242 * @author <hsivonen@iki.fi>
243 * @param string UTF-8 encoded string
244 * @return mixed integer constant describing problem or FALSE if valid UTF-8
245 * @see utf8_bad_explain
246 * @see http://hsivonen.iki.fi/php-utf8/
250 function utf8_bad_identify($str, &$i) {
252 $mState = 0; // cached expected number of octets after the current octet
253 // until the beginning of the next UTF8 character sequence
254 $mUcs4 = 0; // cached Unicode character
255 $mBytes = 1; // cached expected number of octets in the current sequence
259 for($i = 0; $i < $len; $i++) {
265 // When mState is zero we expect either a US-ASCII character or a
266 // multi-octet sequence.
267 if (0 == (0x80 & ($in))) {
268 // US-ASCII, pass straight through.
271 } else if (0xC0 == (0xE0 & ($in))) {
272 // First octet of 2 octet sequence
274 $mUcs4 = ($mUcs4 & 0x1F) << 6;
278 } else if (0xE0 == (0xF0 & ($in))) {
279 // First octet of 3 octet sequence
281 $mUcs4 = ($mUcs4 & 0x0F) << 12;
285 } else if (0xF0 == (0xF8 & ($in))) {
286 // First octet of 4 octet sequence
288 $mUcs4 = ($mUcs4 & 0x07) << 18;
292 } else if (0xF8 == (0xFC & ($in))) {
294 /* First octet of 5 octet sequence.
296 * This is illegal because the encoded codepoint must be either
297 * (a) not the shortest form or
298 * (b) outside the Unicode range of 0-0x10FFFF.
301 return UTF8_BAD_5OCTET;
303 } else if (0xFC == (0xFE & ($in))) {
305 // First octet of 6 octet sequence, see comments for 5 octet sequence.
306 return UTF8_BAD_6OCTET;
309 // Current octet is neither in the US-ASCII range nor a legal first
310 // octet of a multi-octet sequence.
311 return UTF8_BAD_SEQID;
317 // When mState is non-zero, we expect a continuation of the multi-octet
319 if (0x80 == (0xC0 & ($in))) {
321 // Legal continuation.
322 $shift = ($mState - 1) * 6;
324 $tmp = ($tmp & 0x0000003F) << $shift;
328 * End of the multi-octet sequence. mUcs4 now contains the final
329 * Unicode codepoint to be output
331 if (0 == --$mState) {
333 // From Unicode 3.1, non-shortest form is illegal
334 if (((2 == $mBytes) && ($mUcs4 < 0x0080)) ||
335 ((3 == $mBytes) && ($mUcs4 < 0x0800)) ||
336 ((4 == $mBytes) && ($mUcs4 < 0x10000)) ) {
337 return UTF8_BAD_NONSHORT;
339 // From Unicode 3.2, surrogate characters are illegal
340 } else if (($mUcs4 & 0xFFFFF800) == 0xD800) {
341 return UTF8_BAD_SURROGATE;
343 // Codepoints outside the Unicode range are illegal
344 } else if ($mUcs4 > 0x10FFFF) {
345 return UTF8_BAD_UNIOUTRANGE;
348 //initialize UTF8 cache
355 // ((0xC0 & (*in) != 0x80) && (mState != 0))
356 // Incomplete multi-octet sequence.
358 return UTF8_BAD_SEQINCOMPLETE;
363 if ( $mState != 0 ) {
364 // Incomplete multi-octet sequence.
366 return UTF8_BAD_SEQINCOMPLETE;
369 // No bad octets found
374 //--------------------------------------------------------------------
376 * Takes a return code from utf8_bad_identify() are returns a message
377 * (in English) explaining what the problem is.
378 * @param int return code from utf8_bad_identify
379 * @return mixed string message or FALSE if return code unknown
380 * @see utf8_bad_identify
384 function utf8_bad_explain($code) {
388 case UTF8_BAD_5OCTET:
389 return 'Five octet sequences are valid UTF-8 but are not supported by Unicode';
392 case UTF8_BAD_6OCTET:
393 return 'Six octet sequences are valid UTF-8 but are not supported by Unicode';
397 return 'Invalid octet for use as start of multi-byte UTF-8 sequence';
400 case UTF8_BAD_NONSHORT:
401 return 'From Unicode 3.1, non-shortest form is illegal';
404 case UTF8_BAD_SURROGATE:
405 return 'From Unicode 3.2, surrogate characters are illegal';
408 case UTF8_BAD_UNIOUTRANGE:
409 return 'Codepoints outside the Unicode range are illegal';
412 case UTF8_BAD_SEQINCOMPLETE:
413 return 'Incomplete multi-octet sequence';
418 trigger_error('Unknown error code: '.$code,E_USER_WARNING);