1/*
2   +----------------------------------------------------------------------+
3   | PHP Version 5                                                        |
4   +----------------------------------------------------------------------+
5   | Copyright (c) 1997-2014 The PHP Group                                |
6   +----------------------------------------------------------------------+
7   | This source file is subject to version 3.01 of the PHP license,      |
8   | that is bundled with this package in the file LICENSE, and is        |
9   | available through the world-wide-web at the following url:           |
10   | http://www.php.net/license/3_01.txt                                  |
11   | If you did not receive a copy of the PHP license and are unable to   |
12   | obtain it through the world-wide-web, please send a note to          |
13   | license@php.net so we can mail you a copy immediately.               |
14   +----------------------------------------------------------------------+
15   | Authors: Rasmus Lerdorf <rasmus@php.net>                             |
16   |          Jaakko Hyvätti <jaakko.hyvatti@iki.fi>                      |
17   |          Wez Furlong    <wez@thebrainroom.com>                       |
18   |          Gustavo Lopes  <cataphract@php.net>                         |
19   +----------------------------------------------------------------------+
20*/
21
22/* $Id$ */
23
24/*
25 * HTML entity resources:
26 *
27 * http://www.unicode.org/Public/MAPPINGS/OBSOLETE/UNI2SGML.TXT
28 *
29 * XHTML 1.0 DTD
30 * http://www.w3.org/TR/2002/REC-xhtml1-20020801/dtds.html#h-A2
31 *
32 * From HTML 4.01 strict DTD:
33 * http://www.w3.org/TR/html4/HTMLlat1.ent
34 * http://www.w3.org/TR/html4/HTMLsymbol.ent
35 * http://www.w3.org/TR/html4/HTMLspecial.ent
36 *
37 * HTML 5:
38 * http://dev.w3.org/html5/spec/Overview.html#named-character-references
39 */
40
41#include "php.h"
42#if PHP_WIN32
43#include "config.w32.h"
44#else
45#include <php_config.h>
46#endif
47#include "php_standard.h"
48#include "php_string.h"
49#include "SAPI.h"
50#if HAVE_LOCALE_H
51#include <locale.h>
52#endif
53#if HAVE_LANGINFO_H
54#include <langinfo.h>
55#endif
56
57#include <zend_hash.h>
58#include "html_tables.h"
59
60/* Macro for disabling flag of translation of non-basic entities where this isn't supported.
61 * Not appropriate for html_entity_decode/htmlspecialchars_decode */
62#define LIMIT_ALL(all, doctype, charset) do { \
63    (all) = (all) && !CHARSET_PARTIAL_SUPPORT((charset)) && ((doctype) != ENT_HTML_DOC_XML1); \
64} while (0)
65
66#define MB_FAILURE(pos, advance) do { \
67    *cursor = pos + (advance); \
68    *status = FAILURE; \
69    return 0; \
70} while (0)
71
72#define CHECK_LEN(pos, chars_need) ((str_len - (pos)) >= (chars_need))
73
74/* valid as single byte character or leading byte */
75#define utf8_lead(c)  ((c) < 0x80 || ((c) >= 0xC2 && (c) <= 0xF4))
76/* whether it's actually valid depends on other stuff;
77 * this macro cannot check for non-shortest forms, surrogates or
78 * code points above 0x10FFFF */
79#define utf8_trail(c) ((c) >= 0x80 && (c) <= 0xBF)
80
81#define gb2312_lead(c) ((c) != 0x8E && (c) != 0x8F && (c) != 0xA0 && (c) != 0xFF)
82#define gb2312_trail(c) ((c) >= 0xA1 && (c) <= 0xFE)
83
84#define sjis_lead(c) ((c) != 0x80 && (c) != 0xA0 && (c) < 0xFD)
85#define sjis_trail(c) ((c) >= 0x40  && (c) != 0x7F && (c) < 0xFD)
86
87/* {{{ get_default_charset
88 */
89static char *get_default_charset(TSRMLS_D) {
90    if (PG(internal_encoding) && PG(internal_encoding)[0]) {
91        return PG(internal_encoding);
92    } else if (SG(default_charset) && SG(default_charset)[0] ) {
93        return SG(default_charset);
94    }
95    return NULL;
96}
97/* }}} */
98
99/* {{{ get_next_char
100 */
101static inline unsigned int get_next_char(
102        enum entity_charset charset,
103        const unsigned char *str,
104        size_t str_len,
105        size_t *cursor,
106        int *status)
107{
108    size_t pos = *cursor;
109    unsigned int this_char = 0;
110
111    *status = SUCCESS;
112    assert(pos <= str_len);
113
114    if (!CHECK_LEN(pos, 1))
115        MB_FAILURE(pos, 1);
116
117    switch (charset) {
118    case cs_utf_8:
119        {
120            /* We'll follow strategy 2. from section 3.6.1 of UTR #36:
121             * "In a reported illegal byte sequence, do not include any
122             *  non-initial byte that encodes a valid character or is a leading
123             *  byte for a valid sequence." */
124            unsigned char c;
125            c = str[pos];
126            if (c < 0x80) {
127                this_char = c;
128                pos++;
129            } else if (c < 0xc2) {
130                MB_FAILURE(pos, 1);
131            } else if (c < 0xe0) {
132                if (!CHECK_LEN(pos, 2))
133                    MB_FAILURE(pos, 1);
134
135                if (!utf8_trail(str[pos + 1])) {
136                    MB_FAILURE(pos, utf8_lead(str[pos + 1]) ? 1 : 2);
137                }
138                this_char = ((c & 0x1f) << 6) | (str[pos + 1] & 0x3f);
139                if (this_char < 0x80) { /* non-shortest form */
140                    MB_FAILURE(pos, 2);
141                }
142                pos += 2;
143            } else if (c < 0xf0) {
144                size_t avail = str_len - pos;
145
146                if (avail < 3 ||
147                        !utf8_trail(str[pos + 1]) || !utf8_trail(str[pos + 2])) {
148                    if (avail < 2 || utf8_lead(str[pos + 1]))
149                        MB_FAILURE(pos, 1);
150                    else if (avail < 3 || utf8_lead(str[pos + 2]))
151                        MB_FAILURE(pos, 2);
152                    else
153                        MB_FAILURE(pos, 3);
154                }
155
156                this_char = ((c & 0x0f) << 12) | ((str[pos + 1] & 0x3f) << 6) | (str[pos + 2] & 0x3f);
157                if (this_char < 0x800) { /* non-shortest form */
158                    MB_FAILURE(pos, 3);
159                } else if (this_char >= 0xd800 && this_char <= 0xdfff) { /* surrogate */
160                    MB_FAILURE(pos, 3);
161                }
162                pos += 3;
163            } else if (c < 0xf5) {
164                size_t avail = str_len - pos;
165
166                if (avail < 4 ||
167                        !utf8_trail(str[pos + 1]) || !utf8_trail(str[pos + 2]) ||
168                        !utf8_trail(str[pos + 3])) {
169                    if (avail < 2 || utf8_lead(str[pos + 1]))
170                        MB_FAILURE(pos, 1);
171                    else if (avail < 3 || utf8_lead(str[pos + 2]))
172                        MB_FAILURE(pos, 2);
173                    else if (avail < 4 || utf8_lead(str[pos + 3]))
174                        MB_FAILURE(pos, 3);
175                    else
176                        MB_FAILURE(pos, 4);
177                }
178
179                this_char = ((c & 0x07) << 18) | ((str[pos + 1] & 0x3f) << 12) | ((str[pos + 2] & 0x3f) << 6) | (str[pos + 3] & 0x3f);
180                if (this_char < 0x10000 || this_char > 0x10FFFF) { /* non-shortest form or outside range */
181                    MB_FAILURE(pos, 4);
182                }
183                pos += 4;
184            } else {
185                MB_FAILURE(pos, 1);
186            }
187        }
188        break;
189
190    case cs_big5:
191        /* reference http://demo.icu-project.org/icu-bin/convexp?conv=big5 */
192        {
193            unsigned char c = str[pos];
194            if (c >= 0x81 && c <= 0xFE) {
195                unsigned char next;
196                if (!CHECK_LEN(pos, 2))
197                    MB_FAILURE(pos, 1);
198
199                next = str[pos + 1];
200
201                if ((next >= 0x40 && next <= 0x7E) ||
202                        (next >= 0xA1 && next <= 0xFE)) {
203                    this_char = (c << 8) | next;
204                } else {
205                    MB_FAILURE(pos, 1);
206                }
207                pos += 2;
208            } else {
209                this_char = c;
210                pos += 1;
211            }
212        }
213        break;
214
215    case cs_big5hkscs:
216        {
217            unsigned char c = str[pos];
218            if (c >= 0x81 && c <= 0xFE) {
219                unsigned char next;
220                if (!CHECK_LEN(pos, 2))
221                    MB_FAILURE(pos, 1);
222
223                next = str[pos + 1];
224
225                if ((next >= 0x40 && next <= 0x7E) ||
226                        (next >= 0xA1 && next <= 0xFE)) {
227                    this_char = (c << 8) | next;
228                } else if (next != 0x80 && next != 0xFF) {
229                    MB_FAILURE(pos, 1);
230                } else {
231                    MB_FAILURE(pos, 2);
232                }
233                pos += 2;
234            } else {
235                this_char = c;
236                pos += 1;
237            }
238        }
239        break;
240
241    case cs_gb2312: /* EUC-CN */
242        {
243            unsigned char c = str[pos];
244            if (c >= 0xA1 && c <= 0xFE) {
245                unsigned char next;
246                if (!CHECK_LEN(pos, 2))
247                    MB_FAILURE(pos, 1);
248
249                next = str[pos + 1];
250
251                if (gb2312_trail(next)) {
252                    this_char = (c << 8) | next;
253                } else if (gb2312_lead(next)) {
254                    MB_FAILURE(pos, 1);
255                } else {
256                    MB_FAILURE(pos, 2);
257                }
258                pos += 2;
259            } else if (gb2312_lead(c)) {
260                this_char = c;
261                pos += 1;
262            } else {
263                MB_FAILURE(pos, 1);
264            }
265        }
266        break;
267
268    case cs_sjis:
269        {
270            unsigned char c = str[pos];
271            if ((c >= 0x81 && c <= 0x9F) || (c >= 0xE0 && c <= 0xFC)) {
272                unsigned char next;
273                if (!CHECK_LEN(pos, 2))
274                    MB_FAILURE(pos, 1);
275
276                next = str[pos + 1];
277
278                if (sjis_trail(next)) {
279                    this_char = (c << 8) | next;
280                } else if (sjis_lead(next)) {
281                    MB_FAILURE(pos, 1);
282                } else {
283                    MB_FAILURE(pos, 2);
284                }
285                pos += 2;
286            } else if (c < 0x80 || (c >= 0xA1 && c <= 0xDF)) {
287                this_char = c;
288                pos += 1;
289            } else {
290                MB_FAILURE(pos, 1);
291            }
292        }
293        break;
294
295    case cs_eucjp:
296        {
297            unsigned char c = str[pos];
298
299            if (c >= 0xA1 && c <= 0xFE) {
300                unsigned next;
301                if (!CHECK_LEN(pos, 2))
302                    MB_FAILURE(pos, 1);
303                next = str[pos + 1];
304
305                if (next >= 0xA1 && next <= 0xFE) {
306                    /* this a jis kanji char */
307                    this_char = (c << 8) | next;
308                } else {
309                    MB_FAILURE(pos, (next != 0xA0 && next != 0xFF) ? 1 : 2);
310                }
311                pos += 2;
312            } else if (c == 0x8E) {
313                unsigned next;
314                if (!CHECK_LEN(pos, 2))
315                    MB_FAILURE(pos, 1);
316
317                next = str[pos + 1];
318                if (next >= 0xA1 && next <= 0xDF) {
319                    /* JIS X 0201 kana */
320                    this_char = (c << 8) | next;
321                } else {
322                    MB_FAILURE(pos, (next != 0xA0 && next != 0xFF) ? 1 : 2);
323                }
324                pos += 2;
325            } else if (c == 0x8F) {
326                size_t avail = str_len - pos;
327
328                if (avail < 3 || !(str[pos + 1] >= 0xA1 && str[pos + 1] <= 0xFE) ||
329                        !(str[pos + 2] >= 0xA1 && str[pos + 2] <= 0xFE)) {
330                    if (avail < 2 || (str[pos + 1] != 0xA0 && str[pos + 1] != 0xFF))
331                        MB_FAILURE(pos, 1);
332                    else if (avail < 3 || (str[pos + 2] != 0xA0 && str[pos + 2] != 0xFF))
333                        MB_FAILURE(pos, 2);
334                    else
335                        MB_FAILURE(pos, 3);
336                } else {
337                    /* JIS X 0212 hojo-kanji */
338                    this_char = (c << 16) | (str[pos + 1] << 8) | str[pos + 2];
339                }
340                pos += 3;
341            } else if (c != 0xA0 && c != 0xFF) {
342                /* character encoded in 1 code unit */
343                this_char = c;
344                pos += 1;
345            } else {
346                MB_FAILURE(pos, 1);
347            }
348        }
349        break;
350    default:
351        /* single-byte charsets */
352        this_char = str[pos++];
353        break;
354    }
355
356    *cursor = pos;
357    return this_char;
358}
359/* }}} */
360
361/* {{{ php_next_utf8_char
362 * Public interface for get_next_char used with UTF-8 */
363 PHPAPI unsigned int php_next_utf8_char(
364        const unsigned char *str,
365        size_t str_len,
366        size_t *cursor,
367        int *status)
368{
369    return get_next_char(cs_utf_8, str, str_len, cursor, status);
370}
371/* }}} */
372
373/* {{{ entity_charset determine_charset
374 * returns the charset identifier based on current locale or a hint.
375 * defaults to UTF-8 */
376static enum entity_charset determine_charset(char *charset_hint TSRMLS_DC)
377{
378    int i;
379    enum entity_charset charset = cs_utf_8;
380    int len = 0;
381    const zend_encoding *zenc;
382
383    /* Default is now UTF-8 */
384    if (charset_hint == NULL)
385        return cs_utf_8;
386
387    if ((len = strlen(charset_hint)) != 0) {
388        goto det_charset;
389    }
390
391    zenc = zend_multibyte_get_internal_encoding(TSRMLS_C);
392    if (zenc != NULL) {
393        charset_hint = (char *)zend_multibyte_get_encoding_name(zenc);
394        if (charset_hint != NULL && (len=strlen(charset_hint)) != 0) {
395            if ((len == 4) /* sizeof (none|auto|pass) */ &&
396                    (!memcmp("pass", charset_hint, 4) ||
397                     !memcmp("auto", charset_hint, 4) ||
398                     !memcmp("auto", charset_hint, 4))) {
399                charset_hint = NULL;
400                len = 0;
401            } else {
402                goto det_charset;
403            }
404        }
405    }
406
407    charset_hint = SG(default_charset);
408    if (charset_hint != NULL && (len=strlen(charset_hint)) != 0) {
409        goto det_charset;
410    }
411
412    /* try to detect the charset for the locale */
413#if HAVE_NL_LANGINFO && HAVE_LOCALE_H && defined(CODESET)
414    charset_hint = nl_langinfo(CODESET);
415    if (charset_hint != NULL && (len=strlen(charset_hint)) != 0) {
416        goto det_charset;
417    }
418#endif
419
420#if HAVE_LOCALE_H
421    /* try to figure out the charset from the locale */
422    {
423        char *localename;
424        char *dot, *at;
425
426        /* lang[_territory][.codeset][@modifier] */
427        localename = setlocale(LC_CTYPE, NULL);
428
429        dot = strchr(localename, '.');
430        if (dot) {
431            dot++;
432            /* locale specifies a codeset */
433            at = strchr(dot, '@');
434            if (at)
435                len = at - dot;
436            else
437                len = strlen(dot);
438            charset_hint = dot;
439        } else {
440            /* no explicit name; see if the name itself
441             * is the charset */
442            charset_hint = localename;
443            len = strlen(charset_hint);
444        }
445    }
446#endif
447
448det_charset:
449
450    if (charset_hint) {
451        int found = 0;
452
453        /* now walk the charset map and look for the codeset */
454        for (i = 0; charset_map[i].codeset; i++) {
455            if (len == strlen(charset_map[i].codeset) && strncasecmp(charset_hint, charset_map[i].codeset, len) == 0) {
456                charset = charset_map[i].charset;
457                found = 1;
458                break;
459            }
460        }
461        if (!found) {
462            php_error_docref(NULL TSRMLS_CC, E_WARNING, "charset `%s' not supported, assuming utf-8",
463                    charset_hint);
464        }
465    }
466    return charset;
467}
468/* }}} */
469
470/* {{{ php_utf32_utf8 */
471static inline size_t php_utf32_utf8(unsigned char *buf, unsigned k)
472{
473    size_t retval = 0;
474
475    /* assert(0x0 <= k <= 0x10FFFF); */
476
477    if (k < 0x80) {
478        buf[0] = k;
479        retval = 1;
480    } else if (k < 0x800) {
481        buf[0] = 0xc0 | (k >> 6);
482        buf[1] = 0x80 | (k & 0x3f);
483        retval = 2;
484    } else if (k < 0x10000) {
485        buf[0] = 0xe0 | (k >> 12);
486        buf[1] = 0x80 | ((k >> 6) & 0x3f);
487        buf[2] = 0x80 | (k & 0x3f);
488        retval = 3;
489    } else {
490        buf[0] = 0xf0 | (k >> 18);
491        buf[1] = 0x80 | ((k >> 12) & 0x3f);
492        buf[2] = 0x80 | ((k >> 6) & 0x3f);
493        buf[3] = 0x80 | (k & 0x3f);
494        retval = 4;
495    }
496    /* UTF-8 has been restricted to max 4 bytes since RFC 3629 */
497
498    return retval;
499}
500/* }}} */
501
502/* {{{ php_mb2_int_to_char
503 * Convert back big endian int representation of sequence of one or two 8-bit code units. */
504static inline size_t php_mb2_int_to_char(unsigned char *buf, unsigned k)
505{
506    assert(k <= 0xFFFFU);
507    /* one or two bytes */
508    if (k <= 0xFFU) { /* 1 */
509        buf[0] = k;
510        return 1U;
511    } else { /* 2 */
512        buf[0] = k >> 8;
513        buf[1] = k & 0xFFU;
514        return 2U;
515    }
516}
517/* }}} */
518
519/* {{{ php_mb3_int_to_char
520 * Convert back big endian int representation of sequence of one to three 8-bit code units.
521 * For EUC-JP. */
522static inline size_t php_mb3_int_to_char(unsigned char *buf, unsigned k)
523{
524    assert(k <= 0xFFFFFFU);
525    /* one to three bytes */
526    if (k <= 0xFFU) { /* 1 */
527        buf[0] = k;
528        return 1U;
529    } else if (k <= 0xFFFFU) { /* 2 */
530        buf[0] = k >> 8;
531        buf[1] = k & 0xFFU;
532        return 2U;
533    } else {
534        buf[0] = k >> 16;
535        buf[1] = (k >> 8) & 0xFFU;
536        buf[2] = k & 0xFFU;
537        return 3U;
538    }
539}
540/* }}} */
541
542
543/* {{{ unimap_bsearc_cmp
544 * Binary search of unicode code points in unicode <--> charset mapping.
545 * Returns the code point in the target charset (whose mapping table was given) or 0 if
546 * the unicode code point is not in the table.
547 */
548static inline unsigned char unimap_bsearch(const uni_to_enc *table, unsigned code_key_a, size_t num)
549{
550    const uni_to_enc *l = table,
551                     *h = &table[num-1],
552                     *m;
553    unsigned short code_key;
554
555    /* we have no mappings outside the BMP */
556    if (code_key_a > 0xFFFFU)
557        return 0;
558
559    code_key = (unsigned short) code_key_a;
560
561    while (l <= h) {
562        m = l + (h - l) / 2;
563        if (code_key < m->un_code_point)
564            h = m - 1;
565        else if (code_key > m->un_code_point)
566            l = m + 1;
567        else
568            return m->cs_code;
569    }
570    return 0;
571}
572/* }}} */
573
574/* {{{ map_from_unicode */
575static inline int map_from_unicode(unsigned code, enum entity_charset charset, unsigned *res)
576{
577    unsigned char found;
578    const uni_to_enc *table;
579    size_t table_size;
580
581    switch (charset) {
582    case cs_8859_1:
583        /* identity mapping of code points to unicode */
584        if (code > 0xFF) {
585            return FAILURE;
586        }
587        *res = code;
588        break;
589
590    case cs_8859_5:
591        if (code <= 0xA0 || code == 0xAD /* soft hyphen */) {
592            *res = code;
593        } else if (code == 0x2116) {
594            *res = 0xF0; /* numero sign */
595        } else if (code == 0xA7) {
596            *res = 0xFD; /* section sign */
597        } else if (code >= 0x0401 && code <= 0x044F) {
598            if (code == 0x040D || code == 0x0450 || code == 0x045D)
599                return FAILURE;
600            *res = code - 0x360;
601        } else {
602            return FAILURE;
603        }
604        break;
605
606    case cs_8859_15:
607        if (code < 0xA4 || (code > 0xBE && code <= 0xFF)) {
608            *res = code;
609        } else { /* between A4 and 0xBE */
610            found = unimap_bsearch(unimap_iso885915,
611                code, sizeof(unimap_iso885915) / sizeof(*unimap_iso885915));
612            if (found)
613                *res = found;
614            else
615                return FAILURE;
616        }
617        break;
618
619    case cs_cp1252:
620        if (code <= 0x7F || (code >= 0xA0 && code <= 0xFF)) {
621            *res = code;
622        } else {
623            found = unimap_bsearch(unimap_win1252,
624                code, sizeof(unimap_win1252) / sizeof(*unimap_win1252));
625            if (found)
626                *res = found;
627            else
628                return FAILURE;
629        }
630        break;
631
632    case cs_macroman:
633        if (code == 0x7F)
634            return FAILURE;
635        table = unimap_macroman;
636        table_size = sizeof(unimap_macroman) / sizeof(*unimap_macroman);
637        goto table_over_7F;
638    case cs_cp1251:
639        table = unimap_win1251;
640        table_size = sizeof(unimap_win1251) / sizeof(*unimap_win1251);
641        goto table_over_7F;
642    case cs_koi8r:
643        table = unimap_koi8r;
644        table_size = sizeof(unimap_koi8r) / sizeof(*unimap_koi8r);
645        goto table_over_7F;
646    case cs_cp866:
647        table = unimap_cp866;
648        table_size = sizeof(unimap_cp866) / sizeof(*unimap_cp866);
649
650table_over_7F:
651        if (code <= 0x7F) {
652            *res = code;
653        } else {
654            found = unimap_bsearch(table, code, table_size);
655            if (found)
656                *res = found;
657            else
658                return FAILURE;
659        }
660        break;
661
662    /* from here on, only map the possible characters in the ASCII range.
663     * to improve support here, it's a matter of building the unicode mappings.
664     * See <http://www.unicode.org/Public/6.0.0/ucd/Unihan.zip> */
665    case cs_sjis:
666    case cs_eucjp:
667        /* we interpret 0x5C as the Yen symbol. This is not universal.
668         * See <http://www.w3.org/Submission/japanese-xml/#ambiguity_of_yen> */
669        if (code >= 0x20 && code <= 0x7D) {
670            if (code == 0x5C)
671                return FAILURE;
672            *res = code;
673        } else {
674            return FAILURE;
675        }
676        break;
677
678    case cs_big5:
679    case cs_big5hkscs:
680    case cs_gb2312:
681        if (code >= 0x20 && code <= 0x7D) {
682            *res = code;
683        } else {
684            return FAILURE;
685        }
686        break;
687
688    default:
689        return FAILURE;
690    }
691
692    return SUCCESS;
693}
694/* }}} */
695
696/* {{{ */
697static inline void map_to_unicode(unsigned code, const enc_to_uni *table, unsigned *res)
698{
699    /* only single byte encodings are currently supported; assumed code <= 0xFF */
700    *res = table->inner[ENT_ENC_TO_UNI_STAGE1(code)]->uni_cp[ENT_ENC_TO_UNI_STAGE2(code)];
701}
702/* }}} */
703
704/* {{{ unicode_cp_is_allowed */
705static inline int unicode_cp_is_allowed(unsigned uni_cp, int document_type)
706{
707    /* XML 1.0              HTML 4.01           HTML 5
708     * 0x09..0x0A           0x09..0x0A          0x09..0x0A
709     * 0x0D                 0x0D                0x0C..0x0D
710     * 0x0020..0xD7FF       0x20..0x7E          0x20..0x7E
711     *                      0x00A0..0xD7FF      0x00A0..0xD7FF
712     * 0xE000..0xFFFD       0xE000..0x10FFFF    0xE000..0xFDCF
713     * 0x010000..0x10FFFF                       0xFDF0..0x10FFFF (*)
714     *
715     * (*) exclude code points where ((code & 0xFFFF) >= 0xFFFE)
716     *
717     * References:
718     * XML 1.0:   <http://www.w3.org/TR/REC-xml/#charsets>
719     * HTML 4.01: <http://www.w3.org/TR/1999/PR-html40-19990824/sgml/sgmldecl.html>
720     * HTML 5:    <http://dev.w3.org/html5/spec/Overview.html#preprocessing-the-input-stream>
721     *
722     * Not sure this is the relevant part for HTML 5, though. I opted to
723     * disallow the characters that would result in a parse error when
724     * preprocessing of the input stream. See also section 8.1.3.
725     *
726     * It's unclear if XHTML 1.0 allows C1 characters. I'll opt to apply to
727     * XHTML 1.0 the same rules as for XML 1.0.
728     * See <http://cmsmcq.com/2007/C1.xml>.
729     */
730
731    switch (document_type) {
732    case ENT_HTML_DOC_HTML401:
733        return (uni_cp >= 0x20 && uni_cp <= 0x7E) ||
734            (uni_cp == 0x0A || uni_cp == 0x09 || uni_cp == 0x0D) ||
735            (uni_cp >= 0xA0 && uni_cp <= 0xD7FF) ||
736            (uni_cp >= 0xE000 && uni_cp <= 0x10FFFF);
737    case ENT_HTML_DOC_HTML5:
738        return (uni_cp >= 0x20 && uni_cp <= 0x7E) ||
739            (uni_cp >= 0x09 && uni_cp <= 0x0D && uni_cp != 0x0B) || /* form feed U+0C allowed */
740            (uni_cp >= 0xA0 && uni_cp <= 0xD7FF) ||
741            (uni_cp >= 0xE000 && uni_cp <= 0x10FFFF &&
742                ((uni_cp & 0xFFFF) < 0xFFFE) && /* last two of each plane (nonchars) disallowed */
743                (uni_cp < 0xFDD0 || uni_cp > 0xFDEF)); /* U+FDD0-U+FDEF (nonchars) disallowed */
744    case ENT_HTML_DOC_XHTML:
745    case ENT_HTML_DOC_XML1:
746        return (uni_cp >= 0x20 && uni_cp <= 0xD7FF) ||
747            (uni_cp == 0x0A || uni_cp == 0x09 || uni_cp == 0x0D) ||
748            (uni_cp >= 0xE000 && uni_cp <= 0x10FFFF && uni_cp != 0xFFFE && uni_cp != 0xFFFF);
749    default:
750        return 1;
751    }
752}
753/* }}} */
754
755/* {{{ unicode_cp_is_allowed */
756static inline int numeric_entity_is_allowed(unsigned uni_cp, int document_type)
757{
758    /* less restrictive than unicode_cp_is_allowed */
759    switch (document_type) {
760    case ENT_HTML_DOC_HTML401:
761        /* all non-SGML characters (those marked with UNUSED in DESCSET) should be
762         * representable with numeric entities */
763        return uni_cp <= 0x10FFFF;
764    case ENT_HTML_DOC_HTML5:
765        /* 8.1.4. The numeric character reference forms described above are allowed to
766         * reference any Unicode code point other than U+0000, U+000D, permanently
767         * undefined Unicode characters (noncharacters), and control characters other
768         * than space characters (U+0009, U+000A, U+000C and U+000D) */
769        /* seems to allow surrogate characters, then */
770        return (uni_cp >= 0x20 && uni_cp <= 0x7E) ||
771            (uni_cp >= 0x09 && uni_cp <= 0x0C && uni_cp != 0x0B) || /* form feed U+0C allowed, but not U+0D */
772            (uni_cp >= 0xA0 && uni_cp <= 0x10FFFF &&
773                ((uni_cp & 0xFFFF) < 0xFFFE) && /* last two of each plane (nonchars) disallowed */
774                (uni_cp < 0xFDD0 || uni_cp > 0xFDEF)); /* U+FDD0-U+FDEF (nonchars) disallowed */
775    case ENT_HTML_DOC_XHTML:
776    case ENT_HTML_DOC_XML1:
777        /* OTOH, XML 1.0 requires "character references to match the production for Char
778         * See <http://www.w3.org/TR/REC-xml/#NT-CharRef> */
779        return unicode_cp_is_allowed(uni_cp, document_type);
780    default:
781        return 1;
782    }
783}
784/* }}} */
785
786/* {{{ process_numeric_entity
787 * Auxiliary function to traverse_for_entities.
788 * On input, *buf should point to the first character after # and on output, it's the last
789 * byte read, no matter if there was success or insuccess.
790 */
791static inline int process_numeric_entity(const char **buf, unsigned *code_point)
792{
793    zend_long code_l;
794    int hexadecimal = (**buf == 'x' || **buf == 'X'); /* TODO: XML apparently disallows "X" */
795    char *endptr;
796
797    if (hexadecimal && (**buf != '\0'))
798        (*buf)++;
799
800    /* strtol allows whitespace and other stuff in the beginning
801        * we're not interested */
802    if ((hexadecimal && !isxdigit(**buf)) ||
803            (!hexadecimal && !isdigit(**buf))) {
804        return FAILURE;
805    }
806
807    code_l = ZEND_STRTOL(*buf, &endptr, hexadecimal ? 16 : 10);
808    /* we're guaranteed there were valid digits, so *endptr > buf */
809    *buf = endptr;
810
811    if (**buf != ';')
812        return FAILURE;
813
814    /* many more are invalid, but that depends on whether it's HTML
815     * (and which version) or XML. */
816    if (code_l > Z_L(0x10FFFF))
817        return FAILURE;
818
819    if (code_point != NULL)
820        *code_point = (unsigned)code_l;
821
822    return SUCCESS;
823}
824/* }}} */
825
826/* {{{ process_named_entity */
827static inline int process_named_entity_html(const char **buf, const char **start, size_t *length)
828{
829    *start = *buf;
830
831    /* "&" is represented by a 0x26 in all supported encodings. That means
832     * the byte after represents a character or is the leading byte of an
833     * sequence of 8-bit code units. If in the ranges below, it represents
834     * necessarily a alpha character because none of the supported encodings
835     * has an overlap with ASCII in the leading byte (only on the second one) */
836    while ((**buf >= 'a' && **buf <= 'z') ||
837            (**buf >= 'A' && **buf <= 'Z') ||
838            (**buf >= '0' && **buf <= '9')) {
839        (*buf)++;
840    }
841
842    if (**buf != ';')
843        return FAILURE;
844
845    /* cast to size_t OK as the quantity is always non-negative */
846    *length = *buf - *start;
847
848    if (*length == 0)
849        return FAILURE;
850
851    return SUCCESS;
852}
853/* }}} */
854
855/* {{{ resolve_named_entity_html */
856static inline int resolve_named_entity_html(const char *start, size_t length, const entity_ht *ht, unsigned *uni_cp1, unsigned *uni_cp2)
857{
858    const entity_cp_map *s;
859    zend_ulong hash = zend_inline_hash_func(start, length);
860
861    s = ht->buckets[hash % ht->num_elems];
862    while (s->entity) {
863        if (s->entity_len == length) {
864            if (memcmp(start, s->entity, length) == 0) {
865                *uni_cp1 = s->codepoint1;
866                *uni_cp2 = s->codepoint2;
867                return SUCCESS;
868            }
869        }
870        s++;
871    }
872    return FAILURE;
873}
874/* }}} */
875
876static inline size_t write_octet_sequence(unsigned char *buf, enum entity_charset charset, unsigned code) {
877    /* code is not necessarily a unicode code point */
878    switch (charset) {
879    case cs_utf_8:
880        return php_utf32_utf8(buf, code);
881
882    case cs_8859_1:
883    case cs_cp1252:
884    case cs_8859_15:
885    case cs_koi8r:
886    case cs_cp1251:
887    case cs_8859_5:
888    case cs_cp866:
889    case cs_macroman:
890        /* single byte stuff */
891        *buf = code;
892        return 1;
893
894    case cs_big5:
895    case cs_big5hkscs:
896    case cs_sjis:
897    case cs_gb2312:
898        /* we don't have complete unicode mappings for these yet in entity_decode,
899         * and we opt to pass through the octet sequences for these in htmlentities
900         * instead of converting to an int and then converting back. */
901#if 0
902        return php_mb2_int_to_char(buf, code);
903#else
904#if ZEND_DEBUG
905        assert(code <= 0xFFU);
906#endif
907        *buf = code;
908        return 1;
909#endif
910
911    case cs_eucjp:
912#if 0 /* idem */
913        return php_mb2_int_to_char(buf, code);
914#else
915#if ZEND_DEBUG
916        assert(code <= 0xFFU);
917#endif
918        *buf = code;
919        return 1;
920#endif
921
922    default:
923        assert(0);
924        return 0;
925    }
926}
927
928/* {{{ traverse_for_entities
929 * Auxiliary function to php_unescape_html_entities().
930 * - The argument "all" determines if all numeric entities are decode or only those
931 *   that correspond to quotes (depending on quote_style).
932 */
933/* maximum expansion (factor 1.2) for HTML 5 with &nGt; and &nLt; */
934/* +2 is 1 because of rest (probably unnecessary), 1 because of terminating 0 */
935#define TRAVERSE_FOR_ENTITIES_EXPAND_SIZE(oldlen) ((oldlen) + (oldlen) / 5 + 2)
936static void traverse_for_entities(
937    const char *old,
938    size_t oldlen,
939    zend_string *ret, /* should have allocated TRAVERSE_FOR_ENTITIES_EXPAND_SIZE(olden) */
940    int all,
941    int flags,
942    const entity_ht *inv_map,
943    enum entity_charset charset)
944{
945    const char *p,
946               *lim;
947    char       *q;
948    int doctype = flags & ENT_HTML_DOC_TYPE_MASK;
949
950    lim = old + oldlen; /* terminator address */
951    assert(*lim == '\0');
952
953    for (p = old, q = ret->val; p < lim;) {
954        unsigned code, code2 = 0;
955        const char *next = NULL; /* when set, next > p, otherwise possible inf loop */
956
957        /* Shift JIS, Big5 and HKSCS use multi-byte encodings where an
958         * ASCII range byte can be part of a multi-byte sequence.
959         * However, they start at 0x40, therefore if we find a 0x26 byte,
960         * we're sure it represents the '&' character. */
961
962        /* assumes there are no single-char entities */
963        if (p[0] != '&' || (p + 3 >= lim)) {
964            *(q++) = *(p++);
965            continue;
966        }
967
968        /* now p[3] is surely valid and is no terminator */
969
970        /* numerical entity */
971        if (p[1] == '#') {
972            next = &p[2];
973            if (process_numeric_entity(&next, &code) == FAILURE)
974                goto invalid_code;
975
976            /* If we're in htmlspecialchars_decode, we're only decoding entities
977             * that represent &, <, >, " and '. Is this one of them? */
978            if (!all && (code > 63U ||
979                    stage3_table_be_apos_00000[code].data.ent.entity == NULL))
980                goto invalid_code;
981
982            /* are we allowed to decode this entity in this document type?
983             * HTML 5 is the only that has a character that cannot be used in
984             * a numeric entity but is allowed literally (U+000D). The
985             * unoptimized version would be ... || !numeric_entity_is_allowed(code) */
986            if (!unicode_cp_is_allowed(code, doctype) ||
987                    (doctype == ENT_HTML_DOC_HTML5 && code == 0x0D))
988                goto invalid_code;
989        } else {
990            const char *start;
991            size_t ent_len;
992
993            next = &p[1];
994            start = next;
995
996            if (process_named_entity_html(&next, &start, &ent_len) == FAILURE)
997                goto invalid_code;
998
999            if (resolve_named_entity_html(start, ent_len, inv_map, &code, &code2) == FAILURE) {
1000                if (doctype == ENT_HTML_DOC_XHTML && ent_len == 4 && start[0] == 'a'
1001                            && start[1] == 'p' && start[2] == 'o' && start[3] == 's') {
1002                    /* uses html4 inv_map, which doesn't include apos;. This is a
1003                     * hack to support it */
1004                    code = (unsigned) '\'';
1005                } else {
1006                    goto invalid_code;
1007                }
1008            }
1009        }
1010
1011        assert(*next == ';');
1012
1013        if (((code == '\'' && !(flags & ENT_HTML_QUOTE_SINGLE)) ||
1014                (code == '"' && !(flags & ENT_HTML_QUOTE_DOUBLE)))
1015                /* && code2 == '\0' always true for current maps */)
1016            goto invalid_code;
1017
1018        /* UTF-8 doesn't need mapping (ISO-8859-1 doesn't either, but
1019         * the call is needed to ensure the codepoint <= U+00FF)  */
1020        if (charset != cs_utf_8) {
1021            /* replace unicode code point */
1022            if (map_from_unicode(code, charset, &code) == FAILURE || code2 != 0)
1023                goto invalid_code; /* not representable in target charset */
1024        }
1025
1026        q += write_octet_sequence((unsigned char*)q, charset, code);
1027        if (code2) {
1028            q += write_octet_sequence((unsigned char*)q, charset, code2);
1029        }
1030
1031        /* jump over the valid entity; may go beyond size of buffer; np */
1032        p = next + 1;
1033        continue;
1034
1035invalid_code:
1036        for (; p < next; p++) {
1037            *(q++) = *p;
1038        }
1039    }
1040
1041    *q = '\0';
1042    ret->len = (size_t)(q - ret->val);
1043}
1044/* }}} */
1045
1046/* {{{ unescape_inverse_map */
1047static const entity_ht *unescape_inverse_map(int all, int flags)
1048{
1049    int document_type = flags & ENT_HTML_DOC_TYPE_MASK;
1050
1051    if (all) {
1052        switch (document_type) {
1053        case ENT_HTML_DOC_HTML401:
1054        case ENT_HTML_DOC_XHTML: /* but watch out for &apos;...*/
1055            return &ent_ht_html4;
1056        case ENT_HTML_DOC_HTML5:
1057            return &ent_ht_html5;
1058        default:
1059            return &ent_ht_be_apos;
1060        }
1061    } else {
1062        switch (document_type) {
1063        case ENT_HTML_DOC_HTML401:
1064            return &ent_ht_be_noapos;
1065        default:
1066            return &ent_ht_be_apos;
1067        }
1068    }
1069}
1070/* }}} */
1071
1072/* {{{ determine_entity_table
1073 * Entity table to use. Note that entity tables are defined in terms of
1074 * unicode code points */
1075static entity_table_opt determine_entity_table(int all, int doctype)
1076{
1077    entity_table_opt retval = {NULL};
1078
1079    assert(!(doctype == ENT_HTML_DOC_XML1 && all));
1080
1081    if (all) {
1082        retval.ms_table = (doctype == ENT_HTML_DOC_HTML5) ?
1083            entity_ms_table_html5 : entity_ms_table_html4;
1084    } else {
1085        retval.table = (doctype == ENT_HTML_DOC_HTML401) ?
1086            stage3_table_be_noapos_00000 : stage3_table_be_apos_00000;
1087    }
1088    return retval;
1089}
1090/* }}} */
1091
1092/* {{{ php_unescape_html_entities
1093 * The parameter "all" should be true to decode all possible entities, false to decode
1094 * only the basic ones, i.e., those in basic_entities_ex + the numeric entities
1095 * that correspond to quotes.
1096 */
1097PHPAPI zend_string *php_unescape_html_entities(unsigned char *old, size_t oldlen, int all, int flags, char *hint_charset TSRMLS_DC)
1098{
1099    size_t retlen;
1100    zend_string *ret;
1101    enum entity_charset charset;
1102    const entity_ht *inverse_map = NULL;
1103    size_t new_size = TRAVERSE_FOR_ENTITIES_EXPAND_SIZE(oldlen);
1104
1105    if (all) {
1106        charset = determine_charset(hint_charset TSRMLS_CC);
1107    } else {
1108        charset = cs_8859_1; /* charset shouldn't matter, use ISO-8859-1 for performance */
1109    }
1110
1111    /* don't use LIMIT_ALL! */
1112
1113    if (oldlen > new_size) {
1114        /* overflow, refuse to do anything */
1115        ret = zend_string_init((char*)old, oldlen, 0);
1116        retlen = oldlen;
1117        goto empty_source;
1118    }
1119    ret = zend_string_alloc(new_size, 0);
1120    ret->val[0] = '\0';
1121    ret->len = oldlen;
1122    retlen = oldlen;
1123    if (retlen == 0) {
1124        goto empty_source;
1125    }
1126
1127    inverse_map = unescape_inverse_map(all, flags);
1128
1129    /* replace numeric entities */
1130    traverse_for_entities((char*)old, oldlen, ret, all, flags, inverse_map, charset);
1131
1132empty_source:
1133    return ret;
1134}
1135/* }}} */
1136
1137PHPAPI zend_string *php_escape_html_entities(unsigned char *old, size_t oldlen, int all, int flags, char *hint_charset TSRMLS_DC)
1138{
1139    return php_escape_html_entities_ex(old, oldlen, all, flags, hint_charset, 1 TSRMLS_CC);
1140}
1141
1142/* {{{ find_entity_for_char */
1143static inline void find_entity_for_char(
1144    unsigned int k,
1145    enum entity_charset charset,
1146    const entity_stage1_row *table,
1147    const unsigned char **entity,
1148    size_t *entity_len,
1149    unsigned char *old,
1150    size_t oldlen,
1151    size_t *cursor)
1152{
1153    unsigned stage1_idx = ENT_STAGE1_INDEX(k);
1154    const entity_stage3_row *c;
1155
1156    if (stage1_idx > 0x1D) {
1157        *entity     = NULL;
1158        *entity_len = 0;
1159        return;
1160    }
1161
1162    c = &table[stage1_idx][ENT_STAGE2_INDEX(k)][ENT_STAGE3_INDEX(k)];
1163
1164    if (!c->ambiguous) {
1165        *entity     = (const unsigned char *)c->data.ent.entity;
1166        *entity_len = c->data.ent.entity_len;
1167    } else {
1168        /* peek at next char */
1169        size_t   cursor_before  = *cursor;
1170        int      status         = SUCCESS;
1171        unsigned next_char;
1172
1173        if (!(*cursor < oldlen))
1174            goto no_suitable_2nd;
1175
1176        next_char = get_next_char(charset, old, oldlen, cursor, &status);
1177
1178        if (status == FAILURE)
1179            goto no_suitable_2nd;
1180
1181        {
1182            const entity_multicodepoint_row *s, *e;
1183
1184            s = &c->data.multicodepoint_table[1];
1185            e = s - 1 + c->data.multicodepoint_table[0].leading_entry.size;
1186            /* we could do a binary search but it's not worth it since we have
1187             * at most two entries... */
1188            for ( ; s <= e; s++) {
1189                if (s->normal_entry.second_cp == next_char) {
1190                    *entity     = s->normal_entry.entity;
1191                    *entity_len = s->normal_entry.entity_len;
1192                    return;
1193                }
1194            }
1195        }
1196no_suitable_2nd:
1197        *cursor = cursor_before;
1198        *entity = (const unsigned char *)
1199            c->data.multicodepoint_table[0].leading_entry.default_entity;
1200        *entity_len = c->data.multicodepoint_table[0].leading_entry.default_entity_len;
1201    }
1202}
1203/* }}} */
1204
1205/* {{{ find_entity_for_char_basic */
1206static inline void find_entity_for_char_basic(
1207    unsigned int k,
1208    const entity_stage3_row *table,
1209    const unsigned char **entity,
1210    size_t *entity_len)
1211{
1212    if (k >= 64U) {
1213        *entity     = NULL;
1214        *entity_len = 0;
1215        return;
1216    }
1217
1218    *entity     = table[k].data.ent.entity;
1219    *entity_len = table[k].data.ent.entity_len;
1220}
1221/* }}} */
1222
1223/* {{{ php_escape_html_entities
1224 */
1225PHPAPI zend_string *php_escape_html_entities_ex(unsigned char *old, size_t oldlen, int all, int flags, char *hint_charset, zend_bool double_encode TSRMLS_DC)
1226{
1227    size_t cursor, maxlen, len;
1228    zend_string *replaced;
1229    enum entity_charset charset = determine_charset(hint_charset TSRMLS_CC);
1230    int doctype = flags & ENT_HTML_DOC_TYPE_MASK;
1231    entity_table_opt entity_table;
1232    const enc_to_uni *to_uni_table = NULL;
1233    const entity_ht *inv_map = NULL; /* used for !double_encode */
1234    /* only used if flags includes ENT_HTML_IGNORE_ERRORS or ENT_HTML_SUBSTITUTE_DISALLOWED_CHARS */
1235    const unsigned char *replacement = NULL;
1236    size_t replacement_len = 0;
1237
1238    if (all) { /* replace with all named entities */
1239        if (CHARSET_PARTIAL_SUPPORT(charset)) {
1240            php_error_docref0(NULL TSRMLS_CC, E_STRICT, "Only basic entities "
1241                "substitution is supported for multi-byte encodings other than UTF-8; "
1242                "functionality is equivalent to htmlspecialchars");
1243        }
1244        LIMIT_ALL(all, doctype, charset);
1245    }
1246    entity_table = determine_entity_table(all, doctype);
1247    if (all && !CHARSET_UNICODE_COMPAT(charset)) {
1248        to_uni_table = enc_to_uni_index[charset];
1249    }
1250
1251    if (!double_encode) {
1252        /* first arg is 1 because we want to identify valid named entities
1253         * even if we are only encoding the basic ones */
1254        inv_map = unescape_inverse_map(1, flags);
1255    }
1256
1257    if (flags & (ENT_HTML_SUBSTITUTE_ERRORS | ENT_HTML_SUBSTITUTE_DISALLOWED_CHARS)) {
1258        if (charset == cs_utf_8) {
1259            replacement = (const unsigned char*)"\xEF\xBF\xBD";
1260            replacement_len = sizeof("\xEF\xBF\xBD") - 1;
1261        } else {
1262            replacement = (const unsigned char*)"&#xFFFD;";
1263            replacement_len = sizeof("&#xFFFD;") - 1;
1264        }
1265    }
1266
1267    /* initial estimate */
1268    if (oldlen < 64) {
1269        maxlen = 128;
1270    } else {
1271        maxlen = 2 * oldlen;
1272        if (maxlen < oldlen) {
1273            zend_error_noreturn(E_ERROR, "Input string is too long");
1274            return NULL;
1275        }
1276    }
1277
1278    replaced = zend_string_alloc(maxlen, 0);
1279    len = 0;
1280    cursor = 0;
1281    while (cursor < oldlen) {
1282        const unsigned char *mbsequence = NULL;
1283        size_t mbseqlen                 = 0,
1284               cursor_before            = cursor;
1285        int status                      = SUCCESS;
1286        unsigned int this_char          = get_next_char(charset, old, oldlen, &cursor, &status);
1287
1288        /* guarantee we have at least 40 bytes to write.
1289         * In HTML5, entities may take up to 33 bytes */
1290        if (len > maxlen - 40) { /* maxlen can never be smaller than 128 */
1291            replaced = zend_string_safe_realloc(replaced, maxlen, 1, 128, 0);
1292            maxlen += 128;
1293        }
1294
1295        if (status == FAILURE) {
1296            /* invalid MB sequence */
1297            if (flags & ENT_HTML_IGNORE_ERRORS) {
1298                continue;
1299            } else if (flags & ENT_HTML_SUBSTITUTE_ERRORS) {
1300                memcpy(&replaced->val[len], replacement, replacement_len);
1301                len += replacement_len;
1302                continue;
1303            } else {
1304                zend_string_free(replaced);
1305                return STR_EMPTY_ALLOC();
1306            }
1307        } else { /* SUCCESS */
1308            mbsequence = &old[cursor_before];
1309            mbseqlen = cursor - cursor_before;
1310        }
1311
1312        if (this_char != '&') { /* no entity on this position */
1313            const unsigned char *rep    = NULL;
1314            size_t              rep_len = 0;
1315
1316            if (((this_char == '\'' && !(flags & ENT_HTML_QUOTE_SINGLE)) ||
1317                    (this_char == '"' && !(flags & ENT_HTML_QUOTE_DOUBLE))))
1318                goto pass_char_through;
1319
1320            if (all) { /* false that CHARSET_PARTIAL_SUPPORT(charset) */
1321                if (to_uni_table != NULL) {
1322                    /* !CHARSET_UNICODE_COMPAT therefore not UTF-8; since UTF-8
1323                     * is the only multibyte encoding with !CHARSET_PARTIAL_SUPPORT,
1324                     * we're using a single byte encoding */
1325                    map_to_unicode(this_char, to_uni_table, &this_char);
1326                    if (this_char == 0xFFFF) /* no mapping; pass through */
1327                        goto pass_char_through;
1328                }
1329                /* the cursor may advance */
1330                find_entity_for_char(this_char, charset, entity_table.ms_table, &rep,
1331                    &rep_len, old, oldlen, &cursor);
1332            } else {
1333                find_entity_for_char_basic(this_char, entity_table.table, &rep, &rep_len);
1334            }
1335
1336            if (rep != NULL) {
1337                replaced->val[len++] = '&';
1338                memcpy(&replaced->val[len], rep, rep_len);
1339                len += rep_len;
1340                replaced->val[len++] = ';';
1341            } else {
1342                /* we did not find an entity for this char.
1343                 * check for its validity, if its valid pass it unchanged */
1344                if (flags & ENT_HTML_SUBSTITUTE_DISALLOWED_CHARS) {
1345                    if (CHARSET_UNICODE_COMPAT(charset)) {
1346                        if (!unicode_cp_is_allowed(this_char, doctype)) {
1347                            mbsequence = replacement;
1348                            mbseqlen = replacement_len;
1349                        }
1350                    } else if (to_uni_table) {
1351                        if (!all) /* otherwise we already did this */
1352                            map_to_unicode(this_char, to_uni_table, &this_char);
1353                        if (!unicode_cp_is_allowed(this_char, doctype)) {
1354                            mbsequence = replacement;
1355                            mbseqlen = replacement_len;
1356                        }
1357                    } else {
1358                        /* not a unicode code point, unless, coincidentally, it's in
1359                         * the 0x20..0x7D range (except 0x5C in sjis). We know nothing
1360                         * about other code points, because we have no tables. Since
1361                         * Unicode code points in that range are not disallowed in any
1362                         * document type, we could do nothing. However, conversion
1363                         * tables frequently map 0x00-0x1F to the respective C0 code
1364                         * points. Let's play it safe and admit that's the case */
1365                        if (this_char <= 0x7D &&
1366                                !unicode_cp_is_allowed(this_char, doctype)) {
1367                            mbsequence = replacement;
1368                            mbseqlen = replacement_len;
1369                        }
1370                    }
1371                }
1372pass_char_through:
1373                if (mbseqlen > 1) {
1374                    memcpy(replaced->val + len, mbsequence, mbseqlen);
1375                    len += mbseqlen;
1376                } else {
1377                    replaced->val[len++] = mbsequence[0];
1378                }
1379            }
1380        } else { /* this_char == '&' */
1381            if (double_encode) {
1382encode_amp:
1383                memcpy(&replaced->val[len], "&amp;", sizeof("&amp;") - 1);
1384                len += sizeof("&amp;") - 1;
1385            } else { /* no double encode */
1386                /* check if entity is valid */
1387                size_t ent_len; /* not counting & or ; */
1388                /* peek at next char */
1389                if (old[cursor] == '#') { /* numeric entity */
1390                    unsigned code_point;
1391                    int valid;
1392                    char *pos = (char*)&old[cursor+1];
1393                    valid = process_numeric_entity((const char **)&pos, &code_point);
1394                    if (valid == FAILURE)
1395                        goto encode_amp;
1396                    if (flags & ENT_HTML_SUBSTITUTE_DISALLOWED_CHARS) {
1397                        if (!numeric_entity_is_allowed(code_point, doctype))
1398                            goto encode_amp;
1399                    }
1400                    ent_len = pos - (char*)&old[cursor];
1401                } else { /* named entity */
1402                    /* check for vality of named entity */
1403                    const char *start = &old[cursor],
1404                               *next = start;
1405                    unsigned   dummy1, dummy2;
1406
1407                    if (process_named_entity_html(&next, &start, &ent_len) == FAILURE)
1408                        goto encode_amp;
1409                    if (resolve_named_entity_html(start, ent_len, inv_map, &dummy1, &dummy2) == FAILURE) {
1410                        if (!(doctype == ENT_HTML_DOC_XHTML && ent_len == 4 && start[0] == 'a'
1411                                    && start[1] == 'p' && start[2] == 'o' && start[3] == 's')) {
1412                            /* uses html4 inv_map, which doesn't include apos;. This is a
1413                             * hack to support it */
1414                            goto encode_amp;
1415                        }
1416                    }
1417                }
1418                /* checks passed; copy entity to result */
1419                /* entity size is unbounded, we may need more memory */
1420                /* at this point maxlen - len >= 40 */
1421                if (maxlen - len < ent_len + 2 /* & and ; */) {
1422                    /* ent_len < oldlen, which is certainly <= SIZE_MAX/2 */
1423                    replaced = zend_string_safe_realloc(replaced, maxlen, 1, ent_len + 128, 0);
1424                    maxlen += ent_len + 128;
1425                }
1426                replaced->val[len++] = '&';
1427                memcpy(&replaced->val[len], &old[cursor], ent_len);
1428                len += ent_len;
1429                replaced->val[len++] = ';';
1430                cursor += ent_len + 1;
1431            }
1432        }
1433    }
1434    replaced->val[len] = '\0';
1435    replaced->len = len;
1436
1437    return replaced;
1438}
1439/* }}} */
1440
1441/* {{{ php_html_entities
1442 */
1443static void php_html_entities(INTERNAL_FUNCTION_PARAMETERS, int all)
1444{
1445    zend_string *str, *hint_charset = NULL;
1446    char *default_charset;
1447    zend_long flags = ENT_COMPAT;
1448    zend_string *replaced;
1449    zend_bool double_encode = 1;
1450
1451#ifndef FAST_ZPP
1452    if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "S|lS!b", &str, &flags, &hint_charset, &double_encode) == FAILURE) {
1453        return;
1454    }
1455#else
1456    ZEND_PARSE_PARAMETERS_START(1, 4)
1457        Z_PARAM_STR(str)
1458        Z_PARAM_OPTIONAL
1459        Z_PARAM_LONG(flags)
1460        Z_PARAM_STR_EX(hint_charset, 1, 0)
1461        Z_PARAM_BOOL(double_encode);
1462    ZEND_PARSE_PARAMETERS_END();
1463#endif
1464
1465    if (!hint_charset) {
1466        default_charset = get_default_charset(TSRMLS_C);
1467    }
1468    replaced = php_escape_html_entities_ex((unsigned char*)str->val, str->len, all, (int) flags, (hint_charset ? hint_charset->val : default_charset), double_encode TSRMLS_CC);
1469    RETVAL_STR(replaced);
1470}
1471/* }}} */
1472
1473#define HTML_SPECIALCHARS   0
1474#define HTML_ENTITIES       1
1475
1476/* {{{ register_html_constants
1477 */
1478void register_html_constants(INIT_FUNC_ARGS)
1479{
1480    REGISTER_LONG_CONSTANT("HTML_SPECIALCHARS", HTML_SPECIALCHARS, CONST_PERSISTENT|CONST_CS);
1481    REGISTER_LONG_CONSTANT("HTML_ENTITIES", HTML_ENTITIES, CONST_PERSISTENT|CONST_CS);
1482    REGISTER_LONG_CONSTANT("ENT_COMPAT", ENT_COMPAT, CONST_PERSISTENT|CONST_CS);
1483    REGISTER_LONG_CONSTANT("ENT_QUOTES", ENT_QUOTES, CONST_PERSISTENT|CONST_CS);
1484    REGISTER_LONG_CONSTANT("ENT_NOQUOTES", ENT_NOQUOTES, CONST_PERSISTENT|CONST_CS);
1485    REGISTER_LONG_CONSTANT("ENT_IGNORE", ENT_IGNORE, CONST_PERSISTENT|CONST_CS);
1486    REGISTER_LONG_CONSTANT("ENT_SUBSTITUTE", ENT_SUBSTITUTE, CONST_PERSISTENT|CONST_CS);
1487    REGISTER_LONG_CONSTANT("ENT_DISALLOWED", ENT_DISALLOWED, CONST_PERSISTENT|CONST_CS);
1488    REGISTER_LONG_CONSTANT("ENT_HTML401", ENT_HTML401, CONST_PERSISTENT|CONST_CS);
1489    REGISTER_LONG_CONSTANT("ENT_XML1", ENT_XML1, CONST_PERSISTENT|CONST_CS);
1490    REGISTER_LONG_CONSTANT("ENT_XHTML", ENT_XHTML, CONST_PERSISTENT|CONST_CS);
1491    REGISTER_LONG_CONSTANT("ENT_HTML5", ENT_HTML5, CONST_PERSISTENT|CONST_CS);
1492}
1493/* }}} */
1494
1495/* {{{ proto string htmlspecialchars(string string [, int quote_style[, string charset[, bool double_encode]]])
1496   Convert special characters to HTML entities */
1497PHP_FUNCTION(htmlspecialchars)
1498{
1499    php_html_entities(INTERNAL_FUNCTION_PARAM_PASSTHRU, 0);
1500}
1501/* }}} */
1502
1503/* {{{ proto string htmlspecialchars_decode(string string [, int quote_style])
1504   Convert special HTML entities back to characters */
1505PHP_FUNCTION(htmlspecialchars_decode)
1506{
1507    char *str;
1508    size_t str_len;
1509    zend_long quote_style = ENT_COMPAT;
1510    zend_string *replaced;
1511
1512    if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "s|l", &str, &str_len, &quote_style) == FAILURE) {
1513        return;
1514    }
1515
1516    replaced = php_unescape_html_entities((unsigned char*)str, str_len, 0 /*!all*/, quote_style, NULL TSRMLS_CC);
1517    if (replaced) {
1518        RETURN_STR(replaced);
1519    }
1520    RETURN_FALSE;
1521}
1522/* }}} */
1523
1524/* {{{ proto string html_entity_decode(string string [, int quote_style][, string charset])
1525   Convert all HTML entities to their applicable characters */
1526PHP_FUNCTION(html_entity_decode)
1527{
1528    zend_string *str, *hint_charset = NULL;
1529    char *default_charset;
1530    zend_long quote_style = ENT_COMPAT;
1531    zend_string *replaced;
1532
1533#ifndef FAST_ZPP
1534    if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "S|lS", &str,
1535                              &quote_style, &hint_charset) == FAILURE) {
1536        return;
1537    }
1538#else
1539    ZEND_PARSE_PARAMETERS_START(1, 3)
1540        Z_PARAM_STR(str)
1541        Z_PARAM_OPTIONAL
1542        Z_PARAM_LONG(quote_style)
1543        Z_PARAM_STR(hint_charset)
1544    ZEND_PARSE_PARAMETERS_END();
1545#endif
1546
1547    if (!hint_charset) {
1548        default_charset = get_default_charset(TSRMLS_C);
1549    }
1550    replaced = php_unescape_html_entities((unsigned char*)str->val, str->len, 1 /*all*/, quote_style, (hint_charset ? hint_charset->val : default_charset) TSRMLS_CC);
1551
1552    if (replaced) {
1553        RETURN_STR(replaced);
1554    }
1555    RETURN_FALSE;
1556}
1557/* }}} */
1558
1559
1560/* {{{ proto string htmlentities(string string [, int quote_style[, string charset[, bool double_encode]]])
1561   Convert all applicable characters to HTML entities */
1562PHP_FUNCTION(htmlentities)
1563{
1564    php_html_entities(INTERNAL_FUNCTION_PARAM_PASSTHRU, 1);
1565}
1566/* }}} */
1567
1568/* {{{ write_s3row_data */
1569static inline void write_s3row_data(
1570    const entity_stage3_row *r,
1571    unsigned orig_cp,
1572    enum entity_charset charset,
1573    zval *arr)
1574{
1575    char key[9] = ""; /* two unicode code points in UTF-8 */
1576    char entity[LONGEST_ENTITY_LENGTH + 2] = {'&'};
1577    size_t written_k1;
1578
1579    written_k1 = write_octet_sequence((unsigned char*)key, charset, orig_cp);
1580
1581    if (!r->ambiguous) {
1582        size_t l = r->data.ent.entity_len;
1583        memcpy(&entity[1], r->data.ent.entity, l);
1584        entity[l + 1] = ';';
1585        add_assoc_stringl_ex(arr, key, written_k1, entity, l + 2);
1586    } else {
1587        unsigned i,
1588                 num_entries;
1589        const entity_multicodepoint_row *mcpr = r->data.multicodepoint_table;
1590
1591        if (mcpr[0].leading_entry.default_entity != NULL) {
1592            size_t l = mcpr[0].leading_entry.default_entity_len;
1593            memcpy(&entity[1], mcpr[0].leading_entry.default_entity, l);
1594            entity[l + 1] = ';';
1595            add_assoc_stringl_ex(arr, key, written_k1, entity, l + 2);
1596        }
1597        num_entries = mcpr[0].leading_entry.size;
1598        for (i = 1; i <= num_entries; i++) {
1599            size_t   l,
1600                     written_k2;
1601            unsigned uni_cp,
1602                     spe_cp;
1603
1604            uni_cp = mcpr[i].normal_entry.second_cp;
1605            l = mcpr[i].normal_entry.entity_len;
1606
1607            if (!CHARSET_UNICODE_COMPAT(charset)) {
1608                if (map_from_unicode(uni_cp, charset, &spe_cp) == FAILURE)
1609                    continue; /* non representable in this charset */
1610            } else {
1611                spe_cp = uni_cp;
1612            }
1613
1614            written_k2 = write_octet_sequence((unsigned char*)&key[written_k1], charset, spe_cp);
1615            memcpy(&entity[1], mcpr[i].normal_entry.entity, l);
1616            entity[l + 1] = ';';
1617            entity[l + 1] = '\0';
1618            add_assoc_stringl_ex(arr, key, written_k1 + written_k2, entity, l + 1);
1619        }
1620    }
1621}
1622/* }}} */
1623
1624/* {{{ proto array get_html_translation_table([int table [, int flags [, string charset_hint]]])
1625   Returns the internal translation table used by htmlspecialchars and htmlentities */
1626PHP_FUNCTION(get_html_translation_table)
1627{
1628    zend_long all = HTML_SPECIALCHARS,
1629         flags = ENT_COMPAT;
1630    int doctype;
1631    entity_table_opt entity_table;
1632    const enc_to_uni *to_uni_table = NULL;
1633    char *charset_hint = NULL;
1634    size_t charset_hint_len;
1635    enum entity_charset charset;
1636
1637    /* in this function we have to jump through some loops because we're
1638     * getting the translated table from data structures that are optimized for
1639     * random access, not traversal */
1640
1641    if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "|lls",
1642            &all, &flags, &charset_hint, &charset_hint_len) == FAILURE) {
1643        return;
1644    }
1645
1646    charset = determine_charset(charset_hint TSRMLS_CC);
1647    doctype = flags & ENT_HTML_DOC_TYPE_MASK;
1648    LIMIT_ALL(all, doctype, charset);
1649
1650    array_init(return_value);
1651
1652    entity_table = determine_entity_table(all, doctype);
1653    if (all && !CHARSET_UNICODE_COMPAT(charset)) {
1654        to_uni_table = enc_to_uni_index[charset];
1655    }
1656
1657    if (all) { /* HTML_ENTITIES (actually, any non-zero value for 1st param) */
1658        const entity_stage1_row *ms_table = entity_table.ms_table;
1659
1660        if (CHARSET_UNICODE_COMPAT(charset)) {
1661            unsigned i, j, k,
1662                     max_i, max_j, max_k;
1663            /* no mapping to unicode required */
1664            if (CHARSET_SINGLE_BYTE(charset)) { /* ISO-8859-1 */
1665                max_i = 1; max_j = 4; max_k = 64;
1666            } else {
1667                max_i = 0x1E; max_j = 64; max_k = 64;
1668            }
1669
1670            for (i = 0; i < max_i; i++) {
1671                if (ms_table[i] == empty_stage2_table)
1672                    continue;
1673                for (j = 0; j < max_j; j++) {
1674                    if (ms_table[i][j] == empty_stage3_table)
1675                        continue;
1676                    for (k = 0; k < max_k; k++) {
1677                        const entity_stage3_row *r = &ms_table[i][j][k];
1678                        unsigned code;
1679
1680                        if (r->data.ent.entity == NULL)
1681                            continue;
1682
1683                        code = ENT_CODE_POINT_FROM_STAGES(i, j, k);
1684                        if (((code == '\'' && !(flags & ENT_HTML_QUOTE_SINGLE)) ||
1685                                (code == '"' && !(flags & ENT_HTML_QUOTE_DOUBLE))))
1686                            continue;
1687                        write_s3row_data(r, code, charset, return_value);
1688                    }
1689                }
1690            }
1691        } else {
1692            /* we have to iterate through the set of code points for this
1693             * encoding and map them to unicode code points */
1694            unsigned i;
1695            for (i = 0; i <= 0xFF; i++) {
1696                const entity_stage3_row *r;
1697                unsigned uni_cp;
1698
1699                /* can be done before mapping, they're invariant */
1700                if (((i == '\'' && !(flags & ENT_HTML_QUOTE_SINGLE)) ||
1701                        (i == '"' && !(flags & ENT_HTML_QUOTE_DOUBLE))))
1702                    continue;
1703
1704                map_to_unicode(i, to_uni_table, &uni_cp);
1705                r = &ms_table[ENT_STAGE1_INDEX(uni_cp)][ENT_STAGE2_INDEX(uni_cp)][ENT_STAGE3_INDEX(uni_cp)];
1706                if (r->data.ent.entity == NULL)
1707                    continue;
1708
1709                write_s3row_data(r, i, charset, return_value);
1710            }
1711        }
1712    } else {
1713        /* we could use sizeof(stage3_table_be_apos_00000) as well */
1714        unsigned      j,
1715                      numelems = sizeof(stage3_table_be_noapos_00000) /
1716                            sizeof(*stage3_table_be_noapos_00000);
1717
1718        for (j = 0; j < numelems; j++) {
1719            const entity_stage3_row *r = &entity_table.table[j];
1720            if (r->data.ent.entity == NULL)
1721                continue;
1722
1723            if (((j == '\'' && !(flags & ENT_HTML_QUOTE_SINGLE)) ||
1724                    (j == '"' && !(flags & ENT_HTML_QUOTE_DOUBLE))))
1725                continue;
1726
1727            /* charset is indifferent, used cs_8859_1 for efficiency */
1728            write_s3row_data(r, j, cs_8859_1, return_value);
1729        }
1730    }
1731}
1732/* }}} */
1733
1734/*
1735 * Local variables:
1736 * tab-width: 4
1737 * c-basic-offset: 4
1738 * End:
1739 * vim600: sw=4 ts=4 fdm=marker
1740 * vim<600: sw=4 ts=4
1741 */
1742