1/*
2   +----------------------------------------------------------------------+
3   | PHP Version 7                                                        |
4   +----------------------------------------------------------------------+
5   | Copyright (c) 1997-2016 The PHP Group                                |
6   +----------------------------------------------------------------------+
7   | This source file is subject to version 3.01 of the PHP license,      |
8   | that is bundled with this package in the file LICENSE, and is        |
9   | available through the world-wide-web at the following url:           |
10   | http://www.php.net/license/3_01.txt                                  |
11   | If you did not receive a copy of the PHP license and are unable to   |
12   | obtain it through the world-wide-web, please send a note to          |
13   | license@php.net so we can mail you a copy immediately.               |
14   +----------------------------------------------------------------------+
15   | Authors: Rasmus Lerdorf <rasmus@php.net>                             |
16   |          Jaakko Hyvätti <jaakko.hyvatti@iki.fi>                      |
17   |          Wez Furlong    <wez@thebrainroom.com>                       |
18   |          Gustavo Lopes  <cataphract@php.net>                         |
19   +----------------------------------------------------------------------+
20*/
21
22/* $Id$ */
23
24/*
25 * HTML entity resources:
26 *
27 * http://www.unicode.org/Public/MAPPINGS/OBSOLETE/UNI2SGML.TXT
28 *
29 * XHTML 1.0 DTD
30 * http://www.w3.org/TR/2002/REC-xhtml1-20020801/dtds.html#h-A2
31 *
32 * From HTML 4.01 strict DTD:
33 * http://www.w3.org/TR/html4/HTMLlat1.ent
34 * http://www.w3.org/TR/html4/HTMLsymbol.ent
35 * http://www.w3.org/TR/html4/HTMLspecial.ent
36 *
37 * HTML 5:
38 * http://dev.w3.org/html5/spec/Overview.html#named-character-references
39 */
40
41#include "php.h"
42#if PHP_WIN32
43#include "config.w32.h"
44#else
45#include <php_config.h>
46#endif
47#include "php_standard.h"
48#include "php_string.h"
49#include "SAPI.h"
50#if HAVE_LOCALE_H
51#include <locale.h>
52#endif
53#if HAVE_LANGINFO_H
54#include <langinfo.h>
55#endif
56
57#include <zend_hash.h>
58#include "html_tables.h"
59
60/* Macro for disabling flag of translation of non-basic entities where this isn't supported.
61 * Not appropriate for html_entity_decode/htmlspecialchars_decode */
62#define LIMIT_ALL(all, doctype, charset) do { \
63	(all) = (all) && !CHARSET_PARTIAL_SUPPORT((charset)) && ((doctype) != ENT_HTML_DOC_XML1); \
64} while (0)
65
66#define MB_FAILURE(pos, advance) do { \
67	*cursor = pos + (advance); \
68	*status = FAILURE; \
69	return 0; \
70} while (0)
71
72#define CHECK_LEN(pos, chars_need) ((str_len - (pos)) >= (chars_need))
73
74/* valid as single byte character or leading byte */
75#define utf8_lead(c)  ((c) < 0x80 || ((c) >= 0xC2 && (c) <= 0xF4))
76/* whether it's actually valid depends on other stuff;
77 * this macro cannot check for non-shortest forms, surrogates or
78 * code points above 0x10FFFF */
79#define utf8_trail(c) ((c) >= 0x80 && (c) <= 0xBF)
80
81#define gb2312_lead(c) ((c) != 0x8E && (c) != 0x8F && (c) != 0xA0 && (c) != 0xFF)
82#define gb2312_trail(c) ((c) >= 0xA1 && (c) <= 0xFE)
83
84#define sjis_lead(c) ((c) != 0x80 && (c) != 0xA0 && (c) < 0xFD)
85#define sjis_trail(c) ((c) >= 0x40  && (c) != 0x7F && (c) < 0xFD)
86
87/* {{{ get_default_charset
88 */
89static char *get_default_charset(void) {
90	if (PG(internal_encoding) && PG(internal_encoding)[0]) {
91		return PG(internal_encoding);
92	} else if (SG(default_charset) && SG(default_charset)[0] ) {
93		return SG(default_charset);
94	}
95	return NULL;
96}
97/* }}} */
98
99/* {{{ get_next_char
100 */
101static inline unsigned int get_next_char(
102		enum entity_charset charset,
103		const unsigned char *str,
104		size_t str_len,
105		size_t *cursor,
106		int *status)
107{
108	size_t pos = *cursor;
109	unsigned int this_char = 0;
110
111	*status = SUCCESS;
112	assert(pos <= str_len);
113
114	if (!CHECK_LEN(pos, 1))
115		MB_FAILURE(pos, 1);
116
117	switch (charset) {
118	case cs_utf_8:
119		{
120			/* We'll follow strategy 2. from section 3.6.1 of UTR #36:
121			 * "In a reported illegal byte sequence, do not include any
122			 *  non-initial byte that encodes a valid character or is a leading
123			 *  byte for a valid sequence." */
124			unsigned char c;
125			c = str[pos];
126			if (c < 0x80) {
127				this_char = c;
128				pos++;
129			} else if (c < 0xc2) {
130				MB_FAILURE(pos, 1);
131			} else if (c < 0xe0) {
132				if (!CHECK_LEN(pos, 2))
133					MB_FAILURE(pos, 1);
134
135				if (!utf8_trail(str[pos + 1])) {
136					MB_FAILURE(pos, utf8_lead(str[pos + 1]) ? 1 : 2);
137				}
138				this_char = ((c & 0x1f) << 6) | (str[pos + 1] & 0x3f);
139				if (this_char < 0x80) { /* non-shortest form */
140					MB_FAILURE(pos, 2);
141				}
142				pos += 2;
143			} else if (c < 0xf0) {
144				size_t avail = str_len - pos;
145
146				if (avail < 3 ||
147						!utf8_trail(str[pos + 1]) || !utf8_trail(str[pos + 2])) {
148					if (avail < 2 || utf8_lead(str[pos + 1]))
149						MB_FAILURE(pos, 1);
150					else if (avail < 3 || utf8_lead(str[pos + 2]))
151						MB_FAILURE(pos, 2);
152					else
153						MB_FAILURE(pos, 3);
154				}
155
156				this_char = ((c & 0x0f) << 12) | ((str[pos + 1] & 0x3f) << 6) | (str[pos + 2] & 0x3f);
157				if (this_char < 0x800) { /* non-shortest form */
158					MB_FAILURE(pos, 3);
159				} else if (this_char >= 0xd800 && this_char <= 0xdfff) { /* surrogate */
160					MB_FAILURE(pos, 3);
161				}
162				pos += 3;
163			} else if (c < 0xf5) {
164				size_t avail = str_len - pos;
165
166				if (avail < 4 ||
167						!utf8_trail(str[pos + 1]) || !utf8_trail(str[pos + 2]) ||
168						!utf8_trail(str[pos + 3])) {
169					if (avail < 2 || utf8_lead(str[pos + 1]))
170						MB_FAILURE(pos, 1);
171					else if (avail < 3 || utf8_lead(str[pos + 2]))
172						MB_FAILURE(pos, 2);
173					else if (avail < 4 || utf8_lead(str[pos + 3]))
174						MB_FAILURE(pos, 3);
175					else
176						MB_FAILURE(pos, 4);
177				}
178
179				this_char = ((c & 0x07) << 18) | ((str[pos + 1] & 0x3f) << 12) | ((str[pos + 2] & 0x3f) << 6) | (str[pos + 3] & 0x3f);
180				if (this_char < 0x10000 || this_char > 0x10FFFF) { /* non-shortest form or outside range */
181					MB_FAILURE(pos, 4);
182				}
183				pos += 4;
184			} else {
185				MB_FAILURE(pos, 1);
186			}
187		}
188		break;
189
190	case cs_big5:
191		/* reference http://demo.icu-project.org/icu-bin/convexp?conv=big5 */
192		{
193			unsigned char c = str[pos];
194			if (c >= 0x81 && c <= 0xFE) {
195				unsigned char next;
196				if (!CHECK_LEN(pos, 2))
197					MB_FAILURE(pos, 1);
198
199				next = str[pos + 1];
200
201				if ((next >= 0x40 && next <= 0x7E) ||
202						(next >= 0xA1 && next <= 0xFE)) {
203					this_char = (c << 8) | next;
204				} else {
205					MB_FAILURE(pos, 1);
206				}
207				pos += 2;
208			} else {
209				this_char = c;
210				pos += 1;
211			}
212		}
213		break;
214
215	case cs_big5hkscs:
216		{
217			unsigned char c = str[pos];
218			if (c >= 0x81 && c <= 0xFE) {
219				unsigned char next;
220				if (!CHECK_LEN(pos, 2))
221					MB_FAILURE(pos, 1);
222
223				next = str[pos + 1];
224
225				if ((next >= 0x40 && next <= 0x7E) ||
226						(next >= 0xA1 && next <= 0xFE)) {
227					this_char = (c << 8) | next;
228				} else if (next != 0x80 && next != 0xFF) {
229					MB_FAILURE(pos, 1);
230				} else {
231					MB_FAILURE(pos, 2);
232				}
233				pos += 2;
234			} else {
235				this_char = c;
236				pos += 1;
237			}
238		}
239		break;
240
241	case cs_gb2312: /* EUC-CN */
242		{
243			unsigned char c = str[pos];
244			if (c >= 0xA1 && c <= 0xFE) {
245				unsigned char next;
246				if (!CHECK_LEN(pos, 2))
247					MB_FAILURE(pos, 1);
248
249				next = str[pos + 1];
250
251				if (gb2312_trail(next)) {
252					this_char = (c << 8) | next;
253				} else if (gb2312_lead(next)) {
254					MB_FAILURE(pos, 1);
255				} else {
256					MB_FAILURE(pos, 2);
257				}
258				pos += 2;
259			} else if (gb2312_lead(c)) {
260				this_char = c;
261				pos += 1;
262			} else {
263				MB_FAILURE(pos, 1);
264			}
265		}
266		break;
267
268	case cs_sjis:
269		{
270			unsigned char c = str[pos];
271			if ((c >= 0x81 && c <= 0x9F) || (c >= 0xE0 && c <= 0xFC)) {
272				unsigned char next;
273				if (!CHECK_LEN(pos, 2))
274					MB_FAILURE(pos, 1);
275
276				next = str[pos + 1];
277
278				if (sjis_trail(next)) {
279					this_char = (c << 8) | next;
280				} else if (sjis_lead(next)) {
281					MB_FAILURE(pos, 1);
282				} else {
283					MB_FAILURE(pos, 2);
284				}
285				pos += 2;
286			} else if (c < 0x80 || (c >= 0xA1 && c <= 0xDF)) {
287				this_char = c;
288				pos += 1;
289			} else {
290				MB_FAILURE(pos, 1);
291			}
292		}
293		break;
294
295	case cs_eucjp:
296		{
297			unsigned char c = str[pos];
298
299			if (c >= 0xA1 && c <= 0xFE) {
300				unsigned next;
301				if (!CHECK_LEN(pos, 2))
302					MB_FAILURE(pos, 1);
303				next = str[pos + 1];
304
305				if (next >= 0xA1 && next <= 0xFE) {
306					/* this a jis kanji char */
307					this_char = (c << 8) | next;
308				} else {
309					MB_FAILURE(pos, (next != 0xA0 && next != 0xFF) ? 1 : 2);
310				}
311				pos += 2;
312			} else if (c == 0x8E) {
313				unsigned next;
314				if (!CHECK_LEN(pos, 2))
315					MB_FAILURE(pos, 1);
316
317				next = str[pos + 1];
318				if (next >= 0xA1 && next <= 0xDF) {
319					/* JIS X 0201 kana */
320					this_char = (c << 8) | next;
321				} else {
322					MB_FAILURE(pos, (next != 0xA0 && next != 0xFF) ? 1 : 2);
323				}
324				pos += 2;
325			} else if (c == 0x8F) {
326				size_t avail = str_len - pos;
327
328				if (avail < 3 || !(str[pos + 1] >= 0xA1 && str[pos + 1] <= 0xFE) ||
329						!(str[pos + 2] >= 0xA1 && str[pos + 2] <= 0xFE)) {
330					if (avail < 2 || (str[pos + 1] != 0xA0 && str[pos + 1] != 0xFF))
331						MB_FAILURE(pos, 1);
332					else if (avail < 3 || (str[pos + 2] != 0xA0 && str[pos + 2] != 0xFF))
333						MB_FAILURE(pos, 2);
334					else
335						MB_FAILURE(pos, 3);
336				} else {
337					/* JIS X 0212 hojo-kanji */
338					this_char = (c << 16) | (str[pos + 1] << 8) | str[pos + 2];
339				}
340				pos += 3;
341			} else if (c != 0xA0 && c != 0xFF) {
342				/* character encoded in 1 code unit */
343				this_char = c;
344				pos += 1;
345			} else {
346				MB_FAILURE(pos, 1);
347			}
348		}
349		break;
350	default:
351		/* single-byte charsets */
352		this_char = str[pos++];
353		break;
354	}
355
356	*cursor = pos;
357  	return this_char;
358}
359/* }}} */
360
361/* {{{ php_next_utf8_char
362 * Public interface for get_next_char used with UTF-8 */
363 PHPAPI unsigned int php_next_utf8_char(
364		const unsigned char *str,
365		size_t str_len,
366		size_t *cursor,
367		int *status)
368{
369	return get_next_char(cs_utf_8, str, str_len, cursor, status);
370}
371/* }}} */
372
373/* {{{ entity_charset determine_charset
374 * returns the charset identifier based on current locale or a hint.
375 * defaults to UTF-8 */
376static enum entity_charset determine_charset(char *charset_hint)
377{
378	int i;
379	enum entity_charset charset = cs_utf_8;
380	size_t len = 0;
381	const zend_encoding *zenc;
382
383	/* Default is now UTF-8 */
384	if (charset_hint == NULL)
385		return cs_utf_8;
386
387	if ((len = strlen(charset_hint)) != 0) {
388		goto det_charset;
389	}
390
391	zenc = zend_multibyte_get_internal_encoding();
392	if (zenc != NULL) {
393		charset_hint = (char *)zend_multibyte_get_encoding_name(zenc);
394		if (charset_hint != NULL && (len=strlen(charset_hint)) != 0) {
395			if ((len == 4) /* sizeof (auto|pass) */ &&
396					/* XXX should the "wchar" be ignored as well?? */
397					(!memcmp("pass", charset_hint, 4) ||
398					 !memcmp("auto", charset_hint, 4))) {
399				charset_hint = NULL;
400				len = 0;
401			} else {
402				goto det_charset;
403			}
404		}
405	}
406
407	charset_hint = SG(default_charset);
408	if (charset_hint != NULL && (len=strlen(charset_hint)) != 0) {
409		goto det_charset;
410	}
411
412	/* try to detect the charset for the locale */
413#if HAVE_NL_LANGINFO && HAVE_LOCALE_H && defined(CODESET)
414	charset_hint = nl_langinfo(CODESET);
415	if (charset_hint != NULL && (len=strlen(charset_hint)) != 0) {
416		goto det_charset;
417	}
418#endif
419
420#if HAVE_LOCALE_H
421	/* try to figure out the charset from the locale */
422	{
423		char *localename;
424		char *dot, *at;
425
426		/* lang[_territory][.codeset][@modifier] */
427		localename = setlocale(LC_CTYPE, NULL);
428
429		dot = strchr(localename, '.');
430		if (dot) {
431			dot++;
432			/* locale specifies a codeset */
433			at = strchr(dot, '@');
434			if (at)
435				len = at - dot;
436			else
437				len = strlen(dot);
438			charset_hint = dot;
439		} else {
440			/* no explicit name; see if the name itself
441			 * is the charset */
442			charset_hint = localename;
443			len = strlen(charset_hint);
444		}
445	}
446#endif
447
448det_charset:
449
450	if (charset_hint) {
451		int found = 0;
452
453		/* now walk the charset map and look for the codeset */
454		for (i = 0; i < sizeof(charset_map)/sizeof(charset_map[0]); i++) {
455			if (len == charset_map[i].codeset_len &&
456			    zend_binary_strcasecmp(charset_hint, len, charset_map[i].codeset, len) == 0) {
457				charset = charset_map[i].charset;
458				found = 1;
459				break;
460			}
461		}
462		if (!found) {
463			php_error_docref(NULL, E_WARNING, "charset `%s' not supported, assuming utf-8",
464					charset_hint);
465		}
466	}
467	return charset;
468}
469/* }}} */
470
471/* {{{ php_utf32_utf8 */
472static inline size_t php_utf32_utf8(unsigned char *buf, unsigned k)
473{
474	size_t retval = 0;
475
476	/* assert(0x0 <= k <= 0x10FFFF); */
477
478	if (k < 0x80) {
479		buf[0] = k;
480		retval = 1;
481	} else if (k < 0x800) {
482		buf[0] = 0xc0 | (k >> 6);
483		buf[1] = 0x80 | (k & 0x3f);
484		retval = 2;
485	} else if (k < 0x10000) {
486		buf[0] = 0xe0 | (k >> 12);
487		buf[1] = 0x80 | ((k >> 6) & 0x3f);
488		buf[2] = 0x80 | (k & 0x3f);
489		retval = 3;
490	} else {
491		buf[0] = 0xf0 | (k >> 18);
492		buf[1] = 0x80 | ((k >> 12) & 0x3f);
493		buf[2] = 0x80 | ((k >> 6) & 0x3f);
494		buf[3] = 0x80 | (k & 0x3f);
495		retval = 4;
496	}
497	/* UTF-8 has been restricted to max 4 bytes since RFC 3629 */
498
499	return retval;
500}
501/* }}} */
502
503/* {{{ php_mb2_int_to_char
504 * Convert back big endian int representation of sequence of one or two 8-bit code units. */
505static inline size_t php_mb2_int_to_char(unsigned char *buf, unsigned k)
506{
507	assert(k <= 0xFFFFU);
508	/* one or two bytes */
509	if (k <= 0xFFU) { /* 1 */
510		buf[0] = k;
511		return 1U;
512	} else { /* 2 */
513		buf[0] = k >> 8;
514		buf[1] = k & 0xFFU;
515		return 2U;
516	}
517}
518/* }}} */
519
520/* {{{ php_mb3_int_to_char
521 * Convert back big endian int representation of sequence of one to three 8-bit code units.
522 * For EUC-JP. */
523static inline size_t php_mb3_int_to_char(unsigned char *buf, unsigned k)
524{
525	assert(k <= 0xFFFFFFU);
526	/* one to three bytes */
527	if (k <= 0xFFU) { /* 1 */
528		buf[0] = k;
529		return 1U;
530	} else if (k <= 0xFFFFU) { /* 2 */
531		buf[0] = k >> 8;
532		buf[1] = k & 0xFFU;
533		return 2U;
534	} else {
535		buf[0] = k >> 16;
536		buf[1] = (k >> 8) & 0xFFU;
537		buf[2] = k & 0xFFU;
538		return 3U;
539	}
540}
541/* }}} */
542
543
544/* {{{ unimap_bsearc_cmp
545 * Binary search of unicode code points in unicode <--> charset mapping.
546 * Returns the code point in the target charset (whose mapping table was given) or 0 if
547 * the unicode code point is not in the table.
548 */
549static inline unsigned char unimap_bsearch(const uni_to_enc *table, unsigned code_key_a, size_t num)
550{
551	const uni_to_enc *l = table,
552					 *h = &table[num-1],
553					 *m;
554	unsigned short code_key;
555
556	/* we have no mappings outside the BMP */
557	if (code_key_a > 0xFFFFU)
558		return 0;
559
560	code_key = (unsigned short) code_key_a;
561
562	while (l <= h) {
563		m = l + (h - l) / 2;
564		if (code_key < m->un_code_point)
565			h = m - 1;
566		else if (code_key > m->un_code_point)
567			l = m + 1;
568		else
569			return m->cs_code;
570	}
571	return 0;
572}
573/* }}} */
574
575/* {{{ map_from_unicode */
576static inline int map_from_unicode(unsigned code, enum entity_charset charset, unsigned *res)
577{
578	unsigned char found;
579	const uni_to_enc *table;
580	size_t table_size;
581
582	switch (charset) {
583	case cs_8859_1:
584		/* identity mapping of code points to unicode */
585		if (code > 0xFF) {
586			return FAILURE;
587		}
588		*res = code;
589		break;
590
591	case cs_8859_5:
592		if (code <= 0xA0 || code == 0xAD /* soft hyphen */) {
593			*res = code;
594		} else if (code == 0x2116) {
595			*res = 0xF0; /* numero sign */
596		} else if (code == 0xA7) {
597			*res = 0xFD; /* section sign */
598		} else if (code >= 0x0401 && code <= 0x044F) {
599			if (code == 0x040D || code == 0x0450 || code == 0x045D)
600				return FAILURE;
601			*res = code - 0x360;
602		} else {
603			return FAILURE;
604		}
605		break;
606
607	case cs_8859_15:
608		if (code < 0xA4 || (code > 0xBE && code <= 0xFF)) {
609			*res = code;
610		} else { /* between A4 and 0xBE */
611			found = unimap_bsearch(unimap_iso885915,
612				code, sizeof(unimap_iso885915) / sizeof(*unimap_iso885915));
613			if (found)
614				*res = found;
615			else
616				return FAILURE;
617		}
618		break;
619
620	case cs_cp1252:
621		if (code <= 0x7F || (code >= 0xA0 && code <= 0xFF)) {
622			*res = code;
623		} else {
624			found = unimap_bsearch(unimap_win1252,
625				code, sizeof(unimap_win1252) / sizeof(*unimap_win1252));
626			if (found)
627				*res = found;
628			else
629				return FAILURE;
630		}
631		break;
632
633	case cs_macroman:
634		if (code == 0x7F)
635			return FAILURE;
636		table = unimap_macroman;
637		table_size = sizeof(unimap_macroman) / sizeof(*unimap_macroman);
638		goto table_over_7F;
639	case cs_cp1251:
640		table = unimap_win1251;
641		table_size = sizeof(unimap_win1251) / sizeof(*unimap_win1251);
642		goto table_over_7F;
643	case cs_koi8r:
644		table = unimap_koi8r;
645		table_size = sizeof(unimap_koi8r) / sizeof(*unimap_koi8r);
646		goto table_over_7F;
647	case cs_cp866:
648		table = unimap_cp866;
649		table_size = sizeof(unimap_cp866) / sizeof(*unimap_cp866);
650
651table_over_7F:
652		if (code <= 0x7F) {
653			*res = code;
654		} else {
655			found = unimap_bsearch(table, code, table_size);
656			if (found)
657				*res = found;
658			else
659				return FAILURE;
660		}
661		break;
662
663	/* from here on, only map the possible characters in the ASCII range.
664	 * to improve support here, it's a matter of building the unicode mappings.
665	 * See <http://www.unicode.org/Public/6.0.0/ucd/Unihan.zip> */
666	case cs_sjis:
667	case cs_eucjp:
668		/* we interpret 0x5C as the Yen symbol. This is not universal.
669		 * See <http://www.w3.org/Submission/japanese-xml/#ambiguity_of_yen> */
670		if (code >= 0x20 && code <= 0x7D) {
671			if (code == 0x5C)
672				return FAILURE;
673			*res = code;
674		} else {
675			return FAILURE;
676		}
677		break;
678
679	case cs_big5:
680	case cs_big5hkscs:
681	case cs_gb2312:
682		if (code >= 0x20 && code <= 0x7D) {
683			*res = code;
684		} else {
685			return FAILURE;
686		}
687		break;
688
689	default:
690		return FAILURE;
691	}
692
693	return SUCCESS;
694}
695/* }}} */
696
697/* {{{ */
698static inline void map_to_unicode(unsigned code, const enc_to_uni *table, unsigned *res)
699{
700	/* only single byte encodings are currently supported; assumed code <= 0xFF */
701	*res = table->inner[ENT_ENC_TO_UNI_STAGE1(code)]->uni_cp[ENT_ENC_TO_UNI_STAGE2(code)];
702}
703/* }}} */
704
705/* {{{ unicode_cp_is_allowed */
706static inline int unicode_cp_is_allowed(unsigned uni_cp, int document_type)
707{
708	/* XML 1.0				HTML 4.01			HTML 5
709	 * 0x09..0x0A			0x09..0x0A			0x09..0x0A
710	 * 0x0D					0x0D				0x0C..0x0D
711	 * 0x0020..0xD7FF		0x20..0x7E			0x20..0x7E
712	 *						0x00A0..0xD7FF		0x00A0..0xD7FF
713	 * 0xE000..0xFFFD		0xE000..0x10FFFF	0xE000..0xFDCF
714	 * 0x010000..0x10FFFF						0xFDF0..0x10FFFF (*)
715	 *
716	 * (*) exclude code points where ((code & 0xFFFF) >= 0xFFFE)
717	 *
718	 * References:
719	 * XML 1.0:   <http://www.w3.org/TR/REC-xml/#charsets>
720	 * HTML 4.01: <http://www.w3.org/TR/1999/PR-html40-19990824/sgml/sgmldecl.html>
721	 * HTML 5:    <http://dev.w3.org/html5/spec/Overview.html#preprocessing-the-input-stream>
722	 *
723	 * Not sure this is the relevant part for HTML 5, though. I opted to
724	 * disallow the characters that would result in a parse error when
725	 * preprocessing of the input stream. See also section 8.1.3.
726	 *
727	 * It's unclear if XHTML 1.0 allows C1 characters. I'll opt to apply to
728	 * XHTML 1.0 the same rules as for XML 1.0.
729	 * See <http://cmsmcq.com/2007/C1.xml>.
730	 */
731
732	switch (document_type) {
733	case ENT_HTML_DOC_HTML401:
734		return (uni_cp >= 0x20 && uni_cp <= 0x7E) ||
735			(uni_cp == 0x0A || uni_cp == 0x09 || uni_cp == 0x0D) ||
736			(uni_cp >= 0xA0 && uni_cp <= 0xD7FF) ||
737			(uni_cp >= 0xE000 && uni_cp <= 0x10FFFF);
738	case ENT_HTML_DOC_HTML5:
739		return (uni_cp >= 0x20 && uni_cp <= 0x7E) ||
740			(uni_cp >= 0x09 && uni_cp <= 0x0D && uni_cp != 0x0B) || /* form feed U+0C allowed */
741			(uni_cp >= 0xA0 && uni_cp <= 0xD7FF) ||
742			(uni_cp >= 0xE000 && uni_cp <= 0x10FFFF &&
743				((uni_cp & 0xFFFF) < 0xFFFE) && /* last two of each plane (nonchars) disallowed */
744				(uni_cp < 0xFDD0 || uni_cp > 0xFDEF)); /* U+FDD0-U+FDEF (nonchars) disallowed */
745	case ENT_HTML_DOC_XHTML:
746	case ENT_HTML_DOC_XML1:
747		return (uni_cp >= 0x20 && uni_cp <= 0xD7FF) ||
748			(uni_cp == 0x0A || uni_cp == 0x09 || uni_cp == 0x0D) ||
749			(uni_cp >= 0xE000 && uni_cp <= 0x10FFFF && uni_cp != 0xFFFE && uni_cp != 0xFFFF);
750	default:
751		return 1;
752	}
753}
754/* }}} */
755
756/* {{{ unicode_cp_is_allowed */
757static inline int numeric_entity_is_allowed(unsigned uni_cp, int document_type)
758{
759	/* less restrictive than unicode_cp_is_allowed */
760	switch (document_type) {
761	case ENT_HTML_DOC_HTML401:
762		/* all non-SGML characters (those marked with UNUSED in DESCSET) should be
763		 * representable with numeric entities */
764		return uni_cp <= 0x10FFFF;
765	case ENT_HTML_DOC_HTML5:
766		/* 8.1.4. The numeric character reference forms described above are allowed to
767		 * reference any Unicode code point other than U+0000, U+000D, permanently
768		 * undefined Unicode characters (noncharacters), and control characters other
769		 * than space characters (U+0009, U+000A, U+000C and U+000D) */
770		/* seems to allow surrogate characters, then */
771		return (uni_cp >= 0x20 && uni_cp <= 0x7E) ||
772			(uni_cp >= 0x09 && uni_cp <= 0x0C && uni_cp != 0x0B) || /* form feed U+0C allowed, but not U+0D */
773			(uni_cp >= 0xA0 && uni_cp <= 0x10FFFF &&
774				((uni_cp & 0xFFFF) < 0xFFFE) && /* last two of each plane (nonchars) disallowed */
775				(uni_cp < 0xFDD0 || uni_cp > 0xFDEF)); /* U+FDD0-U+FDEF (nonchars) disallowed */
776	case ENT_HTML_DOC_XHTML:
777	case ENT_HTML_DOC_XML1:
778		/* OTOH, XML 1.0 requires "character references to match the production for Char
779		 * See <http://www.w3.org/TR/REC-xml/#NT-CharRef> */
780		return unicode_cp_is_allowed(uni_cp, document_type);
781	default:
782		return 1;
783	}
784}
785/* }}} */
786
787/* {{{ process_numeric_entity
788 * Auxiliary function to traverse_for_entities.
789 * On input, *buf should point to the first character after # and on output, it's the last
790 * byte read, no matter if there was success or insuccess.
791 */
792static inline int process_numeric_entity(const char **buf, unsigned *code_point)
793{
794	zend_long code_l;
795	int hexadecimal = (**buf == 'x' || **buf == 'X'); /* TODO: XML apparently disallows "X" */
796	char *endptr;
797
798	if (hexadecimal && (**buf != '\0'))
799		(*buf)++;
800
801	/* strtol allows whitespace and other stuff in the beginning
802		* we're not interested */
803	if ((hexadecimal && !isxdigit(**buf)) ||
804			(!hexadecimal && !isdigit(**buf))) {
805		return FAILURE;
806	}
807
808	code_l = ZEND_STRTOL(*buf, &endptr, hexadecimal ? 16 : 10);
809	/* we're guaranteed there were valid digits, so *endptr > buf */
810	*buf = endptr;
811
812	if (**buf != ';')
813		return FAILURE;
814
815	/* many more are invalid, but that depends on whether it's HTML
816	 * (and which version) or XML. */
817	if (code_l > Z_L(0x10FFFF))
818		return FAILURE;
819
820	if (code_point != NULL)
821		*code_point = (unsigned)code_l;
822
823	return SUCCESS;
824}
825/* }}} */
826
827/* {{{ process_named_entity */
828static inline int process_named_entity_html(const char **buf, const char **start, size_t *length)
829{
830	*start = *buf;
831
832	/* "&" is represented by a 0x26 in all supported encodings. That means
833	 * the byte after represents a character or is the leading byte of an
834	 * sequence of 8-bit code units. If in the ranges below, it represents
835	 * necessarily a alpha character because none of the supported encodings
836	 * has an overlap with ASCII in the leading byte (only on the second one) */
837	while ((**buf >= 'a' && **buf <= 'z') ||
838			(**buf >= 'A' && **buf <= 'Z') ||
839			(**buf >= '0' && **buf <= '9')) {
840		(*buf)++;
841	}
842
843	if (**buf != ';')
844		return FAILURE;
845
846	/* cast to size_t OK as the quantity is always non-negative */
847	*length = *buf - *start;
848
849	if (*length == 0)
850		return FAILURE;
851
852	return SUCCESS;
853}
854/* }}} */
855
856/* {{{ resolve_named_entity_html */
857static inline int resolve_named_entity_html(const char *start, size_t length, const entity_ht *ht, unsigned *uni_cp1, unsigned *uni_cp2)
858{
859	const entity_cp_map *s;
860	zend_ulong hash = zend_inline_hash_func(start, length);
861
862	s = ht->buckets[hash % ht->num_elems];
863	while (s->entity) {
864		if (s->entity_len == length) {
865			if (memcmp(start, s->entity, length) == 0) {
866				*uni_cp1 = s->codepoint1;
867				*uni_cp2 = s->codepoint2;
868				return SUCCESS;
869			}
870		}
871		s++;
872	}
873	return FAILURE;
874}
875/* }}} */
876
877static inline size_t write_octet_sequence(unsigned char *buf, enum entity_charset charset, unsigned code) {
878	/* code is not necessarily a unicode code point */
879	switch (charset) {
880	case cs_utf_8:
881		return php_utf32_utf8(buf, code);
882
883	case cs_8859_1:
884	case cs_cp1252:
885	case cs_8859_15:
886	case cs_koi8r:
887	case cs_cp1251:
888	case cs_8859_5:
889	case cs_cp866:
890	case cs_macroman:
891		/* single byte stuff */
892		*buf = code;
893		return 1;
894
895	case cs_big5:
896	case cs_big5hkscs:
897	case cs_sjis:
898	case cs_gb2312:
899		/* we don't have complete unicode mappings for these yet in entity_decode,
900		 * and we opt to pass through the octet sequences for these in htmlentities
901		 * instead of converting to an int and then converting back. */
902#if 0
903		return php_mb2_int_to_char(buf, code);
904#else
905#if ZEND_DEBUG
906		assert(code <= 0xFFU);
907#endif
908		*buf = code;
909		return 1;
910#endif
911
912	case cs_eucjp:
913#if 0 /* idem */
914		return php_mb2_int_to_char(buf, code);
915#else
916#if ZEND_DEBUG
917		assert(code <= 0xFFU);
918#endif
919		*buf = code;
920		return 1;
921#endif
922
923	default:
924		assert(0);
925		return 0;
926	}
927}
928
929/* {{{ traverse_for_entities
930 * Auxiliary function to php_unescape_html_entities().
931 * - The argument "all" determines if all numeric entities are decode or only those
932 *   that correspond to quotes (depending on quote_style).
933 */
934/* maximum expansion (factor 1.2) for HTML 5 with &nGt; and &nLt; */
935/* +2 is 1 because of rest (probably unnecessary), 1 because of terminating 0 */
936#define TRAVERSE_FOR_ENTITIES_EXPAND_SIZE(oldlen) ((oldlen) + (oldlen) / 5 + 2)
937static void traverse_for_entities(
938	const char *old,
939	size_t oldlen,
940	zend_string *ret, /* should have allocated TRAVERSE_FOR_ENTITIES_EXPAND_SIZE(olden) */
941	int all,
942	int flags,
943	const entity_ht *inv_map,
944	enum entity_charset charset)
945{
946	const char *p,
947			   *lim;
948	char	   *q;
949	int doctype = flags & ENT_HTML_DOC_TYPE_MASK;
950
951	lim = old + oldlen; /* terminator address */
952	assert(*lim == '\0');
953
954	for (p = old, q = ZSTR_VAL(ret); p < lim;) {
955		unsigned code, code2 = 0;
956		const char *next = NULL; /* when set, next > p, otherwise possible inf loop */
957
958		/* Shift JIS, Big5 and HKSCS use multi-byte encodings where an
959		 * ASCII range byte can be part of a multi-byte sequence.
960		 * However, they start at 0x40, therefore if we find a 0x26 byte,
961		 * we're sure it represents the '&' character. */
962
963		/* assumes there are no single-char entities */
964		if (p[0] != '&' || (p + 3 >= lim)) {
965			*(q++) = *(p++);
966			continue;
967		}
968
969		/* now p[3] is surely valid and is no terminator */
970
971		/* numerical entity */
972		if (p[1] == '#') {
973			next = &p[2];
974			if (process_numeric_entity(&next, &code) == FAILURE)
975				goto invalid_code;
976
977			/* If we're in htmlspecialchars_decode, we're only decoding entities
978			 * that represent &, <, >, " and '. Is this one of them? */
979			if (!all && (code > 63U ||
980					stage3_table_be_apos_00000[code].data.ent.entity == NULL))
981				goto invalid_code;
982
983			/* are we allowed to decode this entity in this document type?
984			 * HTML 5 is the only that has a character that cannot be used in
985			 * a numeric entity but is allowed literally (U+000D). The
986			 * unoptimized version would be ... || !numeric_entity_is_allowed(code) */
987			if (!unicode_cp_is_allowed(code, doctype) ||
988					(doctype == ENT_HTML_DOC_HTML5 && code == 0x0D))
989				goto invalid_code;
990		} else {
991			const char *start;
992			size_t ent_len;
993
994			next = &p[1];
995			start = next;
996
997			if (process_named_entity_html(&next, &start, &ent_len) == FAILURE)
998				goto invalid_code;
999
1000			if (resolve_named_entity_html(start, ent_len, inv_map, &code, &code2) == FAILURE) {
1001				if (doctype == ENT_HTML_DOC_XHTML && ent_len == 4 && start[0] == 'a'
1002							&& start[1] == 'p' && start[2] == 'o' && start[3] == 's') {
1003					/* uses html4 inv_map, which doesn't include apos;. This is a
1004					 * hack to support it */
1005					code = (unsigned) '\'';
1006				} else {
1007					goto invalid_code;
1008				}
1009			}
1010		}
1011
1012		assert(*next == ';');
1013
1014		if (((code == '\'' && !(flags & ENT_HTML_QUOTE_SINGLE)) ||
1015				(code == '"' && !(flags & ENT_HTML_QUOTE_DOUBLE)))
1016				/* && code2 == '\0' always true for current maps */)
1017			goto invalid_code;
1018
1019		/* UTF-8 doesn't need mapping (ISO-8859-1 doesn't either, but
1020		 * the call is needed to ensure the codepoint <= U+00FF)  */
1021		if (charset != cs_utf_8) {
1022			/* replace unicode code point */
1023			if (map_from_unicode(code, charset, &code) == FAILURE || code2 != 0)
1024				goto invalid_code; /* not representable in target charset */
1025		}
1026
1027		q += write_octet_sequence((unsigned char*)q, charset, code);
1028		if (code2) {
1029			q += write_octet_sequence((unsigned char*)q, charset, code2);
1030		}
1031
1032		/* jump over the valid entity; may go beyond size of buffer; np */
1033		p = next + 1;
1034		continue;
1035
1036invalid_code:
1037		for (; p < next; p++) {
1038			*(q++) = *p;
1039		}
1040	}
1041
1042	*q = '\0';
1043	ZSTR_LEN(ret) = (size_t)(q - ZSTR_VAL(ret));
1044}
1045/* }}} */
1046
1047/* {{{ unescape_inverse_map */
1048static const entity_ht *unescape_inverse_map(int all, int flags)
1049{
1050	int document_type = flags & ENT_HTML_DOC_TYPE_MASK;
1051
1052	if (all) {
1053		switch (document_type) {
1054		case ENT_HTML_DOC_HTML401:
1055		case ENT_HTML_DOC_XHTML: /* but watch out for &apos;...*/
1056			return &ent_ht_html4;
1057		case ENT_HTML_DOC_HTML5:
1058			return &ent_ht_html5;
1059		default:
1060			return &ent_ht_be_apos;
1061		}
1062	} else {
1063		switch (document_type) {
1064		case ENT_HTML_DOC_HTML401:
1065			return &ent_ht_be_noapos;
1066		default:
1067			return &ent_ht_be_apos;
1068		}
1069	}
1070}
1071/* }}} */
1072
1073/* {{{ determine_entity_table
1074 * Entity table to use. Note that entity tables are defined in terms of
1075 * unicode code points */
1076static entity_table_opt determine_entity_table(int all, int doctype)
1077{
1078	entity_table_opt retval = {NULL};
1079
1080	assert(!(doctype == ENT_HTML_DOC_XML1 && all));
1081
1082	if (all) {
1083		retval.ms_table = (doctype == ENT_HTML_DOC_HTML5) ?
1084			entity_ms_table_html5 : entity_ms_table_html4;
1085	} else {
1086		retval.table = (doctype == ENT_HTML_DOC_HTML401) ?
1087			stage3_table_be_noapos_00000 : stage3_table_be_apos_00000;
1088	}
1089	return retval;
1090}
1091/* }}} */
1092
1093/* {{{ php_unescape_html_entities
1094 * The parameter "all" should be true to decode all possible entities, false to decode
1095 * only the basic ones, i.e., those in basic_entities_ex + the numeric entities
1096 * that correspond to quotes.
1097 */
1098PHPAPI zend_string *php_unescape_html_entities(unsigned char *old, size_t oldlen, int all, int flags, char *hint_charset)
1099{
1100	size_t retlen;
1101	zend_string *ret;
1102	enum entity_charset charset;
1103	const entity_ht *inverse_map = NULL;
1104	size_t new_size = TRAVERSE_FOR_ENTITIES_EXPAND_SIZE(oldlen);
1105
1106	if (all) {
1107		charset = determine_charset(hint_charset);
1108	} else {
1109		charset = cs_8859_1; /* charset shouldn't matter, use ISO-8859-1 for performance */
1110	}
1111
1112	/* don't use LIMIT_ALL! */
1113
1114	if (oldlen > new_size) {
1115		/* overflow, refuse to do anything */
1116		ret = zend_string_init((char*)old, oldlen, 0);
1117		retlen = oldlen;
1118		goto empty_source;
1119	}
1120	ret = zend_string_alloc(new_size, 0);
1121	ZSTR_VAL(ret)[0] = '\0';
1122	ZSTR_LEN(ret) = oldlen;
1123	retlen = oldlen;
1124	if (retlen == 0) {
1125		goto empty_source;
1126	}
1127
1128	inverse_map = unescape_inverse_map(all, flags);
1129
1130	/* replace numeric entities */
1131	traverse_for_entities((char*)old, oldlen, ret, all, flags, inverse_map, charset);
1132
1133empty_source:
1134	return ret;
1135}
1136/* }}} */
1137
1138PHPAPI zend_string *php_escape_html_entities(unsigned char *old, size_t oldlen, int all, int flags, char *hint_charset)
1139{
1140	return php_escape_html_entities_ex(old, oldlen, all, flags, hint_charset, 1);
1141}
1142
1143/* {{{ find_entity_for_char */
1144static inline void find_entity_for_char(
1145	unsigned int k,
1146	enum entity_charset charset,
1147	const entity_stage1_row *table,
1148	const unsigned char **entity,
1149	size_t *entity_len,
1150	unsigned char *old,
1151	size_t oldlen,
1152	size_t *cursor)
1153{
1154	unsigned stage1_idx = ENT_STAGE1_INDEX(k);
1155	const entity_stage3_row *c;
1156
1157	if (stage1_idx > 0x1D) {
1158		*entity     = NULL;
1159		*entity_len = 0;
1160		return;
1161	}
1162
1163	c = &table[stage1_idx][ENT_STAGE2_INDEX(k)][ENT_STAGE3_INDEX(k)];
1164
1165	if (!c->ambiguous) {
1166		*entity     = (const unsigned char *)c->data.ent.entity;
1167		*entity_len = c->data.ent.entity_len;
1168	} else {
1169		/* peek at next char */
1170		size_t	 cursor_before	= *cursor;
1171		int		 status			= SUCCESS;
1172		unsigned next_char;
1173
1174		if (!(*cursor < oldlen))
1175			goto no_suitable_2nd;
1176
1177		next_char = get_next_char(charset, old, oldlen, cursor, &status);
1178
1179		if (status == FAILURE)
1180			goto no_suitable_2nd;
1181
1182		{
1183			const entity_multicodepoint_row *s, *e;
1184
1185			s = &c->data.multicodepoint_table[1];
1186			e = s - 1 + c->data.multicodepoint_table[0].leading_entry.size;
1187			/* we could do a binary search but it's not worth it since we have
1188			 * at most two entries... */
1189			for ( ; s <= e; s++) {
1190				if (s->normal_entry.second_cp == next_char) {
1191					*entity     = (const unsigned char *) s->normal_entry.entity;
1192					*entity_len = s->normal_entry.entity_len;
1193					return;
1194				}
1195			}
1196		}
1197no_suitable_2nd:
1198		*cursor = cursor_before;
1199		*entity = (const unsigned char *)
1200			c->data.multicodepoint_table[0].leading_entry.default_entity;
1201		*entity_len = c->data.multicodepoint_table[0].leading_entry.default_entity_len;
1202	}
1203}
1204/* }}} */
1205
1206/* {{{ find_entity_for_char_basic */
1207static inline void find_entity_for_char_basic(
1208	unsigned int k,
1209	const entity_stage3_row *table,
1210	const unsigned char **entity,
1211	size_t *entity_len)
1212{
1213	if (k >= 64U) {
1214		*entity     = NULL;
1215		*entity_len = 0;
1216		return;
1217	}
1218
1219	*entity     = (const unsigned char *) table[k].data.ent.entity;
1220	*entity_len = table[k].data.ent.entity_len;
1221}
1222/* }}} */
1223
1224/* {{{ php_escape_html_entities
1225 */
1226PHPAPI zend_string *php_escape_html_entities_ex(unsigned char *old, size_t oldlen, int all, int flags, char *hint_charset, zend_bool double_encode)
1227{
1228	size_t cursor, maxlen, len;
1229	zend_string *replaced;
1230	enum entity_charset charset = determine_charset(hint_charset);
1231	int doctype = flags & ENT_HTML_DOC_TYPE_MASK;
1232	entity_table_opt entity_table;
1233	const enc_to_uni *to_uni_table = NULL;
1234	const entity_ht *inv_map = NULL; /* used for !double_encode */
1235	/* only used if flags includes ENT_HTML_IGNORE_ERRORS or ENT_HTML_SUBSTITUTE_DISALLOWED_CHARS */
1236	const unsigned char *replacement = NULL;
1237	size_t replacement_len = 0;
1238
1239	if (all) { /* replace with all named entities */
1240		if (CHARSET_PARTIAL_SUPPORT(charset)) {
1241			php_error_docref0(NULL, E_STRICT, "Only basic entities "
1242				"substitution is supported for multi-byte encodings other than UTF-8; "
1243				"functionality is equivalent to htmlspecialchars");
1244		}
1245		LIMIT_ALL(all, doctype, charset);
1246	}
1247	entity_table = determine_entity_table(all, doctype);
1248	if (all && !CHARSET_UNICODE_COMPAT(charset)) {
1249		to_uni_table = enc_to_uni_index[charset];
1250	}
1251
1252	if (!double_encode) {
1253		/* first arg is 1 because we want to identify valid named entities
1254		 * even if we are only encoding the basic ones */
1255		inv_map = unescape_inverse_map(1, flags);
1256	}
1257
1258	if (flags & (ENT_HTML_SUBSTITUTE_ERRORS | ENT_HTML_SUBSTITUTE_DISALLOWED_CHARS)) {
1259		if (charset == cs_utf_8) {
1260			replacement = (const unsigned char*)"\xEF\xBF\xBD";
1261			replacement_len = sizeof("\xEF\xBF\xBD") - 1;
1262		} else {
1263			replacement = (const unsigned char*)"&#xFFFD;";
1264			replacement_len = sizeof("&#xFFFD;") - 1;
1265		}
1266	}
1267
1268	/* initial estimate */
1269	if (oldlen < 64) {
1270		maxlen = 128;
1271	} else {
1272		maxlen = 2 * oldlen;
1273		if (maxlen < oldlen) {
1274			zend_error_noreturn(E_ERROR, "Input string is too long");
1275			return NULL;
1276		}
1277	}
1278
1279	replaced = zend_string_alloc(maxlen, 0);
1280	len = 0;
1281	cursor = 0;
1282	while (cursor < oldlen) {
1283		const unsigned char *mbsequence = NULL;
1284		size_t mbseqlen					= 0,
1285		       cursor_before			= cursor;
1286		int status						= SUCCESS;
1287		unsigned int this_char			= get_next_char(charset, old, oldlen, &cursor, &status);
1288
1289		/* guarantee we have at least 40 bytes to write.
1290		 * In HTML5, entities may take up to 33 bytes */
1291		if (len > maxlen - 40) { /* maxlen can never be smaller than 128 */
1292			replaced = zend_string_safe_realloc(replaced, maxlen, 1, 128, 0);
1293			maxlen += 128;
1294		}
1295
1296		if (status == FAILURE) {
1297			/* invalid MB sequence */
1298			if (flags & ENT_HTML_IGNORE_ERRORS) {
1299				continue;
1300			} else if (flags & ENT_HTML_SUBSTITUTE_ERRORS) {
1301				memcpy(&ZSTR_VAL(replaced)[len], replacement, replacement_len);
1302				len += replacement_len;
1303				continue;
1304			} else {
1305				zend_string_free(replaced);
1306				return ZSTR_EMPTY_ALLOC();
1307			}
1308		} else { /* SUCCESS */
1309			mbsequence = &old[cursor_before];
1310			mbseqlen = cursor - cursor_before;
1311		}
1312
1313		if (this_char != '&') { /* no entity on this position */
1314			const unsigned char *rep	= NULL;
1315			size_t				rep_len	= 0;
1316
1317			if (((this_char == '\'' && !(flags & ENT_HTML_QUOTE_SINGLE)) ||
1318					(this_char == '"' && !(flags & ENT_HTML_QUOTE_DOUBLE))))
1319				goto pass_char_through;
1320
1321			if (all) { /* false that CHARSET_PARTIAL_SUPPORT(charset) */
1322				if (to_uni_table != NULL) {
1323					/* !CHARSET_UNICODE_COMPAT therefore not UTF-8; since UTF-8
1324					 * is the only multibyte encoding with !CHARSET_PARTIAL_SUPPORT,
1325					 * we're using a single byte encoding */
1326					map_to_unicode(this_char, to_uni_table, &this_char);
1327					if (this_char == 0xFFFF) /* no mapping; pass through */
1328						goto pass_char_through;
1329				}
1330				/* the cursor may advance */
1331				find_entity_for_char(this_char, charset, entity_table.ms_table, &rep,
1332					&rep_len, old, oldlen, &cursor);
1333			} else {
1334				find_entity_for_char_basic(this_char, entity_table.table, &rep, &rep_len);
1335			}
1336
1337			if (rep != NULL) {
1338				ZSTR_VAL(replaced)[len++] = '&';
1339				memcpy(&ZSTR_VAL(replaced)[len], rep, rep_len);
1340				len += rep_len;
1341				ZSTR_VAL(replaced)[len++] = ';';
1342			} else {
1343				/* we did not find an entity for this char.
1344				 * check for its validity, if its valid pass it unchanged */
1345				if (flags & ENT_HTML_SUBSTITUTE_DISALLOWED_CHARS) {
1346					if (CHARSET_UNICODE_COMPAT(charset)) {
1347						if (!unicode_cp_is_allowed(this_char, doctype)) {
1348							mbsequence = replacement;
1349							mbseqlen = replacement_len;
1350						}
1351					} else if (to_uni_table) {
1352						if (!all) /* otherwise we already did this */
1353							map_to_unicode(this_char, to_uni_table, &this_char);
1354						if (!unicode_cp_is_allowed(this_char, doctype)) {
1355							mbsequence = replacement;
1356							mbseqlen = replacement_len;
1357						}
1358					} else {
1359						/* not a unicode code point, unless, coincidentally, it's in
1360						 * the 0x20..0x7D range (except 0x5C in sjis). We know nothing
1361						 * about other code points, because we have no tables. Since
1362						 * Unicode code points in that range are not disallowed in any
1363						 * document type, we could do nothing. However, conversion
1364						 * tables frequently map 0x00-0x1F to the respective C0 code
1365						 * points. Let's play it safe and admit that's the case */
1366						if (this_char <= 0x7D &&
1367								!unicode_cp_is_allowed(this_char, doctype)) {
1368							mbsequence = replacement;
1369							mbseqlen = replacement_len;
1370						}
1371					}
1372				}
1373pass_char_through:
1374				if (mbseqlen > 1) {
1375					memcpy(ZSTR_VAL(replaced) + len, mbsequence, mbseqlen);
1376					len += mbseqlen;
1377				} else {
1378					ZSTR_VAL(replaced)[len++] = mbsequence[0];
1379				}
1380			}
1381		} else { /* this_char == '&' */
1382			if (double_encode) {
1383encode_amp:
1384				memcpy(&ZSTR_VAL(replaced)[len], "&amp;", sizeof("&amp;") - 1);
1385				len += sizeof("&amp;") - 1;
1386			} else { /* no double encode */
1387				/* check if entity is valid */
1388				size_t ent_len; /* not counting & or ; */
1389				/* peek at next char */
1390				if (old[cursor] == '#') { /* numeric entity */
1391					unsigned code_point;
1392					int valid;
1393					char *pos = (char*)&old[cursor+1];
1394					valid = process_numeric_entity((const char **)&pos, &code_point);
1395					if (valid == FAILURE)
1396						goto encode_amp;
1397					if (flags & ENT_HTML_SUBSTITUTE_DISALLOWED_CHARS) {
1398						if (!numeric_entity_is_allowed(code_point, doctype))
1399							goto encode_amp;
1400					}
1401					ent_len = pos - (char*)&old[cursor];
1402				} else { /* named entity */
1403					/* check for vality of named entity */
1404					const char *start = (const char *) &old[cursor],
1405							   *next = start;
1406					unsigned   dummy1, dummy2;
1407
1408					if (process_named_entity_html(&next, &start, &ent_len) == FAILURE)
1409						goto encode_amp;
1410					if (resolve_named_entity_html(start, ent_len, inv_map, &dummy1, &dummy2) == FAILURE) {
1411						if (!(doctype == ENT_HTML_DOC_XHTML && ent_len == 4 && start[0] == 'a'
1412									&& start[1] == 'p' && start[2] == 'o' && start[3] == 's')) {
1413							/* uses html4 inv_map, which doesn't include apos;. This is a
1414							 * hack to support it */
1415							goto encode_amp;
1416						}
1417					}
1418				}
1419				/* checks passed; copy entity to result */
1420				/* entity size is unbounded, we may need more memory */
1421				/* at this point maxlen - len >= 40 */
1422				if (maxlen - len < ent_len + 2 /* & and ; */) {
1423					/* ent_len < oldlen, which is certainly <= SIZE_MAX/2 */
1424					replaced = zend_string_safe_realloc(replaced, maxlen, 1, ent_len + 128, 0);
1425					maxlen += ent_len + 128;
1426				}
1427				ZSTR_VAL(replaced)[len++] = '&';
1428				memcpy(&ZSTR_VAL(replaced)[len], &old[cursor], ent_len);
1429				len += ent_len;
1430				ZSTR_VAL(replaced)[len++] = ';';
1431				cursor += ent_len + 1;
1432			}
1433		}
1434	}
1435	ZSTR_VAL(replaced)[len] = '\0';
1436	ZSTR_LEN(replaced) = len;
1437
1438	return replaced;
1439}
1440/* }}} */
1441
1442/* {{{ php_html_entities
1443 */
1444static void php_html_entities(INTERNAL_FUNCTION_PARAMETERS, int all)
1445{
1446	zend_string *str, *hint_charset = NULL;
1447	char *default_charset;
1448	zend_long flags = ENT_COMPAT;
1449	zend_string *replaced;
1450	zend_bool double_encode = 1;
1451
1452#ifndef FAST_ZPP
1453	if (zend_parse_parameters(ZEND_NUM_ARGS(), "S|lS!b", &str, &flags, &hint_charset, &double_encode) == FAILURE) {
1454		return;
1455	}
1456#else
1457	ZEND_PARSE_PARAMETERS_START(1, 4)
1458		Z_PARAM_STR(str)
1459		Z_PARAM_OPTIONAL
1460		Z_PARAM_LONG(flags)
1461		Z_PARAM_STR_EX(hint_charset, 1, 0)
1462		Z_PARAM_BOOL(double_encode);
1463	ZEND_PARSE_PARAMETERS_END();
1464#endif
1465
1466	if (!hint_charset) {
1467		default_charset = get_default_charset();
1468	}
1469	replaced = php_escape_html_entities_ex((unsigned char*)ZSTR_VAL(str), ZSTR_LEN(str), all, (int) flags, (hint_charset ? ZSTR_VAL(hint_charset) : default_charset), double_encode);
1470	RETVAL_STR(replaced);
1471}
1472/* }}} */
1473
1474#define HTML_SPECIALCHARS 	0
1475#define HTML_ENTITIES	 	1
1476
1477/* {{{ register_html_constants
1478 */
1479void register_html_constants(INIT_FUNC_ARGS)
1480{
1481	REGISTER_LONG_CONSTANT("HTML_SPECIALCHARS", HTML_SPECIALCHARS, CONST_PERSISTENT|CONST_CS);
1482	REGISTER_LONG_CONSTANT("HTML_ENTITIES", HTML_ENTITIES, CONST_PERSISTENT|CONST_CS);
1483	REGISTER_LONG_CONSTANT("ENT_COMPAT", ENT_COMPAT, CONST_PERSISTENT|CONST_CS);
1484	REGISTER_LONG_CONSTANT("ENT_QUOTES", ENT_QUOTES, CONST_PERSISTENT|CONST_CS);
1485	REGISTER_LONG_CONSTANT("ENT_NOQUOTES", ENT_NOQUOTES, CONST_PERSISTENT|CONST_CS);
1486	REGISTER_LONG_CONSTANT("ENT_IGNORE", ENT_IGNORE, CONST_PERSISTENT|CONST_CS);
1487	REGISTER_LONG_CONSTANT("ENT_SUBSTITUTE", ENT_SUBSTITUTE, CONST_PERSISTENT|CONST_CS);
1488	REGISTER_LONG_CONSTANT("ENT_DISALLOWED", ENT_DISALLOWED, CONST_PERSISTENT|CONST_CS);
1489	REGISTER_LONG_CONSTANT("ENT_HTML401", ENT_HTML401, CONST_PERSISTENT|CONST_CS);
1490	REGISTER_LONG_CONSTANT("ENT_XML1", ENT_XML1, CONST_PERSISTENT|CONST_CS);
1491	REGISTER_LONG_CONSTANT("ENT_XHTML", ENT_XHTML, CONST_PERSISTENT|CONST_CS);
1492	REGISTER_LONG_CONSTANT("ENT_HTML5", ENT_HTML5, CONST_PERSISTENT|CONST_CS);
1493}
1494/* }}} */
1495
1496/* {{{ proto string htmlspecialchars(string string [, int quote_style[, string encoding[, bool double_encode]]])
1497   Convert special characters to HTML entities */
1498PHP_FUNCTION(htmlspecialchars)
1499{
1500	php_html_entities(INTERNAL_FUNCTION_PARAM_PASSTHRU, 0);
1501}
1502/* }}} */
1503
1504/* {{{ proto string htmlspecialchars_decode(string string [, int quote_style])
1505   Convert special HTML entities back to characters */
1506PHP_FUNCTION(htmlspecialchars_decode)
1507{
1508	char *str;
1509	size_t str_len;
1510	zend_long quote_style = ENT_COMPAT;
1511	zend_string *replaced;
1512
1513	if (zend_parse_parameters(ZEND_NUM_ARGS(), "s|l", &str, &str_len, &quote_style) == FAILURE) {
1514		return;
1515	}
1516
1517	replaced = php_unescape_html_entities((unsigned char*)str, str_len, 0 /*!all*/, (int)quote_style, NULL);
1518	if (replaced) {
1519		RETURN_STR(replaced);
1520	}
1521	RETURN_FALSE;
1522}
1523/* }}} */
1524
1525/* {{{ proto string html_entity_decode(string string [, int quote_style][, string encoding])
1526   Convert all HTML entities to their applicable characters */
1527PHP_FUNCTION(html_entity_decode)
1528{
1529	zend_string *str, *hint_charset = NULL;
1530	char *default_charset;
1531	zend_long quote_style = ENT_COMPAT;
1532	zend_string *replaced;
1533
1534#ifndef FAST_ZPP
1535	if (zend_parse_parameters(ZEND_NUM_ARGS(), "S|lS", &str,
1536							  &quote_style, &hint_charset) == FAILURE) {
1537		return;
1538	}
1539#else
1540	ZEND_PARSE_PARAMETERS_START(1, 3)
1541		Z_PARAM_STR(str)
1542		Z_PARAM_OPTIONAL
1543		Z_PARAM_LONG(quote_style)
1544		Z_PARAM_STR(hint_charset)
1545	ZEND_PARSE_PARAMETERS_END();
1546#endif
1547
1548	if (!hint_charset) {
1549		default_charset = get_default_charset();
1550	}
1551	replaced = php_unescape_html_entities((unsigned char*)ZSTR_VAL(str), ZSTR_LEN(str), 1 /*all*/, (int)quote_style, (hint_charset ? ZSTR_VAL(hint_charset) : default_charset));
1552
1553	if (replaced) {
1554		RETURN_STR(replaced);
1555	}
1556	RETURN_FALSE;
1557}
1558/* }}} */
1559
1560
1561/* {{{ proto string htmlentities(string string [, int quote_style[, string encoding[, bool double_encode]]])
1562   Convert all applicable characters to HTML entities */
1563PHP_FUNCTION(htmlentities)
1564{
1565	php_html_entities(INTERNAL_FUNCTION_PARAM_PASSTHRU, 1);
1566}
1567/* }}} */
1568
1569/* {{{ write_s3row_data */
1570static inline void write_s3row_data(
1571	const entity_stage3_row *r,
1572	unsigned orig_cp,
1573	enum entity_charset charset,
1574	zval *arr)
1575{
1576	char key[9] = ""; /* two unicode code points in UTF-8 */
1577	char entity[LONGEST_ENTITY_LENGTH + 2] = {'&'};
1578	size_t written_k1;
1579
1580	written_k1 = write_octet_sequence((unsigned char*)key, charset, orig_cp);
1581
1582	if (!r->ambiguous) {
1583		size_t l = r->data.ent.entity_len;
1584		memcpy(&entity[1], r->data.ent.entity, l);
1585		entity[l + 1] = ';';
1586		add_assoc_stringl_ex(arr, key, written_k1, entity, l + 2);
1587	} else {
1588		unsigned i,
1589			     num_entries;
1590		const entity_multicodepoint_row *mcpr = r->data.multicodepoint_table;
1591
1592		if (mcpr[0].leading_entry.default_entity != NULL) {
1593			size_t l = mcpr[0].leading_entry.default_entity_len;
1594			memcpy(&entity[1], mcpr[0].leading_entry.default_entity, l);
1595			entity[l + 1] = ';';
1596			add_assoc_stringl_ex(arr, key, written_k1, entity, l + 2);
1597		}
1598		num_entries = mcpr[0].leading_entry.size;
1599		for (i = 1; i <= num_entries; i++) {
1600			size_t   l,
1601				     written_k2;
1602			unsigned uni_cp,
1603					 spe_cp;
1604
1605			uni_cp = mcpr[i].normal_entry.second_cp;
1606			l = mcpr[i].normal_entry.entity_len;
1607
1608			if (!CHARSET_UNICODE_COMPAT(charset)) {
1609				if (map_from_unicode(uni_cp, charset, &spe_cp) == FAILURE)
1610					continue; /* non representable in this charset */
1611			} else {
1612				spe_cp = uni_cp;
1613			}
1614
1615			written_k2 = write_octet_sequence((unsigned char*)&key[written_k1], charset, spe_cp);
1616			memcpy(&entity[1], mcpr[i].normal_entry.entity, l);
1617			entity[l + 1] = ';';
1618			entity[l + 1] = '\0';
1619			add_assoc_stringl_ex(arr, key, written_k1 + written_k2, entity, l + 1);
1620		}
1621	}
1622}
1623/* }}} */
1624
1625/* {{{ proto array get_html_translation_table([int table [, int flags [, string encoding]]])
1626   Returns the internal translation table used by htmlspecialchars and htmlentities */
1627PHP_FUNCTION(get_html_translation_table)
1628{
1629	zend_long all = HTML_SPECIALCHARS,
1630		 flags = ENT_COMPAT;
1631	int doctype;
1632	entity_table_opt entity_table;
1633	const enc_to_uni *to_uni_table = NULL;
1634	char *charset_hint = NULL;
1635	size_t charset_hint_len;
1636	enum entity_charset charset;
1637
1638	/* in this function we have to jump through some loops because we're
1639	 * getting the translated table from data structures that are optimized for
1640	 * random access, not traversal */
1641
1642	if (zend_parse_parameters(ZEND_NUM_ARGS(), "|lls",
1643			&all, &flags, &charset_hint, &charset_hint_len) == FAILURE) {
1644		return;
1645	}
1646
1647	charset = determine_charset(charset_hint);
1648	doctype = flags & ENT_HTML_DOC_TYPE_MASK;
1649	LIMIT_ALL(all, doctype, charset);
1650
1651	array_init(return_value);
1652
1653	entity_table = determine_entity_table((int)all, doctype);
1654	if (all && !CHARSET_UNICODE_COMPAT(charset)) {
1655		to_uni_table = enc_to_uni_index[charset];
1656	}
1657
1658	if (all) { /* HTML_ENTITIES (actually, any non-zero value for 1st param) */
1659		const entity_stage1_row *ms_table = entity_table.ms_table;
1660
1661		if (CHARSET_UNICODE_COMPAT(charset)) {
1662			unsigned i, j, k,
1663					 max_i, max_j, max_k;
1664			/* no mapping to unicode required */
1665			if (CHARSET_SINGLE_BYTE(charset)) { /* ISO-8859-1 */
1666				max_i = 1; max_j = 4; max_k = 64;
1667			} else {
1668				max_i = 0x1E; max_j = 64; max_k = 64;
1669			}
1670
1671			for (i = 0; i < max_i; i++) {
1672				if (ms_table[i] == empty_stage2_table)
1673					continue;
1674				for (j = 0; j < max_j; j++) {
1675					if (ms_table[i][j] == empty_stage3_table)
1676						continue;
1677					for (k = 0; k < max_k; k++) {
1678						const entity_stage3_row *r = &ms_table[i][j][k];
1679						unsigned code;
1680
1681						if (r->data.ent.entity == NULL)
1682							continue;
1683
1684						code = ENT_CODE_POINT_FROM_STAGES(i, j, k);
1685						if (((code == '\'' && !(flags & ENT_HTML_QUOTE_SINGLE)) ||
1686								(code == '"' && !(flags & ENT_HTML_QUOTE_DOUBLE))))
1687							continue;
1688						write_s3row_data(r, code, charset, return_value);
1689					}
1690				}
1691			}
1692		} else {
1693			/* we have to iterate through the set of code points for this
1694			 * encoding and map them to unicode code points */
1695			unsigned i;
1696			for (i = 0; i <= 0xFF; i++) {
1697				const entity_stage3_row *r;
1698				unsigned uni_cp;
1699
1700				/* can be done before mapping, they're invariant */
1701				if (((i == '\'' && !(flags & ENT_HTML_QUOTE_SINGLE)) ||
1702						(i == '"' && !(flags & ENT_HTML_QUOTE_DOUBLE))))
1703					continue;
1704
1705				map_to_unicode(i, to_uni_table, &uni_cp);
1706				r = &ms_table[ENT_STAGE1_INDEX(uni_cp)][ENT_STAGE2_INDEX(uni_cp)][ENT_STAGE3_INDEX(uni_cp)];
1707				if (r->data.ent.entity == NULL)
1708					continue;
1709
1710				write_s3row_data(r, i, charset, return_value);
1711			}
1712		}
1713	} else {
1714		/* we could use sizeof(stage3_table_be_apos_00000) as well */
1715		unsigned	  j,
1716					  numelems = sizeof(stage3_table_be_noapos_00000) /
1717							sizeof(*stage3_table_be_noapos_00000);
1718
1719		for (j = 0; j < numelems; j++) {
1720			const entity_stage3_row *r = &entity_table.table[j];
1721			if (r->data.ent.entity == NULL)
1722				continue;
1723
1724			if (((j == '\'' && !(flags & ENT_HTML_QUOTE_SINGLE)) ||
1725					(j == '"' && !(flags & ENT_HTML_QUOTE_DOUBLE))))
1726				continue;
1727
1728			/* charset is indifferent, used cs_8859_1 for efficiency */
1729			write_s3row_data(r, j, cs_8859_1, return_value);
1730		}
1731	}
1732}
1733/* }}} */
1734
1735/*
1736 * Local variables:
1737 * tab-width: 4
1738 * c-basic-offset: 4
1739 * End:
1740 * vim600: sw=4 ts=4 fdm=marker
1741 * vim<600: sw=4 ts=4
1742 */
1743