1/*
2   +----------------------------------------------------------------------+
3   | Zend Engine                                                          |
4   +----------------------------------------------------------------------+
5   | Copyright (c) 1998-2016 Zend Technologies Ltd. (http://www.zend.com) |
6   +----------------------------------------------------------------------+
7   | This source file is subject to version 2.00 of the Zend license,     |
8   | that is bundled with this package in the file LICENSE, and is        |
9   | available through the world-wide-web at the following url:           |
10   | http://www.zend.com/license/2_00.txt.                                |
11   | If you did not receive a copy of the Zend license and are unable to  |
12   | obtain it through the world-wide-web, please send a note to          |
13   | license@zend.com so we can mail you a copy immediately.              |
14   +----------------------------------------------------------------------+
15   | Authors: Marcus Boerger <helly@php.net>                              |
16   |          Nuno Lopes <nlopess@php.net>                                |
17   |          Scott MacVicar <scottmac@php.net>                           |
18   | Flex version authors:                                                |
19   |          Andi Gutmans <andi@zend.com>                                |
20   |          Zeev Suraski <zeev@zend.com>                                |
21   +----------------------------------------------------------------------+
22*/
23
24/* $Id$ */
25
26#if 0
27# define YYDEBUG(s, c) printf("state: %d char: %c\n", s, c)
28#else
29# define YYDEBUG(s, c)
30#endif
31
32#include "zend_language_scanner_defs.h"
33
34#include <errno.h>
35#include "zend.h"
36#ifdef ZEND_WIN32
37# include <Winuser.h>
38#endif
39#include "zend_alloc.h"
40#include <zend_language_parser.h>
41#include "zend_compile.h"
42#include "zend_language_scanner.h"
43#include "zend_highlight.h"
44#include "zend_constants.h"
45#include "zend_variables.h"
46#include "zend_operators.h"
47#include "zend_API.h"
48#include "zend_strtod.h"
49#include "zend_exceptions.h"
50#include "zend_virtual_cwd.h"
51#include "tsrm_config_common.h"
52
53#define YYCTYPE   unsigned char
54#define YYFILL(n) { if ((YYCURSOR + n) >= (YYLIMIT + ZEND_MMAP_AHEAD)) { return 0; } }
55#define YYCURSOR  SCNG(yy_cursor)
56#define YYLIMIT   SCNG(yy_limit)
57#define YYMARKER  SCNG(yy_marker)
58
59#define YYGETCONDITION()  SCNG(yy_state)
60#define YYSETCONDITION(s) SCNG(yy_state) = s
61
62#define STATE(name)  yyc##name
63
64/* emulate flex constructs */
65#define BEGIN(state) YYSETCONDITION(STATE(state))
66#define YYSTATE      YYGETCONDITION()
67#define yytext       ((char*)SCNG(yy_text))
68#define yyleng       SCNG(yy_leng)
69#define yyless(x)    do { YYCURSOR = (unsigned char*)yytext + x; \
70                          yyleng   = (unsigned int)x; } while(0)
71#define yymore()     goto yymore_restart
72
73/* perform sanity check. If this message is triggered you should
74   increase the ZEND_MMAP_AHEAD value in the zend_streams.h file */
75/*!max:re2c */
76#if ZEND_MMAP_AHEAD < YYMAXFILL
77# error ZEND_MMAP_AHEAD should be greater than or equal to YYMAXFILL
78#endif
79
80#ifdef HAVE_STDARG_H
81# include <stdarg.h>
82#endif
83
84#ifdef HAVE_UNISTD_H
85# include <unistd.h>
86#endif
87
88/* Globals Macros */
89#define SCNG	LANG_SCNG
90#ifdef ZTS
91ZEND_API ts_rsrc_id language_scanner_globals_id;
92#else
93ZEND_API zend_php_scanner_globals language_scanner_globals;
94#endif
95
96#define HANDLE_NEWLINES(s, l)													\
97do {																			\
98	char *p = (s), *boundary = p+(l);											\
99																				\
100	while (p<boundary) {														\
101		if (*p == '\n' || (*p == '\r' && (*(p+1) != '\n'))) {					\
102			CG(zend_lineno)++;													\
103		}																		\
104		p++;																	\
105	}																			\
106} while (0)
107
108#define HANDLE_NEWLINE(c) \
109{ \
110	if (c == '\n' || c == '\r') { \
111		CG(zend_lineno)++; \
112	} \
113}
114
115/* To save initial string length after scanning to first variable */
116#define SET_DOUBLE_QUOTES_SCANNED_LENGTH(len) SCNG(scanned_string_len) = (len)
117#define GET_DOUBLE_QUOTES_SCANNED_LENGTH()    SCNG(scanned_string_len)
118
119#define IS_LABEL_START(c) (((c) >= 'a' && (c) <= 'z') || ((c) >= 'A' && (c) <= 'Z') || (c) == '_' || (c) >= 0x80)
120
121#define ZEND_IS_OCT(c)  ((c)>='0' && (c)<='7')
122#define ZEND_IS_HEX(c)  (((c)>='0' && (c)<='9') || ((c)>='a' && (c)<='f') || ((c)>='A' && (c)<='F'))
123
124BEGIN_EXTERN_C()
125
126static size_t encoding_filter_script_to_internal(unsigned char **to, size_t *to_length, const unsigned char *from, size_t from_length)
127{
128	const zend_encoding *internal_encoding = zend_multibyte_get_internal_encoding();
129	ZEND_ASSERT(internal_encoding);
130	return zend_multibyte_encoding_converter(to, to_length, from, from_length, internal_encoding, LANG_SCNG(script_encoding));
131}
132
133static size_t encoding_filter_script_to_intermediate(unsigned char **to, size_t *to_length, const unsigned char *from, size_t from_length)
134{
135	return zend_multibyte_encoding_converter(to, to_length, from, from_length, zend_multibyte_encoding_utf8, LANG_SCNG(script_encoding));
136}
137
138static size_t encoding_filter_intermediate_to_script(unsigned char **to, size_t *to_length, const unsigned char *from, size_t from_length)
139{
140	return zend_multibyte_encoding_converter(to, to_length, from, from_length,
141LANG_SCNG(script_encoding), zend_multibyte_encoding_utf8);
142}
143
144static size_t encoding_filter_intermediate_to_internal(unsigned char **to, size_t *to_length, const unsigned char *from, size_t from_length)
145{
146	const zend_encoding *internal_encoding = zend_multibyte_get_internal_encoding();
147	ZEND_ASSERT(internal_encoding);
148	return zend_multibyte_encoding_converter(to, to_length, from, from_length,
149internal_encoding, zend_multibyte_encoding_utf8);
150}
151
152
153static void _yy_push_state(int new_state)
154{
155	zend_stack_push(&SCNG(state_stack), (void *) &YYGETCONDITION());
156	YYSETCONDITION(new_state);
157}
158
159#define yy_push_state(state_and_tsrm) _yy_push_state(yyc##state_and_tsrm)
160
161static void yy_pop_state(void)
162{
163	int *stack_state = zend_stack_top(&SCNG(state_stack));
164	YYSETCONDITION(*stack_state);
165	zend_stack_del_top(&SCNG(state_stack));
166}
167
168static void yy_scan_buffer(char *str, unsigned int len)
169{
170	YYCURSOR       = (YYCTYPE*)str;
171	YYLIMIT        = YYCURSOR + len;
172	if (!SCNG(yy_start)) {
173		SCNG(yy_start) = YYCURSOR;
174	}
175}
176
177void startup_scanner(void)
178{
179	CG(parse_error) = 0;
180	CG(doc_comment) = NULL;
181	zend_stack_init(&SCNG(state_stack), sizeof(int));
182	zend_ptr_stack_init(&SCNG(heredoc_label_stack));
183}
184
185static void heredoc_label_dtor(zend_heredoc_label *heredoc_label) {
186    efree(heredoc_label->label);
187}
188
189void shutdown_scanner(void)
190{
191	CG(parse_error) = 0;
192	RESET_DOC_COMMENT();
193	zend_stack_destroy(&SCNG(state_stack));
194	zend_ptr_stack_clean(&SCNG(heredoc_label_stack), (void (*)(void *)) &heredoc_label_dtor, 1);
195	zend_ptr_stack_destroy(&SCNG(heredoc_label_stack));
196	SCNG(on_event) = NULL;
197}
198
199ZEND_API void zend_save_lexical_state(zend_lex_state *lex_state)
200{
201	lex_state->yy_leng   = SCNG(yy_leng);
202	lex_state->yy_start  = SCNG(yy_start);
203	lex_state->yy_text   = SCNG(yy_text);
204	lex_state->yy_cursor = SCNG(yy_cursor);
205	lex_state->yy_marker = SCNG(yy_marker);
206	lex_state->yy_limit  = SCNG(yy_limit);
207
208	lex_state->state_stack = SCNG(state_stack);
209	zend_stack_init(&SCNG(state_stack), sizeof(int));
210
211	lex_state->heredoc_label_stack = SCNG(heredoc_label_stack);
212	zend_ptr_stack_init(&SCNG(heredoc_label_stack));
213
214	lex_state->in = SCNG(yy_in);
215	lex_state->yy_state = YYSTATE;
216	lex_state->filename = zend_get_compiled_filename();
217	lex_state->lineno = CG(zend_lineno);
218
219	lex_state->script_org = SCNG(script_org);
220	lex_state->script_org_size = SCNG(script_org_size);
221	lex_state->script_filtered = SCNG(script_filtered);
222	lex_state->script_filtered_size = SCNG(script_filtered_size);
223	lex_state->input_filter = SCNG(input_filter);
224	lex_state->output_filter = SCNG(output_filter);
225	lex_state->script_encoding = SCNG(script_encoding);
226
227	lex_state->on_event = SCNG(on_event);
228
229	lex_state->ast = CG(ast);
230	lex_state->ast_arena = CG(ast_arena);
231}
232
233ZEND_API void zend_restore_lexical_state(zend_lex_state *lex_state)
234{
235	SCNG(yy_leng)   = lex_state->yy_leng;
236	SCNG(yy_start)  = lex_state->yy_start;
237	SCNG(yy_text)   = lex_state->yy_text;
238	SCNG(yy_cursor) = lex_state->yy_cursor;
239	SCNG(yy_marker) = lex_state->yy_marker;
240	SCNG(yy_limit)  = lex_state->yy_limit;
241
242	zend_stack_destroy(&SCNG(state_stack));
243	SCNG(state_stack) = lex_state->state_stack;
244
245	zend_ptr_stack_clean(&SCNG(heredoc_label_stack), (void (*)(void *)) &heredoc_label_dtor, 1);
246	zend_ptr_stack_destroy(&SCNG(heredoc_label_stack));
247	SCNG(heredoc_label_stack) = lex_state->heredoc_label_stack;
248
249	SCNG(yy_in) = lex_state->in;
250	YYSETCONDITION(lex_state->yy_state);
251	CG(zend_lineno) = lex_state->lineno;
252	zend_restore_compiled_filename(lex_state->filename);
253
254	if (SCNG(script_filtered)) {
255		efree(SCNG(script_filtered));
256		SCNG(script_filtered) = NULL;
257	}
258	SCNG(script_org) = lex_state->script_org;
259	SCNG(script_org_size) = lex_state->script_org_size;
260	SCNG(script_filtered) = lex_state->script_filtered;
261	SCNG(script_filtered_size) = lex_state->script_filtered_size;
262	SCNG(input_filter) = lex_state->input_filter;
263	SCNG(output_filter) = lex_state->output_filter;
264	SCNG(script_encoding) = lex_state->script_encoding;
265
266	SCNG(on_event) = lex_state->on_event;
267
268	CG(ast) = lex_state->ast;
269	CG(ast_arena) = lex_state->ast_arena;
270
271	RESET_DOC_COMMENT();
272}
273
274ZEND_API void zend_destroy_file_handle(zend_file_handle *file_handle)
275{
276	zend_llist_del_element(&CG(open_files), file_handle, (int (*)(void *, void *)) zend_compare_file_handles);
277	/* zend_file_handle_dtor() operates on the copy, so we have to NULLify the original here */
278	file_handle->opened_path = NULL;
279	if (file_handle->free_filename) {
280		file_handle->filename = NULL;
281	}
282}
283
284ZEND_API void zend_lex_tstring(zval *zv)
285{
286	if (SCNG(on_event)) SCNG(on_event)(ON_FEEDBACK, T_STRING, 0);
287
288	ZVAL_STRINGL(zv, (char*)SCNG(yy_text), SCNG(yy_leng));
289}
290
291#define BOM_UTF32_BE	"\x00\x00\xfe\xff"
292#define	BOM_UTF32_LE	"\xff\xfe\x00\x00"
293#define	BOM_UTF16_BE	"\xfe\xff"
294#define	BOM_UTF16_LE	"\xff\xfe"
295#define	BOM_UTF8		"\xef\xbb\xbf"
296
297static const zend_encoding *zend_multibyte_detect_utf_encoding(const unsigned char *script, size_t script_size)
298{
299	const unsigned char *p;
300	int wchar_size = 2;
301	int le = 0;
302
303	/* utf-16 or utf-32? */
304	p = script;
305	assert(p >= script);
306	while ((size_t)(p-script) < script_size) {
307		p = memchr(p, 0, script_size-(p-script)-2);
308		if (!p) {
309			break;
310		}
311		if (*(p+1) == '\0' && *(p+2) == '\0') {
312			wchar_size = 4;
313			break;
314		}
315
316		/* searching for UTF-32 specific byte orders, so this will do */
317		p += 4;
318	}
319
320	/* BE or LE? */
321	p = script;
322	assert(p >= script);
323	while ((size_t)(p-script) < script_size) {
324		if (*p == '\0' && *(p+wchar_size-1) != '\0') {
325			/* BE */
326			le = 0;
327			break;
328		} else if (*p != '\0' && *(p+wchar_size-1) == '\0') {
329			/* LE* */
330			le = 1;
331			break;
332		}
333		p += wchar_size;
334	}
335
336	if (wchar_size == 2) {
337		return le ? zend_multibyte_encoding_utf16le : zend_multibyte_encoding_utf16be;
338	} else {
339		return le ? zend_multibyte_encoding_utf32le : zend_multibyte_encoding_utf32be;
340	}
341
342	return NULL;
343}
344
345static const zend_encoding* zend_multibyte_detect_unicode(void)
346{
347	const zend_encoding *script_encoding = NULL;
348	int bom_size;
349	unsigned char *pos1, *pos2;
350
351	if (LANG_SCNG(script_org_size) < sizeof(BOM_UTF32_LE)-1) {
352		return NULL;
353	}
354
355	/* check out BOM */
356	if (!memcmp(LANG_SCNG(script_org), BOM_UTF32_BE, sizeof(BOM_UTF32_BE)-1)) {
357		script_encoding = zend_multibyte_encoding_utf32be;
358		bom_size = sizeof(BOM_UTF32_BE)-1;
359	} else if (!memcmp(LANG_SCNG(script_org), BOM_UTF32_LE, sizeof(BOM_UTF32_LE)-1)) {
360		script_encoding = zend_multibyte_encoding_utf32le;
361		bom_size = sizeof(BOM_UTF32_LE)-1;
362	} else if (!memcmp(LANG_SCNG(script_org), BOM_UTF16_BE, sizeof(BOM_UTF16_BE)-1)) {
363		script_encoding = zend_multibyte_encoding_utf16be;
364		bom_size = sizeof(BOM_UTF16_BE)-1;
365	} else if (!memcmp(LANG_SCNG(script_org), BOM_UTF16_LE, sizeof(BOM_UTF16_LE)-1)) {
366		script_encoding = zend_multibyte_encoding_utf16le;
367		bom_size = sizeof(BOM_UTF16_LE)-1;
368	} else if (!memcmp(LANG_SCNG(script_org), BOM_UTF8, sizeof(BOM_UTF8)-1)) {
369		script_encoding = zend_multibyte_encoding_utf8;
370		bom_size = sizeof(BOM_UTF8)-1;
371	}
372
373	if (script_encoding) {
374		/* remove BOM */
375		LANG_SCNG(script_org) += bom_size;
376		LANG_SCNG(script_org_size) -= bom_size;
377
378		return script_encoding;
379	}
380
381	/* script contains NULL bytes -> auto-detection */
382	if ((pos1 = memchr(LANG_SCNG(script_org), 0, LANG_SCNG(script_org_size)))) {
383		/* check if the NULL byte is after the __HALT_COMPILER(); */
384		pos2 = LANG_SCNG(script_org);
385
386		while (pos1 - pos2 >= sizeof("__HALT_COMPILER();")-1) {
387			pos2 = memchr(pos2, '_', pos1 - pos2);
388			if (!pos2) break;
389			pos2++;
390			if (strncasecmp((char*)pos2, "_HALT_COMPILER", sizeof("_HALT_COMPILER")-1) == 0) {
391				pos2 += sizeof("_HALT_COMPILER")-1;
392				while (*pos2 == ' '  ||
393					   *pos2 == '\t' ||
394					   *pos2 == '\r' ||
395					   *pos2 == '\n') {
396					pos2++;
397				}
398				if (*pos2 == '(') {
399					pos2++;
400					while (*pos2 == ' '  ||
401						   *pos2 == '\t' ||
402						   *pos2 == '\r' ||
403						   *pos2 == '\n') {
404						pos2++;
405					}
406					if (*pos2 == ')') {
407						pos2++;
408						while (*pos2 == ' '  ||
409							   *pos2 == '\t' ||
410							   *pos2 == '\r' ||
411							   *pos2 == '\n') {
412							pos2++;
413						}
414						if (*pos2 == ';') {
415							return NULL;
416						}
417					}
418				}
419			}
420		}
421		/* make best effort if BOM is missing */
422		return zend_multibyte_detect_utf_encoding(LANG_SCNG(script_org), LANG_SCNG(script_org_size));
423	}
424
425	return NULL;
426}
427
428static const zend_encoding* zend_multibyte_find_script_encoding(void)
429{
430	const zend_encoding *script_encoding;
431
432	if (CG(detect_unicode)) {
433		/* check out bom(byte order mark) and see if containing wchars */
434		script_encoding = zend_multibyte_detect_unicode();
435		if (script_encoding != NULL) {
436			/* bom or wchar detection is prior to 'script_encoding' option */
437			return script_encoding;
438		}
439	}
440
441	/* if no script_encoding specified, just leave alone */
442	if (!CG(script_encoding_list) || !CG(script_encoding_list_size)) {
443		return NULL;
444	}
445
446	/* if multiple encodings specified, detect automagically */
447	if (CG(script_encoding_list_size) > 1) {
448		return zend_multibyte_encoding_detector(LANG_SCNG(script_org), LANG_SCNG(script_org_size), CG(script_encoding_list), CG(script_encoding_list_size));
449	}
450
451	return CG(script_encoding_list)[0];
452}
453
454ZEND_API int zend_multibyte_set_filter(const zend_encoding *onetime_encoding)
455{
456	const zend_encoding *internal_encoding = zend_multibyte_get_internal_encoding();
457	const zend_encoding *script_encoding = onetime_encoding ? onetime_encoding: zend_multibyte_find_script_encoding();
458
459	if (!script_encoding) {
460		return FAILURE;
461	}
462
463	/* judge input/output filter */
464	LANG_SCNG(script_encoding) = script_encoding;
465	LANG_SCNG(input_filter) = NULL;
466	LANG_SCNG(output_filter) = NULL;
467
468	if (!internal_encoding || LANG_SCNG(script_encoding) == internal_encoding) {
469		if (!zend_multibyte_check_lexer_compatibility(LANG_SCNG(script_encoding))) {
470			/* and if not, work around w/ script_encoding -> utf-8 -> script_encoding conversion */
471			LANG_SCNG(input_filter) = encoding_filter_script_to_intermediate;
472			LANG_SCNG(output_filter) = encoding_filter_intermediate_to_script;
473		} else {
474			LANG_SCNG(input_filter) = NULL;
475			LANG_SCNG(output_filter) = NULL;
476		}
477		return SUCCESS;
478	}
479
480	if (zend_multibyte_check_lexer_compatibility(internal_encoding)) {
481		LANG_SCNG(input_filter) = encoding_filter_script_to_internal;
482		LANG_SCNG(output_filter) = NULL;
483	} else if (zend_multibyte_check_lexer_compatibility(LANG_SCNG(script_encoding))) {
484		LANG_SCNG(input_filter) = NULL;
485		LANG_SCNG(output_filter) = encoding_filter_script_to_internal;
486	} else {
487		/* both script and internal encodings are incompatible w/ flex */
488		LANG_SCNG(input_filter) = encoding_filter_script_to_intermediate;
489		LANG_SCNG(output_filter) = encoding_filter_intermediate_to_internal;
490	}
491
492	return 0;
493}
494
495ZEND_API int open_file_for_scanning(zend_file_handle *file_handle)
496{
497	char *buf;
498	size_t size, offset = 0;
499	zend_string *compiled_filename;
500
501	/* The shebang line was read, get the current position to obtain the buffer start */
502	if (CG(start_lineno) == 2 && file_handle->type == ZEND_HANDLE_FP && file_handle->handle.fp) {
503		if ((offset = ftell(file_handle->handle.fp)) == -1) {
504			offset = 0;
505		}
506	}
507
508	if (zend_stream_fixup(file_handle, &buf, &size) == FAILURE) {
509		return FAILURE;
510	}
511
512	zend_llist_add_element(&CG(open_files), file_handle);
513	if (file_handle->handle.stream.handle >= (void*)file_handle && file_handle->handle.stream.handle <= (void*)(file_handle+1)) {
514		zend_file_handle *fh = (zend_file_handle*)zend_llist_get_last(&CG(open_files));
515		size_t diff = (char*)file_handle->handle.stream.handle - (char*)file_handle;
516		fh->handle.stream.handle = (void*)(((char*)fh) + diff);
517		file_handle->handle.stream.handle = fh->handle.stream.handle;
518	}
519
520	/* Reset the scanner for scanning the new file */
521	SCNG(yy_in) = file_handle;
522	SCNG(yy_start) = NULL;
523
524	if (size != -1) {
525		if (CG(multibyte)) {
526			SCNG(script_org) = (unsigned char*)buf;
527			SCNG(script_org_size) = size;
528			SCNG(script_filtered) = NULL;
529
530			zend_multibyte_set_filter(NULL);
531
532			if (SCNG(input_filter)) {
533				if ((size_t)-1 == SCNG(input_filter)(&SCNG(script_filtered), &SCNG(script_filtered_size), SCNG(script_org), SCNG(script_org_size))) {
534					zend_error_noreturn(E_COMPILE_ERROR, "Could not convert the script from the detected "
535							"encoding \"%s\" to a compatible encoding", zend_multibyte_get_encoding_name(LANG_SCNG(script_encoding)));
536				}
537				buf = (char*)SCNG(script_filtered);
538				size = SCNG(script_filtered_size);
539			}
540		}
541		SCNG(yy_start) = (unsigned char *)buf - offset;
542		yy_scan_buffer(buf, (unsigned int)size);
543	} else {
544		zend_error_noreturn(E_COMPILE_ERROR, "zend_stream_mmap() failed");
545	}
546
547	BEGIN(INITIAL);
548
549	if (file_handle->opened_path) {
550		compiled_filename = zend_string_copy(file_handle->opened_path);
551	} else {
552		compiled_filename = zend_string_init(file_handle->filename, strlen(file_handle->filename), 0);
553	}
554
555	zend_set_compiled_filename(compiled_filename);
556	zend_string_release(compiled_filename);
557
558	if (CG(start_lineno)) {
559		CG(zend_lineno) = CG(start_lineno);
560		CG(start_lineno) = 0;
561	} else {
562		CG(zend_lineno) = 1;
563	}
564
565	RESET_DOC_COMMENT();
566	CG(increment_lineno) = 0;
567	return SUCCESS;
568}
569END_EXTERN_C()
570
571static zend_op_array *zend_compile(int type)
572{
573	zend_op_array *op_array = NULL;
574	zend_bool original_in_compilation = CG(in_compilation);
575
576	CG(in_compilation) = 1;
577	CG(ast) = NULL;
578	CG(ast_arena) = zend_arena_create(1024 * 32);
579
580	if (!zendparse()) {
581		zend_file_context original_file_context;
582		zend_oparray_context original_oparray_context;
583		zend_op_array *original_active_op_array = CG(active_op_array);
584
585		op_array = emalloc(sizeof(zend_op_array));
586		init_op_array(op_array, type, INITIAL_OP_ARRAY_SIZE);
587		CG(active_op_array) = op_array;
588
589		if (zend_ast_process) {
590			zend_ast_process(CG(ast));
591		}
592
593		zend_file_context_begin(&original_file_context);
594		zend_oparray_context_begin(&original_oparray_context);
595		zend_compile_top_stmt(CG(ast));
596		zend_emit_final_return(type == ZEND_USER_FUNCTION);
597		op_array->line_start = 1;
598		op_array->line_end = CG(zend_lineno);
599		pass_two(op_array);
600		zend_oparray_context_end(&original_oparray_context);
601		zend_file_context_end(&original_file_context);
602
603		CG(active_op_array) = original_active_op_array;
604	}
605
606	zend_ast_destroy(CG(ast));
607	zend_arena_destroy(CG(ast_arena));
608
609	CG(in_compilation) = original_in_compilation;
610
611	return op_array;
612}
613
614ZEND_API zend_op_array *compile_file(zend_file_handle *file_handle, int type)
615{
616	zend_lex_state original_lex_state;
617	zend_op_array *op_array = NULL;
618	zend_save_lexical_state(&original_lex_state);
619
620	if (open_file_for_scanning(file_handle)==FAILURE) {
621		if (type==ZEND_REQUIRE) {
622			zend_message_dispatcher(ZMSG_FAILED_REQUIRE_FOPEN, file_handle->filename);
623			zend_bailout();
624		} else {
625			zend_message_dispatcher(ZMSG_FAILED_INCLUDE_FOPEN, file_handle->filename);
626		}
627	} else {
628		op_array = zend_compile(ZEND_USER_FUNCTION);
629	}
630
631	zend_restore_lexical_state(&original_lex_state);
632	return op_array;
633}
634
635
636zend_op_array *compile_filename(int type, zval *filename)
637{
638	zend_file_handle file_handle;
639	zval tmp;
640	zend_op_array *retval;
641	zend_string *opened_path = NULL;
642
643	if (Z_TYPE_P(filename) != IS_STRING) {
644		tmp = *filename;
645		zval_copy_ctor(&tmp);
646		convert_to_string(&tmp);
647		filename = &tmp;
648	}
649	file_handle.filename = Z_STRVAL_P(filename);
650	file_handle.free_filename = 0;
651	file_handle.type = ZEND_HANDLE_FILENAME;
652	file_handle.opened_path = NULL;
653	file_handle.handle.fp = NULL;
654
655	retval = zend_compile_file(&file_handle, type);
656	if (retval && file_handle.handle.stream.handle) {
657		if (!file_handle.opened_path) {
658			file_handle.opened_path = opened_path = zend_string_copy(Z_STR_P(filename));
659		}
660
661		zend_hash_add_empty_element(&EG(included_files), file_handle.opened_path);
662
663		if (opened_path) {
664			zend_string_release(opened_path);
665		}
666	}
667	zend_destroy_file_handle(&file_handle);
668
669	if (filename==&tmp) {
670		zval_dtor(&tmp);
671	}
672	return retval;
673}
674
675ZEND_API int zend_prepare_string_for_scanning(zval *str, char *filename)
676{
677	char *buf;
678	size_t size, old_len;
679	zend_string *new_compiled_filename;
680
681	/* enforce ZEND_MMAP_AHEAD trailing NULLs for flex... */
682	old_len = Z_STRLEN_P(str);
683	Z_STR_P(str) = zend_string_extend(Z_STR_P(str), old_len + ZEND_MMAP_AHEAD, 0);
684	Z_TYPE_INFO_P(str) = IS_STRING_EX;
685	memset(Z_STRVAL_P(str) + old_len, 0, ZEND_MMAP_AHEAD + 1);
686
687	SCNG(yy_in) = NULL;
688	SCNG(yy_start) = NULL;
689
690	buf = Z_STRVAL_P(str);
691	size = old_len;
692
693	if (CG(multibyte)) {
694		SCNG(script_org) = (unsigned char*)buf;
695		SCNG(script_org_size) = size;
696		SCNG(script_filtered) = NULL;
697
698		zend_multibyte_set_filter(zend_multibyte_get_internal_encoding());
699
700		if (SCNG(input_filter)) {
701			if ((size_t)-1 == SCNG(input_filter)(&SCNG(script_filtered), &SCNG(script_filtered_size), SCNG(script_org), SCNG(script_org_size))) {
702				zend_error_noreturn(E_COMPILE_ERROR, "Could not convert the script from the detected "
703						"encoding \"%s\" to a compatible encoding", zend_multibyte_get_encoding_name(LANG_SCNG(script_encoding)));
704			}
705			buf = (char*)SCNG(script_filtered);
706			size = SCNG(script_filtered_size);
707		}
708	}
709
710	yy_scan_buffer(buf, (unsigned int)size);
711
712	new_compiled_filename = zend_string_init(filename, strlen(filename), 0);
713	zend_set_compiled_filename(new_compiled_filename);
714	zend_string_release(new_compiled_filename);
715	CG(zend_lineno) = 1;
716	CG(increment_lineno) = 0;
717	RESET_DOC_COMMENT();
718	return SUCCESS;
719}
720
721
722ZEND_API size_t zend_get_scanned_file_offset(void)
723{
724	size_t offset = SCNG(yy_cursor) - SCNG(yy_start);
725	if (SCNG(input_filter)) {
726		size_t original_offset = offset, length = 0;
727		do {
728			unsigned char *p = NULL;
729			if ((size_t)-1 == SCNG(input_filter)(&p, &length, SCNG(script_org), offset)) {
730				return (size_t)-1;
731			}
732			efree(p);
733			if (length > original_offset) {
734				offset--;
735			} else if (length < original_offset) {
736				offset++;
737			}
738		} while (original_offset != length);
739	}
740	return offset;
741}
742
743zend_op_array *compile_string(zval *source_string, char *filename)
744{
745	zend_lex_state original_lex_state;
746	zend_op_array *op_array = NULL;
747	zval tmp;
748
749	if (Z_STRLEN_P(source_string)==0) {
750		return NULL;
751	}
752
753	ZVAL_DUP(&tmp, source_string);
754	convert_to_string(&tmp);
755	source_string = &tmp;
756
757	zend_save_lexical_state(&original_lex_state);
758	if (zend_prepare_string_for_scanning(source_string, filename) == SUCCESS) {
759		BEGIN(ST_IN_SCRIPTING);
760		op_array = zend_compile(ZEND_EVAL_CODE);
761	}
762
763	zend_restore_lexical_state(&original_lex_state);
764	zval_dtor(&tmp);
765
766	return op_array;
767}
768
769
770BEGIN_EXTERN_C()
771int highlight_file(char *filename, zend_syntax_highlighter_ini *syntax_highlighter_ini)
772{
773	zend_lex_state original_lex_state;
774	zend_file_handle file_handle;
775
776	file_handle.type = ZEND_HANDLE_FILENAME;
777	file_handle.filename = filename;
778	file_handle.free_filename = 0;
779	file_handle.opened_path = NULL;
780	zend_save_lexical_state(&original_lex_state);
781	if (open_file_for_scanning(&file_handle)==FAILURE) {
782		zend_message_dispatcher(ZMSG_FAILED_HIGHLIGHT_FOPEN, filename);
783		zend_restore_lexical_state(&original_lex_state);
784		return FAILURE;
785	}
786	zend_highlight(syntax_highlighter_ini);
787	if (SCNG(script_filtered)) {
788		efree(SCNG(script_filtered));
789		SCNG(script_filtered) = NULL;
790	}
791	zend_destroy_file_handle(&file_handle);
792	zend_restore_lexical_state(&original_lex_state);
793	return SUCCESS;
794}
795
796int highlight_string(zval *str, zend_syntax_highlighter_ini *syntax_highlighter_ini, char *str_name)
797{
798	zend_lex_state original_lex_state;
799	zval tmp = *str;
800
801	str = &tmp;
802	zval_copy_ctor(str);
803	zend_save_lexical_state(&original_lex_state);
804	if (zend_prepare_string_for_scanning(str, str_name)==FAILURE) {
805		zend_restore_lexical_state(&original_lex_state);
806		return FAILURE;
807	}
808	BEGIN(INITIAL);
809	zend_highlight(syntax_highlighter_ini);
810	if (SCNG(script_filtered)) {
811		efree(SCNG(script_filtered));
812		SCNG(script_filtered) = NULL;
813	}
814	zend_restore_lexical_state(&original_lex_state);
815	zval_dtor(str);
816	return SUCCESS;
817}
818
819ZEND_API void zend_multibyte_yyinput_again(zend_encoding_filter old_input_filter, const zend_encoding *old_encoding)
820{
821	size_t length;
822	unsigned char *new_yy_start;
823
824	/* convert and set */
825	if (!SCNG(input_filter)) {
826		if (SCNG(script_filtered)) {
827			efree(SCNG(script_filtered));
828			SCNG(script_filtered) = NULL;
829		}
830		SCNG(script_filtered_size) = 0;
831		length = SCNG(script_org_size);
832		new_yy_start = SCNG(script_org);
833	} else {
834		if ((size_t)-1 == SCNG(input_filter)(&new_yy_start, &length, SCNG(script_org), SCNG(script_org_size))) {
835			zend_error_noreturn(E_COMPILE_ERROR, "Could not convert the script from the detected "
836					"encoding \"%s\" to a compatible encoding", zend_multibyte_get_encoding_name(LANG_SCNG(script_encoding)));
837		}
838		if (SCNG(script_filtered)) {
839			efree(SCNG(script_filtered));
840		}
841		SCNG(script_filtered) = new_yy_start;
842		SCNG(script_filtered_size) = length;
843	}
844
845	SCNG(yy_cursor) = new_yy_start + (SCNG(yy_cursor) - SCNG(yy_start));
846	SCNG(yy_marker) = new_yy_start + (SCNG(yy_marker) - SCNG(yy_start));
847	SCNG(yy_text) = new_yy_start + (SCNG(yy_text) - SCNG(yy_start));
848	SCNG(yy_limit) = new_yy_start + length;
849
850	SCNG(yy_start) = new_yy_start;
851}
852
853
854// TODO: avoid reallocation ???
855# define zend_copy_value(zendlval, yytext, yyleng) \
856	if (SCNG(output_filter)) { \
857		size_t sz = 0; \
858		char *s = NULL; \
859		SCNG(output_filter)((unsigned char **)&s, &sz, (unsigned char *)yytext, (size_t)yyleng); \
860		ZVAL_STRINGL(zendlval, s, sz); \
861		efree(s); \
862	} else { \
863		ZVAL_STRINGL(zendlval, yytext, yyleng); \
864	}
865
866static int zend_scan_escape_string(zval *zendlval, char *str, int len, char quote_type)
867{
868	register char *s, *t;
869	char *end;
870
871	ZVAL_STRINGL(zendlval, str, len);
872
873	/* convert escape sequences */
874	s = t = Z_STRVAL_P(zendlval);
875	end = s+Z_STRLEN_P(zendlval);
876	while (s<end) {
877		if (*s=='\\') {
878			s++;
879			if (s >= end) {
880				*t++ = '\\';
881				break;
882			}
883
884			switch(*s) {
885				case 'n':
886					*t++ = '\n';
887					Z_STRLEN_P(zendlval)--;
888					break;
889				case 'r':
890					*t++ = '\r';
891					Z_STRLEN_P(zendlval)--;
892					break;
893				case 't':
894					*t++ = '\t';
895					Z_STRLEN_P(zendlval)--;
896					break;
897				case 'f':
898					*t++ = '\f';
899					Z_STRLEN_P(zendlval)--;
900					break;
901				case 'v':
902					*t++ = '\v';
903					Z_STRLEN_P(zendlval)--;
904					break;
905				case 'e':
906#ifdef ZEND_WIN32
907					*t++ = VK_ESCAPE;
908#else
909					*t++ = '\e';
910#endif
911					Z_STRLEN_P(zendlval)--;
912					break;
913				case '"':
914				case '`':
915					if (*s != quote_type) {
916						*t++ = '\\';
917						*t++ = *s;
918						break;
919					}
920				case '\\':
921				case '$':
922					*t++ = *s;
923					Z_STRLEN_P(zendlval)--;
924					break;
925				case 'x':
926				case 'X':
927					if (ZEND_IS_HEX(*(s+1))) {
928						char hex_buf[3] = { 0, 0, 0 };
929
930						Z_STRLEN_P(zendlval)--; /* for the 'x' */
931
932						hex_buf[0] = *(++s);
933						Z_STRLEN_P(zendlval)--;
934						if (ZEND_IS_HEX(*(s+1))) {
935							hex_buf[1] = *(++s);
936							Z_STRLEN_P(zendlval)--;
937						}
938						*t++ = (char) ZEND_STRTOL(hex_buf, NULL, 16);
939					} else {
940						*t++ = '\\';
941						*t++ = *s;
942					}
943					break;
944				/* UTF-8 codepoint escape, format: /\\u\{\x+\}/ */
945				case 'u':
946					{
947						/* cache where we started so we can parse after validating */
948						char *start = s + 1;
949						size_t len = 0;
950						zend_bool valid = 1;
951						unsigned long codepoint;
952						size_t byte_len = 0;
953
954						if (*start != '{') {
955							/* we silently let this pass to avoid breaking code
956							 * with JSON in string literals (e.g. "\"\u202e\""
957							 */
958							*t++ = '\\';
959							*t++ = 'u';
960							break;
961						} else {
962							/* on the other hand, invalid \u{blah} errors */
963							s++;
964							len++;
965							s++;
966							while (*s != '}') {
967								if (!ZEND_IS_HEX(*s)) {
968									valid = 0;
969									break;
970								} else {
971									len++;
972								}
973								s++;
974							}
975							if (*s == '}') {
976								valid = 1;
977								len++;
978							}
979						}
980
981						/* \u{} is invalid */
982						if (len <= 2) {
983							valid = 0;
984						}
985
986						if (!valid) {
987							zend_throw_exception(zend_ce_parse_error,
988								"Invalid UTF-8 codepoint escape sequence", 0);
989							zval_ptr_dtor(zendlval);
990							ZVAL_UNDEF(zendlval);
991							return FAILURE;
992						}
993
994						errno = 0;
995						codepoint = strtoul(start + 1, NULL, 16);
996
997						/* per RFC 3629, UTF-8 can only represent 21 bits */
998						if (codepoint > 0x10FFFF || errno) {
999							zend_throw_exception(zend_ce_parse_error,
1000								"Invalid UTF-8 codepoint escape sequence: Codepoint too large", 0);
1001							zval_ptr_dtor(zendlval);
1002							ZVAL_UNDEF(zendlval);
1003							return FAILURE;
1004						}
1005
1006						/* based on https://en.wikipedia.org/wiki/UTF-8#Sample_code */
1007						if (codepoint < 0x80) {
1008							byte_len = 1;
1009							*t++ = codepoint;
1010						} else if (codepoint <= 0x7FF) {
1011							byte_len = 2;
1012							*t++ = (codepoint >> 6) + 0xC0;
1013							*t++ = (codepoint & 0x3F) + 0x80;
1014						} else if (codepoint <= 0xFFFF) {
1015							byte_len = 3;
1016							*t++ = (codepoint >> 12) + 0xE0;
1017							*t++ = ((codepoint >> 6) & 0x3F) + 0x80;
1018							*t++ = (codepoint & 0x3F) + 0x80;
1019						} else if (codepoint <= 0x10FFFF) {
1020							byte_len = 4;
1021							*t++ = (codepoint >> 18) + 0xF0;
1022							*t++ = ((codepoint >> 12) & 0x3F) + 0x80;
1023							*t++ = ((codepoint >> 6) & 0x3F) + 0x80;
1024							*t++ = (codepoint & 0x3F) + 0x80;
1025						}
1026
1027						Z_STRLEN_P(zendlval) -= 2; /* \u */
1028						Z_STRLEN_P(zendlval) -= (len - byte_len);
1029					}
1030					break;
1031				default:
1032					/* check for an octal */
1033					if (ZEND_IS_OCT(*s)) {
1034						char octal_buf[4] = { 0, 0, 0, 0 };
1035
1036						octal_buf[0] = *s;
1037						Z_STRLEN_P(zendlval)--;
1038						if (ZEND_IS_OCT(*(s+1))) {
1039							octal_buf[1] = *(++s);
1040							Z_STRLEN_P(zendlval)--;
1041							if (ZEND_IS_OCT(*(s+1))) {
1042								octal_buf[2] = *(++s);
1043								Z_STRLEN_P(zendlval)--;
1044							}
1045						}
1046						*t++ = (char) ZEND_STRTOL(octal_buf, NULL, 8);
1047					} else {
1048						*t++ = '\\';
1049						*t++ = *s;
1050					}
1051					break;
1052			}
1053		} else {
1054			*t++ = *s;
1055		}
1056
1057		if (*s == '\n' || (*s == '\r' && (*(s+1) != '\n'))) {
1058			CG(zend_lineno)++;
1059		}
1060		s++;
1061	}
1062	*t = 0;
1063	if (SCNG(output_filter)) {
1064		size_t sz = 0;
1065		unsigned char *str;
1066		// TODO: avoid realocation ???
1067		s = Z_STRVAL_P(zendlval);
1068		SCNG(output_filter)(&str, &sz, (unsigned char *)s, (size_t)Z_STRLEN_P(zendlval));
1069		zval_ptr_dtor(zendlval);
1070		ZVAL_STRINGL(zendlval, (char *) str, sz);
1071		efree(str);
1072	}
1073	return SUCCESS;
1074}
1075
1076static zend_always_inline int emit_token(int token, int token_line)
1077{
1078	if(SCNG(on_event)) SCNG(on_event)(ON_TOKEN, token, token_line);
1079
1080	return token;
1081}
1082
1083#define RETURN_TOKEN(token) return emit_token(token, start_line);
1084
1085int lex_scan(zval *zendlval)
1086{
1087
1088int start_line = CG(zend_lineno);
1089
1090restart:
1091	SCNG(yy_text) = YYCURSOR;
1092
1093/*!re2c
1094re2c:yyfill:check = 0;
1095LNUM	[0-9]+
1096DNUM	([0-9]*"."[0-9]+)|([0-9]+"."[0-9]*)
1097EXPONENT_DNUM	(({LNUM}|{DNUM})[eE][+-]?{LNUM})
1098HNUM	"0x"[0-9a-fA-F]+
1099BNUM	"0b"[01]+
1100LABEL	[a-zA-Z_\x80-\xff][a-zA-Z0-9_\x80-\xff]*
1101WHITESPACE [ \n\r\t]+
1102TABS_AND_SPACES [ \t]*
1103TOKENS [;:,.\[\]()|^&+-/*=%!~$<>?@]
1104ANY_CHAR [^]
1105NEWLINE ("\r"|"\n"|"\r\n")
1106
1107/* compute yyleng before each rule */
1108<!*> := yyleng = YYCURSOR - SCNG(yy_text);
1109
1110<ST_IN_SCRIPTING>"exit" {
1111	RETURN_TOKEN(T_EXIT);
1112}
1113
1114<ST_IN_SCRIPTING>"die" {
1115	RETURN_TOKEN(T_EXIT);
1116}
1117
1118<ST_IN_SCRIPTING>"function" {
1119	RETURN_TOKEN(T_FUNCTION);
1120}
1121
1122<ST_IN_SCRIPTING>"const" {
1123	RETURN_TOKEN(T_CONST);
1124}
1125
1126<ST_IN_SCRIPTING>"return" {
1127	RETURN_TOKEN(T_RETURN);
1128}
1129
1130<ST_IN_SCRIPTING>"yield"{WHITESPACE}"from" {
1131	HANDLE_NEWLINES(yytext, yyleng);
1132	RETURN_TOKEN(T_YIELD_FROM);
1133}
1134
1135<ST_IN_SCRIPTING>"yield" {
1136	RETURN_TOKEN(T_YIELD);
1137}
1138
1139<ST_IN_SCRIPTING>"try" {
1140	RETURN_TOKEN(T_TRY);
1141}
1142
1143<ST_IN_SCRIPTING>"catch" {
1144	RETURN_TOKEN(T_CATCH);
1145}
1146
1147<ST_IN_SCRIPTING>"finally" {
1148	RETURN_TOKEN(T_FINALLY);
1149}
1150
1151<ST_IN_SCRIPTING>"throw" {
1152	RETURN_TOKEN(T_THROW);
1153}
1154
1155<ST_IN_SCRIPTING>"if" {
1156	RETURN_TOKEN(T_IF);
1157}
1158
1159<ST_IN_SCRIPTING>"elseif" {
1160	RETURN_TOKEN(T_ELSEIF);
1161}
1162
1163<ST_IN_SCRIPTING>"endif" {
1164	RETURN_TOKEN(T_ENDIF);
1165}
1166
1167<ST_IN_SCRIPTING>"else" {
1168	RETURN_TOKEN(T_ELSE);
1169}
1170
1171<ST_IN_SCRIPTING>"while" {
1172	RETURN_TOKEN(T_WHILE);
1173}
1174
1175<ST_IN_SCRIPTING>"endwhile" {
1176	RETURN_TOKEN(T_ENDWHILE);
1177}
1178
1179<ST_IN_SCRIPTING>"do" {
1180	RETURN_TOKEN(T_DO);
1181}
1182
1183<ST_IN_SCRIPTING>"for" {
1184	RETURN_TOKEN(T_FOR);
1185}
1186
1187<ST_IN_SCRIPTING>"endfor" {
1188	RETURN_TOKEN(T_ENDFOR);
1189}
1190
1191<ST_IN_SCRIPTING>"foreach" {
1192	RETURN_TOKEN(T_FOREACH);
1193}
1194
1195<ST_IN_SCRIPTING>"endforeach" {
1196	RETURN_TOKEN(T_ENDFOREACH);
1197}
1198
1199<ST_IN_SCRIPTING>"declare" {
1200	RETURN_TOKEN(T_DECLARE);
1201}
1202
1203<ST_IN_SCRIPTING>"enddeclare" {
1204	RETURN_TOKEN(T_ENDDECLARE);
1205}
1206
1207<ST_IN_SCRIPTING>"instanceof" {
1208	RETURN_TOKEN(T_INSTANCEOF);
1209}
1210
1211<ST_IN_SCRIPTING>"as" {
1212	RETURN_TOKEN(T_AS);
1213}
1214
1215<ST_IN_SCRIPTING>"switch" {
1216	RETURN_TOKEN(T_SWITCH);
1217}
1218
1219<ST_IN_SCRIPTING>"endswitch" {
1220	RETURN_TOKEN(T_ENDSWITCH);
1221}
1222
1223<ST_IN_SCRIPTING>"case" {
1224	RETURN_TOKEN(T_CASE);
1225}
1226
1227<ST_IN_SCRIPTING>"default" {
1228	RETURN_TOKEN(T_DEFAULT);
1229}
1230
1231<ST_IN_SCRIPTING>"break" {
1232	RETURN_TOKEN(T_BREAK);
1233}
1234
1235<ST_IN_SCRIPTING>"continue" {
1236	RETURN_TOKEN(T_CONTINUE);
1237}
1238
1239<ST_IN_SCRIPTING>"goto" {
1240	RETURN_TOKEN(T_GOTO);
1241}
1242
1243<ST_IN_SCRIPTING>"echo" {
1244	RETURN_TOKEN(T_ECHO);
1245}
1246
1247<ST_IN_SCRIPTING>"print" {
1248	RETURN_TOKEN(T_PRINT);
1249}
1250
1251<ST_IN_SCRIPTING>"class" {
1252	RETURN_TOKEN(T_CLASS);
1253}
1254
1255<ST_IN_SCRIPTING>"interface" {
1256	RETURN_TOKEN(T_INTERFACE);
1257}
1258
1259<ST_IN_SCRIPTING>"trait" {
1260	RETURN_TOKEN(T_TRAIT);
1261}
1262
1263<ST_IN_SCRIPTING>"extends" {
1264	RETURN_TOKEN(T_EXTENDS);
1265}
1266
1267<ST_IN_SCRIPTING>"implements" {
1268	RETURN_TOKEN(T_IMPLEMENTS);
1269}
1270
1271<ST_IN_SCRIPTING>"->" {
1272	yy_push_state(ST_LOOKING_FOR_PROPERTY);
1273	RETURN_TOKEN(T_OBJECT_OPERATOR);
1274}
1275
1276<ST_IN_SCRIPTING,ST_LOOKING_FOR_PROPERTY>{WHITESPACE}+ {
1277	HANDLE_NEWLINES(yytext, yyleng);
1278	RETURN_TOKEN(T_WHITESPACE);
1279}
1280
1281<ST_LOOKING_FOR_PROPERTY>"->" {
1282	RETURN_TOKEN(T_OBJECT_OPERATOR);
1283}
1284
1285<ST_LOOKING_FOR_PROPERTY>{LABEL} {
1286	yy_pop_state();
1287	zend_copy_value(zendlval, yytext, yyleng);
1288	RETURN_TOKEN(T_STRING);
1289}
1290
1291<ST_LOOKING_FOR_PROPERTY>{ANY_CHAR} {
1292	yyless(0);
1293	yy_pop_state();
1294	goto restart;
1295}
1296
1297<ST_IN_SCRIPTING>"::" {
1298	RETURN_TOKEN(T_PAAMAYIM_NEKUDOTAYIM);
1299}
1300
1301<ST_IN_SCRIPTING>"\\" {
1302	RETURN_TOKEN(T_NS_SEPARATOR);
1303}
1304
1305<ST_IN_SCRIPTING>"..." {
1306	RETURN_TOKEN(T_ELLIPSIS);
1307}
1308
1309<ST_IN_SCRIPTING>"??" {
1310	RETURN_TOKEN(T_COALESCE);
1311}
1312
1313<ST_IN_SCRIPTING>"new" {
1314	RETURN_TOKEN(T_NEW);
1315}
1316
1317<ST_IN_SCRIPTING>"clone" {
1318	RETURN_TOKEN(T_CLONE);
1319}
1320
1321<ST_IN_SCRIPTING>"var" {
1322	RETURN_TOKEN(T_VAR);
1323}
1324
1325<ST_IN_SCRIPTING>"("{TABS_AND_SPACES}("int"|"integer"){TABS_AND_SPACES}")" {
1326	RETURN_TOKEN(T_INT_CAST);
1327}
1328
1329<ST_IN_SCRIPTING>"("{TABS_AND_SPACES}("real"|"double"|"float"){TABS_AND_SPACES}")" {
1330	RETURN_TOKEN(T_DOUBLE_CAST);
1331}
1332
1333<ST_IN_SCRIPTING>"("{TABS_AND_SPACES}("string"|"binary"){TABS_AND_SPACES}")" {
1334	RETURN_TOKEN(T_STRING_CAST);
1335}
1336
1337<ST_IN_SCRIPTING>"("{TABS_AND_SPACES}"array"{TABS_AND_SPACES}")" {
1338	RETURN_TOKEN(T_ARRAY_CAST);
1339}
1340
1341<ST_IN_SCRIPTING>"("{TABS_AND_SPACES}"object"{TABS_AND_SPACES}")" {
1342	RETURN_TOKEN(T_OBJECT_CAST);
1343}
1344
1345<ST_IN_SCRIPTING>"("{TABS_AND_SPACES}("bool"|"boolean"){TABS_AND_SPACES}")" {
1346	RETURN_TOKEN(T_BOOL_CAST);
1347}
1348
1349<ST_IN_SCRIPTING>"("{TABS_AND_SPACES}("unset"){TABS_AND_SPACES}")" {
1350	RETURN_TOKEN(T_UNSET_CAST);
1351}
1352
1353<ST_IN_SCRIPTING>"eval" {
1354	RETURN_TOKEN(T_EVAL);
1355}
1356
1357<ST_IN_SCRIPTING>"include" {
1358	RETURN_TOKEN(T_INCLUDE);
1359}
1360
1361<ST_IN_SCRIPTING>"include_once" {
1362	RETURN_TOKEN(T_INCLUDE_ONCE);
1363}
1364
1365<ST_IN_SCRIPTING>"require" {
1366	RETURN_TOKEN(T_REQUIRE);
1367}
1368
1369<ST_IN_SCRIPTING>"require_once" {
1370	RETURN_TOKEN(T_REQUIRE_ONCE);
1371}
1372
1373<ST_IN_SCRIPTING>"namespace" {
1374	RETURN_TOKEN(T_NAMESPACE);
1375}
1376
1377<ST_IN_SCRIPTING>"use" {
1378	RETURN_TOKEN(T_USE);
1379}
1380
1381<ST_IN_SCRIPTING>"insteadof" {
1382    RETURN_TOKEN(T_INSTEADOF);
1383}
1384
1385<ST_IN_SCRIPTING>"global" {
1386	RETURN_TOKEN(T_GLOBAL);
1387}
1388
1389<ST_IN_SCRIPTING>"isset" {
1390	RETURN_TOKEN(T_ISSET);
1391}
1392
1393<ST_IN_SCRIPTING>"empty" {
1394	RETURN_TOKEN(T_EMPTY);
1395}
1396
1397<ST_IN_SCRIPTING>"__halt_compiler" {
1398	RETURN_TOKEN(T_HALT_COMPILER);
1399}
1400
1401<ST_IN_SCRIPTING>"static" {
1402	RETURN_TOKEN(T_STATIC);
1403}
1404
1405<ST_IN_SCRIPTING>"abstract" {
1406	RETURN_TOKEN(T_ABSTRACT);
1407}
1408
1409<ST_IN_SCRIPTING>"final" {
1410	RETURN_TOKEN(T_FINAL);
1411}
1412
1413<ST_IN_SCRIPTING>"private" {
1414	RETURN_TOKEN(T_PRIVATE);
1415}
1416
1417<ST_IN_SCRIPTING>"protected" {
1418	RETURN_TOKEN(T_PROTECTED);
1419}
1420
1421<ST_IN_SCRIPTING>"public" {
1422	RETURN_TOKEN(T_PUBLIC);
1423}
1424
1425<ST_IN_SCRIPTING>"unset" {
1426	RETURN_TOKEN(T_UNSET);
1427}
1428
1429<ST_IN_SCRIPTING>"=>" {
1430	RETURN_TOKEN(T_DOUBLE_ARROW);
1431}
1432
1433<ST_IN_SCRIPTING>"list" {
1434	RETURN_TOKEN(T_LIST);
1435}
1436
1437<ST_IN_SCRIPTING>"array" {
1438	RETURN_TOKEN(T_ARRAY);
1439}
1440
1441<ST_IN_SCRIPTING>"callable" {
1442	RETURN_TOKEN(T_CALLABLE);
1443}
1444
1445<ST_IN_SCRIPTING>"++" {
1446	RETURN_TOKEN(T_INC);
1447}
1448
1449<ST_IN_SCRIPTING>"--" {
1450	RETURN_TOKEN(T_DEC);
1451}
1452
1453<ST_IN_SCRIPTING>"===" {
1454	RETURN_TOKEN(T_IS_IDENTICAL);
1455}
1456
1457<ST_IN_SCRIPTING>"!==" {
1458	RETURN_TOKEN(T_IS_NOT_IDENTICAL);
1459}
1460
1461<ST_IN_SCRIPTING>"==" {
1462	RETURN_TOKEN(T_IS_EQUAL);
1463}
1464
1465<ST_IN_SCRIPTING>"!="|"<>" {
1466	RETURN_TOKEN(T_IS_NOT_EQUAL);
1467}
1468
1469<ST_IN_SCRIPTING>"<=>" {
1470	RETURN_TOKEN(T_SPACESHIP);
1471}
1472
1473<ST_IN_SCRIPTING>"<=" {
1474	RETURN_TOKEN(T_IS_SMALLER_OR_EQUAL);
1475}
1476
1477<ST_IN_SCRIPTING>">=" {
1478	RETURN_TOKEN(T_IS_GREATER_OR_EQUAL);
1479}
1480
1481<ST_IN_SCRIPTING>"+=" {
1482	RETURN_TOKEN(T_PLUS_EQUAL);
1483}
1484
1485<ST_IN_SCRIPTING>"-=" {
1486	RETURN_TOKEN(T_MINUS_EQUAL);
1487}
1488
1489<ST_IN_SCRIPTING>"*=" {
1490	RETURN_TOKEN(T_MUL_EQUAL);
1491}
1492
1493<ST_IN_SCRIPTING>"*\*" {
1494	RETURN_TOKEN(T_POW);
1495}
1496
1497<ST_IN_SCRIPTING>"*\*=" {
1498	RETURN_TOKEN(T_POW_EQUAL);
1499}
1500
1501<ST_IN_SCRIPTING>"/=" {
1502	RETURN_TOKEN(T_DIV_EQUAL);
1503}
1504
1505<ST_IN_SCRIPTING>".=" {
1506	RETURN_TOKEN(T_CONCAT_EQUAL);
1507}
1508
1509<ST_IN_SCRIPTING>"%=" {
1510	RETURN_TOKEN(T_MOD_EQUAL);
1511}
1512
1513<ST_IN_SCRIPTING>"<<=" {
1514	RETURN_TOKEN(T_SL_EQUAL);
1515}
1516
1517<ST_IN_SCRIPTING>">>=" {
1518	RETURN_TOKEN(T_SR_EQUAL);
1519}
1520
1521<ST_IN_SCRIPTING>"&=" {
1522	RETURN_TOKEN(T_AND_EQUAL);
1523}
1524
1525<ST_IN_SCRIPTING>"|=" {
1526	RETURN_TOKEN(T_OR_EQUAL);
1527}
1528
1529<ST_IN_SCRIPTING>"^=" {
1530	RETURN_TOKEN(T_XOR_EQUAL);
1531}
1532
1533<ST_IN_SCRIPTING>"||" {
1534	RETURN_TOKEN(T_BOOLEAN_OR);
1535}
1536
1537<ST_IN_SCRIPTING>"&&" {
1538	RETURN_TOKEN(T_BOOLEAN_AND);
1539}
1540
1541<ST_IN_SCRIPTING>"OR" {
1542	RETURN_TOKEN(T_LOGICAL_OR);
1543}
1544
1545<ST_IN_SCRIPTING>"AND" {
1546	RETURN_TOKEN(T_LOGICAL_AND);
1547}
1548
1549<ST_IN_SCRIPTING>"XOR" {
1550	RETURN_TOKEN(T_LOGICAL_XOR);
1551}
1552
1553<ST_IN_SCRIPTING>"<<" {
1554	RETURN_TOKEN(T_SL);
1555}
1556
1557<ST_IN_SCRIPTING>">>" {
1558	RETURN_TOKEN(T_SR);
1559}
1560
1561<ST_IN_SCRIPTING>{TOKENS} {
1562	RETURN_TOKEN(yytext[0]);
1563}
1564
1565
1566<ST_IN_SCRIPTING>"{" {
1567	yy_push_state(ST_IN_SCRIPTING);
1568	RETURN_TOKEN('{');
1569}
1570
1571
1572<ST_DOUBLE_QUOTES,ST_BACKQUOTE,ST_HEREDOC>"${" {
1573	yy_push_state(ST_LOOKING_FOR_VARNAME);
1574	RETURN_TOKEN(T_DOLLAR_OPEN_CURLY_BRACES);
1575}
1576
1577
1578<ST_IN_SCRIPTING>"}" {
1579	RESET_DOC_COMMENT();
1580	if (!zend_stack_is_empty(&SCNG(state_stack))) {
1581		yy_pop_state();
1582	}
1583	RETURN_TOKEN('}');
1584}
1585
1586
1587<ST_LOOKING_FOR_VARNAME>{LABEL}[[}] {
1588	yyless(yyleng - 1);
1589	zend_copy_value(zendlval, yytext, yyleng);
1590	yy_pop_state();
1591	yy_push_state(ST_IN_SCRIPTING);
1592	RETURN_TOKEN(T_STRING_VARNAME);
1593}
1594
1595
1596<ST_LOOKING_FOR_VARNAME>{ANY_CHAR} {
1597	yyless(0);
1598	yy_pop_state();
1599	yy_push_state(ST_IN_SCRIPTING);
1600	goto restart;
1601}
1602
1603<ST_IN_SCRIPTING>{BNUM} {
1604	char *bin = yytext + 2; /* Skip "0b" */
1605	int len = yyleng - 2;
1606	char *end;
1607
1608	/* Skip any leading 0s */
1609	while (*bin == '0') {
1610		++bin;
1611		--len;
1612	}
1613
1614	if (len < SIZEOF_ZEND_LONG * 8) {
1615		if (len == 0) {
1616			ZVAL_LONG(zendlval, 0);
1617		} else {
1618			errno = 0;
1619			ZVAL_LONG(zendlval, ZEND_STRTOL(bin, &end, 2));
1620			ZEND_ASSERT(!errno && end == yytext + yyleng);
1621		}
1622		RETURN_TOKEN(T_LNUMBER);
1623	} else {
1624		ZVAL_DOUBLE(zendlval, zend_bin_strtod(bin, (const char **)&end));
1625		/* errno isn't checked since we allow HUGE_VAL/INF overflow */
1626		ZEND_ASSERT(end == yytext + yyleng);
1627		RETURN_TOKEN(T_DNUMBER);
1628	}
1629}
1630
1631<ST_IN_SCRIPTING>{LNUM} {
1632	char *end;
1633	if (yyleng < MAX_LENGTH_OF_LONG - 1) { /* Won't overflow */
1634		errno = 0;
1635		ZVAL_LONG(zendlval, ZEND_STRTOL(yytext, &end, 0));
1636		/* This isn't an assert, we need to ensure 019 isn't valid octal
1637		 * Because the lexing itself doesn't do that for us
1638		 */
1639		if (end != yytext + yyleng) {
1640			zend_throw_exception(zend_ce_parse_error, "Invalid numeric literal", 0);
1641			ZVAL_UNDEF(zendlval);
1642			RETURN_TOKEN(T_LNUMBER);
1643		}
1644	} else {
1645		errno = 0;
1646		ZVAL_LONG(zendlval, ZEND_STRTOL(yytext, &end, 0));
1647		if (errno == ERANGE) { /* Overflow */
1648			errno = 0;
1649			if (yytext[0] == '0') { /* octal overflow */
1650				errno = 0;
1651				ZVAL_DOUBLE(zendlval, zend_oct_strtod(yytext, (const char **)&end));
1652			} else {
1653				ZVAL_DOUBLE(zendlval, zend_strtod(yytext, (const char **)&end));
1654			}
1655			/* Also not an assert for the same reason */
1656			if (end != yytext + yyleng) {
1657				zend_throw_exception(zend_ce_parse_error,
1658					"Invalid numeric literal", 0);
1659				ZVAL_UNDEF(zendlval);
1660				RETURN_TOKEN(T_DNUMBER);
1661			}
1662			ZEND_ASSERT(!errno);
1663			RETURN_TOKEN(T_DNUMBER);
1664		}
1665		/* Also not an assert for the same reason */
1666		if (end != yytext + yyleng) {
1667			zend_throw_exception(zend_ce_parse_error, "Invalid numeric literal", 0);
1668			ZVAL_UNDEF(zendlval);
1669			RETURN_TOKEN(T_DNUMBER);
1670		}
1671	}
1672	ZEND_ASSERT(!errno);
1673	RETURN_TOKEN(T_LNUMBER);
1674}
1675
1676<ST_IN_SCRIPTING>{HNUM} {
1677	char *hex = yytext + 2; /* Skip "0x" */
1678	int len = yyleng - 2;
1679	char *end;
1680
1681	/* Skip any leading 0s */
1682	while (*hex == '0') {
1683		hex++;
1684		len--;
1685	}
1686
1687	if (len < SIZEOF_ZEND_LONG * 2 || (len == SIZEOF_ZEND_LONG * 2 && *hex <= '7')) {
1688		if (len == 0) {
1689			ZVAL_LONG(zendlval, 0);
1690		} else {
1691			errno = 0;
1692			ZVAL_LONG(zendlval, ZEND_STRTOL(hex, &end, 16));
1693			ZEND_ASSERT(!errno && end == hex + len);
1694		}
1695		RETURN_TOKEN(T_LNUMBER);
1696	} else {
1697		ZVAL_DOUBLE(zendlval, zend_hex_strtod(hex, (const char **)&end));
1698		/* errno isn't checked since we allow HUGE_VAL/INF overflow */
1699		ZEND_ASSERT(end == hex + len);
1700		RETURN_TOKEN(T_DNUMBER);
1701	}
1702}
1703
1704<ST_VAR_OFFSET>[0]|([1-9][0-9]*) { /* Offset could be treated as a long */
1705	if (yyleng < MAX_LENGTH_OF_LONG - 1 || (yyleng == MAX_LENGTH_OF_LONG - 1 && strcmp(yytext, long_min_digits) < 0)) {
1706		char *end;
1707		errno = 0;
1708		ZVAL_LONG(zendlval, ZEND_STRTOL(yytext, &end, 10));
1709		if (errno == ERANGE) {
1710			goto string;
1711		}
1712		ZEND_ASSERT(end == yytext + yyleng);
1713	} else {
1714string:
1715		ZVAL_STRINGL(zendlval, yytext, yyleng);
1716	}
1717	RETURN_TOKEN(T_NUM_STRING);
1718}
1719
1720<ST_VAR_OFFSET>{LNUM}|{HNUM}|{BNUM} { /* Offset must be treated as a string */
1721	ZVAL_STRINGL(zendlval, yytext, yyleng);
1722	RETURN_TOKEN(T_NUM_STRING);
1723}
1724
1725<ST_IN_SCRIPTING>{DNUM}|{EXPONENT_DNUM} {
1726	const char *end;
1727
1728	ZVAL_DOUBLE(zendlval, zend_strtod(yytext, &end));
1729	/* errno isn't checked since we allow HUGE_VAL/INF overflow */
1730	ZEND_ASSERT(end == yytext + yyleng);
1731	RETURN_TOKEN(T_DNUMBER);
1732}
1733
1734<ST_IN_SCRIPTING>"__CLASS__" {
1735	RETURN_TOKEN(T_CLASS_C);
1736}
1737
1738<ST_IN_SCRIPTING>"__TRAIT__" {
1739	RETURN_TOKEN(T_TRAIT_C);
1740}
1741
1742<ST_IN_SCRIPTING>"__FUNCTION__" {
1743	RETURN_TOKEN(T_FUNC_C);
1744}
1745
1746<ST_IN_SCRIPTING>"__METHOD__" {
1747	RETURN_TOKEN(T_METHOD_C);
1748}
1749
1750<ST_IN_SCRIPTING>"__LINE__" {
1751	RETURN_TOKEN(T_LINE);
1752}
1753
1754<ST_IN_SCRIPTING>"__FILE__" {
1755	RETURN_TOKEN(T_FILE);
1756}
1757
1758<ST_IN_SCRIPTING>"__DIR__" {
1759	RETURN_TOKEN(T_DIR);
1760}
1761
1762<ST_IN_SCRIPTING>"__NAMESPACE__" {
1763	RETURN_TOKEN(T_NS_C);
1764}
1765
1766
1767<INITIAL>"<?=" {
1768	BEGIN(ST_IN_SCRIPTING);
1769	RETURN_TOKEN(T_OPEN_TAG_WITH_ECHO);
1770}
1771
1772
1773<INITIAL>"<?php"([ \t]|{NEWLINE}) {
1774	HANDLE_NEWLINE(yytext[yyleng-1]);
1775	BEGIN(ST_IN_SCRIPTING);
1776	RETURN_TOKEN(T_OPEN_TAG);
1777}
1778
1779
1780<INITIAL>"<?" {
1781	if (CG(short_tags)) {
1782		BEGIN(ST_IN_SCRIPTING);
1783		RETURN_TOKEN(T_OPEN_TAG);
1784	} else {
1785		goto inline_char_handler;
1786	}
1787}
1788
1789<INITIAL>{ANY_CHAR} {
1790	if (YYCURSOR > YYLIMIT) {
1791		RETURN_TOKEN(END);
1792	}
1793
1794inline_char_handler:
1795
1796	while (1) {
1797		YYCTYPE *ptr = memchr(YYCURSOR, '<', YYLIMIT - YYCURSOR);
1798
1799		YYCURSOR = ptr ? ptr + 1 : YYLIMIT;
1800
1801		if (YYCURSOR >= YYLIMIT) {
1802			break;
1803		}
1804
1805		if (*YYCURSOR == '?') {
1806			if (CG(short_tags) || !strncasecmp((char*)YYCURSOR + 1, "php", 3) || (*(YYCURSOR + 1) == '=')) { /* Assume [ \t\n\r] follows "php" */
1807
1808				YYCURSOR--;
1809				break;
1810			}
1811		}
1812	}
1813
1814	yyleng = YYCURSOR - SCNG(yy_text);
1815
1816	if (SCNG(output_filter)) {
1817		size_t readsize;
1818		char *s = NULL;
1819		size_t sz = 0;
1820		// TODO: avoid reallocation ???
1821		readsize = SCNG(output_filter)((unsigned char **)&s, &sz, (unsigned char *)yytext, (size_t)yyleng);
1822		ZVAL_STRINGL(zendlval, s, sz);
1823		efree(s);
1824		if (readsize < yyleng) {
1825			yyless(readsize);
1826		}
1827	} else {
1828	  ZVAL_STRINGL(zendlval, yytext, yyleng);
1829	}
1830	HANDLE_NEWLINES(yytext, yyleng);
1831	RETURN_TOKEN(T_INLINE_HTML);
1832}
1833
1834
1835/* Make sure a label character follows "->", otherwise there is no property
1836 * and "->" will be taken literally
1837 */
1838<ST_DOUBLE_QUOTES,ST_HEREDOC,ST_BACKQUOTE>"$"{LABEL}"->"[a-zA-Z_\x80-\xff] {
1839	yyless(yyleng - 3);
1840	yy_push_state(ST_LOOKING_FOR_PROPERTY);
1841	zend_copy_value(zendlval, (yytext+1), (yyleng-1));
1842	RETURN_TOKEN(T_VARIABLE);
1843}
1844
1845/* A [ always designates a variable offset, regardless of what follows
1846 */
1847<ST_DOUBLE_QUOTES,ST_HEREDOC,ST_BACKQUOTE>"$"{LABEL}"[" {
1848	yyless(yyleng - 1);
1849	yy_push_state(ST_VAR_OFFSET);
1850	zend_copy_value(zendlval, (yytext+1), (yyleng-1));
1851	RETURN_TOKEN(T_VARIABLE);
1852}
1853
1854<ST_IN_SCRIPTING,ST_DOUBLE_QUOTES,ST_HEREDOC,ST_BACKQUOTE,ST_VAR_OFFSET>"$"{LABEL} {
1855	zend_copy_value(zendlval, (yytext+1), (yyleng-1));
1856	RETURN_TOKEN(T_VARIABLE);
1857}
1858
1859<ST_VAR_OFFSET>"]" {
1860	yy_pop_state();
1861	RETURN_TOKEN(']');
1862}
1863
1864<ST_VAR_OFFSET>{TOKENS}|[{}"`] {
1865	/* Only '[' can be valid, but returning other tokens will allow a more explicit parse error */
1866	RETURN_TOKEN(yytext[0]);
1867}
1868
1869<ST_VAR_OFFSET>[ \n\r\t\\'#] {
1870	/* Invalid rule to return a more explicit parse error with proper line number */
1871	yyless(0);
1872	yy_pop_state();
1873	ZVAL_NULL(zendlval);
1874	RETURN_TOKEN(T_ENCAPSED_AND_WHITESPACE);
1875}
1876
1877<ST_IN_SCRIPTING,ST_VAR_OFFSET>{LABEL} {
1878	zend_copy_value(zendlval, yytext, yyleng);
1879	RETURN_TOKEN(T_STRING);
1880}
1881
1882
1883<ST_IN_SCRIPTING>"#"|"//" {
1884	while (YYCURSOR < YYLIMIT) {
1885		switch (*YYCURSOR++) {
1886			case '\r':
1887				if (*YYCURSOR == '\n') {
1888					YYCURSOR++;
1889				}
1890				/* fall through */
1891			case '\n':
1892				CG(zend_lineno)++;
1893				break;
1894			case '?':
1895				if (*YYCURSOR == '>') {
1896					YYCURSOR--;
1897					break;
1898				}
1899				/* fall through */
1900			default:
1901				continue;
1902		}
1903
1904		break;
1905	}
1906
1907	yyleng = YYCURSOR - SCNG(yy_text);
1908
1909	RETURN_TOKEN(T_COMMENT);
1910}
1911
1912<ST_IN_SCRIPTING>"/*"|"/**"{WHITESPACE} {
1913	int doc_com;
1914
1915	if (yyleng > 2) {
1916		doc_com = 1;
1917		RESET_DOC_COMMENT();
1918	} else {
1919		doc_com = 0;
1920	}
1921
1922	while (YYCURSOR < YYLIMIT) {
1923		if (*YYCURSOR++ == '*' && *YYCURSOR == '/') {
1924			break;
1925		}
1926	}
1927
1928	if (YYCURSOR < YYLIMIT) {
1929		YYCURSOR++;
1930	} else {
1931		zend_error(E_COMPILE_WARNING, "Unterminated comment starting line %d", CG(zend_lineno));
1932	}
1933
1934	yyleng = YYCURSOR - SCNG(yy_text);
1935	HANDLE_NEWLINES(yytext, yyleng);
1936
1937	if (doc_com) {
1938		CG(doc_comment) = zend_string_init(yytext, yyleng, 0);
1939		RETURN_TOKEN(T_DOC_COMMENT);
1940	}
1941
1942	RETURN_TOKEN(T_COMMENT);
1943}
1944
1945<ST_IN_SCRIPTING>"?>"{NEWLINE}? {
1946	BEGIN(INITIAL);
1947	RETURN_TOKEN(T_CLOSE_TAG);  /* implicit ';' at php-end tag */
1948}
1949
1950
1951<ST_IN_SCRIPTING>b?['] {
1952	register char *s, *t;
1953	char *end;
1954	int bprefix = (yytext[0] != '\'') ? 1 : 0;
1955
1956	while (1) {
1957		if (YYCURSOR < YYLIMIT) {
1958			if (*YYCURSOR == '\'') {
1959				YYCURSOR++;
1960				yyleng = YYCURSOR - SCNG(yy_text);
1961
1962				break;
1963			} else if (*YYCURSOR++ == '\\' && YYCURSOR < YYLIMIT) {
1964				YYCURSOR++;
1965			}
1966		} else {
1967			yyleng = YYLIMIT - SCNG(yy_text);
1968
1969			/* Unclosed single quotes; treat similar to double quotes, but without a separate token
1970			 * for ' (unrecognized by parser), instead of old flex fallback to "Unexpected character..."
1971			 * rule, which continued in ST_IN_SCRIPTING state after the quote */
1972			ZVAL_NULL(zendlval);
1973			RETURN_TOKEN(T_ENCAPSED_AND_WHITESPACE);
1974		}
1975	}
1976
1977	ZVAL_STRINGL(zendlval, yytext+bprefix+1, yyleng-bprefix-2);
1978
1979	/* convert escape sequences */
1980	s = t = Z_STRVAL_P(zendlval);
1981	end = s+Z_STRLEN_P(zendlval);
1982	while (s<end) {
1983		if (*s=='\\') {
1984			s++;
1985
1986			switch(*s) {
1987				case '\\':
1988				case '\'':
1989					*t++ = *s;
1990					Z_STRLEN_P(zendlval)--;
1991					break;
1992				default:
1993					*t++ = '\\';
1994					*t++ = *s;
1995					break;
1996			}
1997		} else {
1998			*t++ = *s;
1999		}
2000
2001		if (*s == '\n' || (*s == '\r' && (*(s+1) != '\n'))) {
2002			CG(zend_lineno)++;
2003		}
2004		s++;
2005	}
2006	*t = 0;
2007
2008	if (SCNG(output_filter)) {
2009		size_t sz = 0;
2010		char *str = NULL;
2011		s = Z_STRVAL_P(zendlval);
2012		// TODO: avoid reallocation ???
2013		SCNG(output_filter)((unsigned char **)&str, &sz, (unsigned char *)s, (size_t)Z_STRLEN_P(zendlval));
2014		ZVAL_STRINGL(zendlval, str, sz);
2015	}
2016	RETURN_TOKEN(T_CONSTANT_ENCAPSED_STRING);
2017}
2018
2019
2020<ST_IN_SCRIPTING>b?["] {
2021	int bprefix = (yytext[0] != '"') ? 1 : 0;
2022
2023	while (YYCURSOR < YYLIMIT) {
2024		switch (*YYCURSOR++) {
2025			case '"':
2026				yyleng = YYCURSOR - SCNG(yy_text);
2027				zend_scan_escape_string(zendlval, yytext+bprefix+1, yyleng-bprefix-2, '"');
2028				RETURN_TOKEN(T_CONSTANT_ENCAPSED_STRING);
2029			case '$':
2030				if (IS_LABEL_START(*YYCURSOR) || *YYCURSOR == '{') {
2031					break;
2032				}
2033				continue;
2034			case '{':
2035				if (*YYCURSOR == '$') {
2036					break;
2037				}
2038				continue;
2039			case '\\':
2040				if (YYCURSOR < YYLIMIT) {
2041					YYCURSOR++;
2042				}
2043				/* fall through */
2044			default:
2045				continue;
2046		}
2047
2048		YYCURSOR--;
2049		break;
2050	}
2051
2052	/* Remember how much was scanned to save rescanning */
2053	SET_DOUBLE_QUOTES_SCANNED_LENGTH(YYCURSOR - SCNG(yy_text) - yyleng);
2054
2055	YYCURSOR = SCNG(yy_text) + yyleng;
2056
2057	BEGIN(ST_DOUBLE_QUOTES);
2058	RETURN_TOKEN('"');
2059}
2060
2061
2062<ST_IN_SCRIPTING>b?"<<<"{TABS_AND_SPACES}({LABEL}|([']{LABEL}['])|(["]{LABEL}["])){NEWLINE} {
2063	char *s;
2064	int bprefix = (yytext[0] != '<') ? 1 : 0;
2065	zend_heredoc_label *heredoc_label = emalloc(sizeof(zend_heredoc_label));
2066
2067	CG(zend_lineno)++;
2068	heredoc_label->length = yyleng-bprefix-3-1-(yytext[yyleng-2]=='\r'?1:0);
2069	s = yytext+bprefix+3;
2070	while ((*s == ' ') || (*s == '\t')) {
2071		s++;
2072		heredoc_label->length--;
2073	}
2074
2075	if (*s == '\'') {
2076		s++;
2077		heredoc_label->length -= 2;
2078
2079		BEGIN(ST_NOWDOC);
2080	} else {
2081		if (*s == '"') {
2082			s++;
2083			heredoc_label->length -= 2;
2084		}
2085
2086		BEGIN(ST_HEREDOC);
2087	}
2088
2089	heredoc_label->label = estrndup(s, heredoc_label->length);
2090
2091	/* Check for ending label on the next line */
2092	if (heredoc_label->length < YYLIMIT - YYCURSOR && !memcmp(YYCURSOR, s, heredoc_label->length)) {
2093		YYCTYPE *end = YYCURSOR + heredoc_label->length;
2094
2095		if (*end == ';') {
2096			end++;
2097		}
2098
2099		if (*end == '\n' || *end == '\r') {
2100			BEGIN(ST_END_HEREDOC);
2101		}
2102	}
2103
2104	zend_ptr_stack_push(&SCNG(heredoc_label_stack), (void *) heredoc_label);
2105
2106	RETURN_TOKEN(T_START_HEREDOC);
2107}
2108
2109
2110<ST_IN_SCRIPTING>[`] {
2111	BEGIN(ST_BACKQUOTE);
2112	RETURN_TOKEN('`');
2113}
2114
2115
2116<ST_END_HEREDOC>{ANY_CHAR} {
2117	zend_heredoc_label *heredoc_label = zend_ptr_stack_pop(&SCNG(heredoc_label_stack));
2118
2119	YYCURSOR += heredoc_label->length - 1;
2120	yyleng = heredoc_label->length;
2121
2122	heredoc_label_dtor(heredoc_label);
2123	efree(heredoc_label);
2124
2125	BEGIN(ST_IN_SCRIPTING);
2126	RETURN_TOKEN(T_END_HEREDOC);
2127}
2128
2129
2130<ST_DOUBLE_QUOTES,ST_BACKQUOTE,ST_HEREDOC>"{$" {
2131	Z_LVAL_P(zendlval) = (zend_long) '{';
2132	yy_push_state(ST_IN_SCRIPTING);
2133	yyless(1);
2134	RETURN_TOKEN(T_CURLY_OPEN);
2135}
2136
2137
2138<ST_DOUBLE_QUOTES>["] {
2139	BEGIN(ST_IN_SCRIPTING);
2140	RETURN_TOKEN('"');
2141}
2142
2143<ST_BACKQUOTE>[`] {
2144	BEGIN(ST_IN_SCRIPTING);
2145	RETURN_TOKEN('`');
2146}
2147
2148
2149<ST_DOUBLE_QUOTES>{ANY_CHAR} {
2150	if (GET_DOUBLE_QUOTES_SCANNED_LENGTH()) {
2151		YYCURSOR += GET_DOUBLE_QUOTES_SCANNED_LENGTH() - 1;
2152		SET_DOUBLE_QUOTES_SCANNED_LENGTH(0);
2153
2154		goto double_quotes_scan_done;
2155	}
2156
2157	if (YYCURSOR > YYLIMIT) {
2158		RETURN_TOKEN(END);
2159	}
2160	if (yytext[0] == '\\' && YYCURSOR < YYLIMIT) {
2161		YYCURSOR++;
2162	}
2163
2164	while (YYCURSOR < YYLIMIT) {
2165		switch (*YYCURSOR++) {
2166			case '"':
2167				break;
2168			case '$':
2169				if (IS_LABEL_START(*YYCURSOR) || *YYCURSOR == '{') {
2170					break;
2171				}
2172				continue;
2173			case '{':
2174				if (*YYCURSOR == '$') {
2175					break;
2176				}
2177				continue;
2178			case '\\':
2179				if (YYCURSOR < YYLIMIT) {
2180					YYCURSOR++;
2181				}
2182				/* fall through */
2183			default:
2184				continue;
2185		}
2186
2187		YYCURSOR--;
2188		break;
2189	}
2190
2191double_quotes_scan_done:
2192	yyleng = YYCURSOR - SCNG(yy_text);
2193
2194	zend_scan_escape_string(zendlval, yytext, yyleng, '"');
2195	RETURN_TOKEN(T_ENCAPSED_AND_WHITESPACE);
2196}
2197
2198
2199<ST_BACKQUOTE>{ANY_CHAR} {
2200	if (YYCURSOR > YYLIMIT) {
2201		RETURN_TOKEN(END);
2202	}
2203	if (yytext[0] == '\\' && YYCURSOR < YYLIMIT) {
2204		YYCURSOR++;
2205	}
2206
2207	while (YYCURSOR < YYLIMIT) {
2208		switch (*YYCURSOR++) {
2209			case '`':
2210				break;
2211			case '$':
2212				if (IS_LABEL_START(*YYCURSOR) || *YYCURSOR == '{') {
2213					break;
2214				}
2215				continue;
2216			case '{':
2217				if (*YYCURSOR == '$') {
2218					break;
2219				}
2220				continue;
2221			case '\\':
2222				if (YYCURSOR < YYLIMIT) {
2223					YYCURSOR++;
2224				}
2225				/* fall through */
2226			default:
2227				continue;
2228		}
2229
2230		YYCURSOR--;
2231		break;
2232	}
2233
2234	yyleng = YYCURSOR - SCNG(yy_text);
2235
2236	zend_scan_escape_string(zendlval, yytext, yyleng, '`');
2237	RETURN_TOKEN(T_ENCAPSED_AND_WHITESPACE);
2238}
2239
2240
2241<ST_HEREDOC>{ANY_CHAR} {
2242	int newline = 0;
2243
2244	zend_heredoc_label *heredoc_label = zend_ptr_stack_top(&SCNG(heredoc_label_stack));
2245
2246	if (YYCURSOR > YYLIMIT) {
2247		RETURN_TOKEN(END);
2248	}
2249
2250	YYCURSOR--;
2251
2252	while (YYCURSOR < YYLIMIT) {
2253		switch (*YYCURSOR++) {
2254			case '\r':
2255				if (*YYCURSOR == '\n') {
2256					YYCURSOR++;
2257				}
2258				/* fall through */
2259			case '\n':
2260				/* Check for ending label on the next line */
2261				if (IS_LABEL_START(*YYCURSOR) && heredoc_label->length < YYLIMIT - YYCURSOR && !memcmp(YYCURSOR, heredoc_label->label, heredoc_label->length)) {
2262					YYCTYPE *end = YYCURSOR + heredoc_label->length;
2263
2264					if (*end == ';') {
2265						end++;
2266					}
2267
2268					if (*end == '\n' || *end == '\r') {
2269						/* newline before label will be subtracted from returned text, but
2270						 * yyleng/yytext will include it, for zend_highlight/strip, tokenizer, etc. */
2271						if (YYCURSOR[-2] == '\r' && YYCURSOR[-1] == '\n') {
2272							newline = 2; /* Windows newline */
2273						} else {
2274							newline = 1;
2275						}
2276
2277						CG(increment_lineno) = 1; /* For newline before label */
2278						BEGIN(ST_END_HEREDOC);
2279
2280						goto heredoc_scan_done;
2281					}
2282				}
2283				continue;
2284			case '$':
2285				if (IS_LABEL_START(*YYCURSOR) || *YYCURSOR == '{') {
2286					break;
2287				}
2288				continue;
2289			case '{':
2290				if (*YYCURSOR == '$') {
2291					break;
2292				}
2293				continue;
2294			case '\\':
2295				if (YYCURSOR < YYLIMIT && *YYCURSOR != '\n' && *YYCURSOR != '\r') {
2296					YYCURSOR++;
2297				}
2298				/* fall through */
2299			default:
2300				continue;
2301		}
2302
2303		YYCURSOR--;
2304		break;
2305	}
2306
2307heredoc_scan_done:
2308	yyleng = YYCURSOR - SCNG(yy_text);
2309
2310	zend_scan_escape_string(zendlval, yytext, yyleng - newline, 0);
2311	RETURN_TOKEN(T_ENCAPSED_AND_WHITESPACE);
2312}
2313
2314
2315<ST_NOWDOC>{ANY_CHAR} {
2316	int newline = 0;
2317
2318	zend_heredoc_label *heredoc_label = zend_ptr_stack_top(&SCNG(heredoc_label_stack));
2319
2320	if (YYCURSOR > YYLIMIT) {
2321		RETURN_TOKEN(END);
2322	}
2323
2324	YYCURSOR--;
2325
2326	while (YYCURSOR < YYLIMIT) {
2327		switch (*YYCURSOR++) {
2328			case '\r':
2329				if (*YYCURSOR == '\n') {
2330					YYCURSOR++;
2331				}
2332				/* fall through */
2333			case '\n':
2334				/* Check for ending label on the next line */
2335				if (IS_LABEL_START(*YYCURSOR) && heredoc_label->length < YYLIMIT - YYCURSOR && !memcmp(YYCURSOR, heredoc_label->label, heredoc_label->length)) {
2336					YYCTYPE *end = YYCURSOR + heredoc_label->length;
2337
2338					if (*end == ';') {
2339						end++;
2340					}
2341
2342					if (*end == '\n' || *end == '\r') {
2343						/* newline before label will be subtracted from returned text, but
2344						 * yyleng/yytext will include it, for zend_highlight/strip, tokenizer, etc. */
2345						if (YYCURSOR[-2] == '\r' && YYCURSOR[-1] == '\n') {
2346							newline = 2; /* Windows newline */
2347						} else {
2348							newline = 1;
2349						}
2350
2351						CG(increment_lineno) = 1; /* For newline before label */
2352						BEGIN(ST_END_HEREDOC);
2353
2354						goto nowdoc_scan_done;
2355					}
2356				}
2357				/* fall through */
2358			default:
2359				continue;
2360		}
2361	}
2362
2363nowdoc_scan_done:
2364	yyleng = YYCURSOR - SCNG(yy_text);
2365
2366	zend_copy_value(zendlval, yytext, yyleng - newline);
2367	HANDLE_NEWLINES(yytext, yyleng - newline);
2368	RETURN_TOKEN(T_ENCAPSED_AND_WHITESPACE);
2369}
2370
2371
2372<ST_IN_SCRIPTING,ST_VAR_OFFSET>{ANY_CHAR} {
2373	if (YYCURSOR > YYLIMIT) {
2374		RETURN_TOKEN(END);
2375	}
2376
2377	zend_error(E_COMPILE_WARNING,"Unexpected character in input:  '%c' (ASCII=%d) state=%d", yytext[0], yytext[0], YYSTATE);
2378	goto restart;
2379}
2380
2381*/
2382}
2383