1/*
2   +----------------------------------------------------------------------+
3   | Zend Engine                                                          |
4   +----------------------------------------------------------------------+
5   | Copyright (c) 1998-2016 Zend Technologies Ltd. (http://www.zend.com) |
6   +----------------------------------------------------------------------+
7   | This source file is subject to version 2.00 of the Zend license,     |
8   | that is bundled with this package in the file LICENSE, and is        |
9   | available through the world-wide-web at the following url:           |
10   | http://www.zend.com/license/2_00.txt.                                |
11   | If you did not receive a copy of the Zend license and are unable to  |
12   | obtain it through the world-wide-web, please send a note to          |
13   | license@zend.com so we can mail you a copy immediately.              |
14   +----------------------------------------------------------------------+
15   | Authors: Marcus Boerger <helly@php.net>                              |
16   |          Nuno Lopes <nlopess@php.net>                                |
17   |          Scott MacVicar <scottmac@php.net>                           |
18   | Flex version authors:                                                |
19   |          Andi Gutmans <andi@zend.com>                                |
20   |          Zeev Suraski <zeev@zend.com>                                |
21   +----------------------------------------------------------------------+
22*/
23
24/* $Id$ */
25
26#if 0
27# define YYDEBUG(s, c) printf("state: %d char: %c\n", s, c)
28#else
29# define YYDEBUG(s, c)
30#endif
31
32#include "zend_language_scanner_defs.h"
33
34#include <errno.h>
35#include "zend.h"
36#ifdef ZEND_WIN32
37# include <Winuser.h>
38#endif
39#include "zend_alloc.h"
40#include <zend_language_parser.h>
41#include "zend_compile.h"
42#include "zend_language_scanner.h"
43#include "zend_highlight.h"
44#include "zend_constants.h"
45#include "zend_variables.h"
46#include "zend_operators.h"
47#include "zend_API.h"
48#include "zend_strtod.h"
49#include "zend_exceptions.h"
50#include "zend_virtual_cwd.h"
51#include "tsrm_config_common.h"
52
53#define YYCTYPE   unsigned char
54#define YYFILL(n) { if ((YYCURSOR + n) >= (YYLIMIT + ZEND_MMAP_AHEAD)) { return 0; } }
55#define YYCURSOR  SCNG(yy_cursor)
56#define YYLIMIT   SCNG(yy_limit)
57#define YYMARKER  SCNG(yy_marker)
58
59#define YYGETCONDITION()  SCNG(yy_state)
60#define YYSETCONDITION(s) SCNG(yy_state) = s
61
62#define STATE(name)  yyc##name
63
64/* emulate flex constructs */
65#define BEGIN(state) YYSETCONDITION(STATE(state))
66#define YYSTATE      YYGETCONDITION()
67#define yytext       ((char*)SCNG(yy_text))
68#define yyleng       SCNG(yy_leng)
69#define yyless(x)    do { YYCURSOR = (unsigned char*)yytext + x; \
70                          yyleng   = (unsigned int)x; } while(0)
71#define yymore()     goto yymore_restart
72
73/* perform sanity check. If this message is triggered you should
74   increase the ZEND_MMAP_AHEAD value in the zend_streams.h file */
75/*!max:re2c */
76#if ZEND_MMAP_AHEAD < YYMAXFILL
77# error ZEND_MMAP_AHEAD should be greater than or equal to YYMAXFILL
78#endif
79
80#ifdef HAVE_STDARG_H
81# include <stdarg.h>
82#endif
83
84#ifdef HAVE_UNISTD_H
85# include <unistd.h>
86#endif
87
88/* Globals Macros */
89#define SCNG	LANG_SCNG
90#ifdef ZTS
91ZEND_API ts_rsrc_id language_scanner_globals_id;
92#else
93ZEND_API zend_php_scanner_globals language_scanner_globals;
94#endif
95
96#define HANDLE_NEWLINES(s, l)													\
97do {																			\
98	char *p = (s), *boundary = p+(l);											\
99																				\
100	while (p<boundary) {														\
101		if (*p == '\n' || (*p == '\r' && (*(p+1) != '\n'))) {					\
102			CG(zend_lineno)++;													\
103		}																		\
104		p++;																	\
105	}																			\
106} while (0)
107
108#define HANDLE_NEWLINE(c) \
109{ \
110	if (c == '\n' || c == '\r') { \
111		CG(zend_lineno)++; \
112	} \
113}
114
115/* To save initial string length after scanning to first variable */
116#define SET_DOUBLE_QUOTES_SCANNED_LENGTH(len) SCNG(scanned_string_len) = (len)
117#define GET_DOUBLE_QUOTES_SCANNED_LENGTH()    SCNG(scanned_string_len)
118
119#define IS_LABEL_START(c) (((c) >= 'a' && (c) <= 'z') || ((c) >= 'A' && (c) <= 'Z') || (c) == '_' || (c) >= 0x80)
120
121#define ZEND_IS_OCT(c)  ((c)>='0' && (c)<='7')
122#define ZEND_IS_HEX(c)  (((c)>='0' && (c)<='9') || ((c)>='a' && (c)<='f') || ((c)>='A' && (c)<='F'))
123
124BEGIN_EXTERN_C()
125
126static size_t encoding_filter_script_to_internal(unsigned char **to, size_t *to_length, const unsigned char *from, size_t from_length)
127{
128	const zend_encoding *internal_encoding = zend_multibyte_get_internal_encoding();
129	ZEND_ASSERT(internal_encoding);
130	return zend_multibyte_encoding_converter(to, to_length, from, from_length, internal_encoding, LANG_SCNG(script_encoding));
131}
132
133static size_t encoding_filter_script_to_intermediate(unsigned char **to, size_t *to_length, const unsigned char *from, size_t from_length)
134{
135	return zend_multibyte_encoding_converter(to, to_length, from, from_length, zend_multibyte_encoding_utf8, LANG_SCNG(script_encoding));
136}
137
138static size_t encoding_filter_intermediate_to_script(unsigned char **to, size_t *to_length, const unsigned char *from, size_t from_length)
139{
140	return zend_multibyte_encoding_converter(to, to_length, from, from_length,
141LANG_SCNG(script_encoding), zend_multibyte_encoding_utf8);
142}
143
144static size_t encoding_filter_intermediate_to_internal(unsigned char **to, size_t *to_length, const unsigned char *from, size_t from_length)
145{
146	const zend_encoding *internal_encoding = zend_multibyte_get_internal_encoding();
147	ZEND_ASSERT(internal_encoding);
148	return zend_multibyte_encoding_converter(to, to_length, from, from_length,
149internal_encoding, zend_multibyte_encoding_utf8);
150}
151
152
153static void _yy_push_state(int new_state)
154{
155	zend_stack_push(&SCNG(state_stack), (void *) &YYGETCONDITION());
156	YYSETCONDITION(new_state);
157}
158
159#define yy_push_state(state_and_tsrm) _yy_push_state(yyc##state_and_tsrm)
160
161static void yy_pop_state(void)
162{
163	int *stack_state = zend_stack_top(&SCNG(state_stack));
164	YYSETCONDITION(*stack_state);
165	zend_stack_del_top(&SCNG(state_stack));
166}
167
168static void yy_scan_buffer(char *str, unsigned int len)
169{
170	YYCURSOR       = (YYCTYPE*)str;
171	YYLIMIT        = YYCURSOR + len;
172	if (!SCNG(yy_start)) {
173		SCNG(yy_start) = YYCURSOR;
174	}
175}
176
177void startup_scanner(void)
178{
179	CG(parse_error) = 0;
180	CG(doc_comment) = NULL;
181	CG(extra_fn_flags) = 0;
182	zend_stack_init(&SCNG(state_stack), sizeof(int));
183	zend_ptr_stack_init(&SCNG(heredoc_label_stack));
184}
185
186static void heredoc_label_dtor(zend_heredoc_label *heredoc_label) {
187    efree(heredoc_label->label);
188}
189
190void shutdown_scanner(void)
191{
192	CG(parse_error) = 0;
193	RESET_DOC_COMMENT();
194	zend_stack_destroy(&SCNG(state_stack));
195	zend_ptr_stack_clean(&SCNG(heredoc_label_stack), (void (*)(void *)) &heredoc_label_dtor, 1);
196	zend_ptr_stack_destroy(&SCNG(heredoc_label_stack));
197	SCNG(on_event) = NULL;
198}
199
200ZEND_API void zend_save_lexical_state(zend_lex_state *lex_state)
201{
202	lex_state->yy_leng   = SCNG(yy_leng);
203	lex_state->yy_start  = SCNG(yy_start);
204	lex_state->yy_text   = SCNG(yy_text);
205	lex_state->yy_cursor = SCNG(yy_cursor);
206	lex_state->yy_marker = SCNG(yy_marker);
207	lex_state->yy_limit  = SCNG(yy_limit);
208
209	lex_state->state_stack = SCNG(state_stack);
210	zend_stack_init(&SCNG(state_stack), sizeof(int));
211
212	lex_state->heredoc_label_stack = SCNG(heredoc_label_stack);
213	zend_ptr_stack_init(&SCNG(heredoc_label_stack));
214
215	lex_state->in = SCNG(yy_in);
216	lex_state->yy_state = YYSTATE;
217	lex_state->filename = zend_get_compiled_filename();
218	lex_state->lineno = CG(zend_lineno);
219
220	lex_state->script_org = SCNG(script_org);
221	lex_state->script_org_size = SCNG(script_org_size);
222	lex_state->script_filtered = SCNG(script_filtered);
223	lex_state->script_filtered_size = SCNG(script_filtered_size);
224	lex_state->input_filter = SCNG(input_filter);
225	lex_state->output_filter = SCNG(output_filter);
226	lex_state->script_encoding = SCNG(script_encoding);
227
228	lex_state->on_event = SCNG(on_event);
229
230	lex_state->ast = CG(ast);
231	lex_state->ast_arena = CG(ast_arena);
232}
233
234ZEND_API void zend_restore_lexical_state(zend_lex_state *lex_state)
235{
236	SCNG(yy_leng)   = lex_state->yy_leng;
237	SCNG(yy_start)  = lex_state->yy_start;
238	SCNG(yy_text)   = lex_state->yy_text;
239	SCNG(yy_cursor) = lex_state->yy_cursor;
240	SCNG(yy_marker) = lex_state->yy_marker;
241	SCNG(yy_limit)  = lex_state->yy_limit;
242
243	zend_stack_destroy(&SCNG(state_stack));
244	SCNG(state_stack) = lex_state->state_stack;
245
246	zend_ptr_stack_clean(&SCNG(heredoc_label_stack), (void (*)(void *)) &heredoc_label_dtor, 1);
247	zend_ptr_stack_destroy(&SCNG(heredoc_label_stack));
248	SCNG(heredoc_label_stack) = lex_state->heredoc_label_stack;
249
250	SCNG(yy_in) = lex_state->in;
251	YYSETCONDITION(lex_state->yy_state);
252	CG(zend_lineno) = lex_state->lineno;
253	zend_restore_compiled_filename(lex_state->filename);
254
255	if (SCNG(script_filtered)) {
256		efree(SCNG(script_filtered));
257		SCNG(script_filtered) = NULL;
258	}
259	SCNG(script_org) = lex_state->script_org;
260	SCNG(script_org_size) = lex_state->script_org_size;
261	SCNG(script_filtered) = lex_state->script_filtered;
262	SCNG(script_filtered_size) = lex_state->script_filtered_size;
263	SCNG(input_filter) = lex_state->input_filter;
264	SCNG(output_filter) = lex_state->output_filter;
265	SCNG(script_encoding) = lex_state->script_encoding;
266
267	SCNG(on_event) = lex_state->on_event;
268
269	CG(ast) = lex_state->ast;
270	CG(ast_arena) = lex_state->ast_arena;
271
272	RESET_DOC_COMMENT();
273}
274
275ZEND_API void zend_destroy_file_handle(zend_file_handle *file_handle)
276{
277	zend_llist_del_element(&CG(open_files), file_handle, (int (*)(void *, void *)) zend_compare_file_handles);
278	/* zend_file_handle_dtor() operates on the copy, so we have to NULLify the original here */
279	file_handle->opened_path = NULL;
280	if (file_handle->free_filename) {
281		file_handle->filename = NULL;
282	}
283}
284
285ZEND_API void zend_lex_tstring(zval *zv)
286{
287	if (SCNG(on_event)) SCNG(on_event)(ON_FEEDBACK, T_STRING, 0);
288
289	ZVAL_STRINGL(zv, (char*)SCNG(yy_text), SCNG(yy_leng));
290}
291
292#define BOM_UTF32_BE	"\x00\x00\xfe\xff"
293#define	BOM_UTF32_LE	"\xff\xfe\x00\x00"
294#define	BOM_UTF16_BE	"\xfe\xff"
295#define	BOM_UTF16_LE	"\xff\xfe"
296#define	BOM_UTF8		"\xef\xbb\xbf"
297
298static const zend_encoding *zend_multibyte_detect_utf_encoding(const unsigned char *script, size_t script_size)
299{
300	const unsigned char *p;
301	int wchar_size = 2;
302	int le = 0;
303
304	/* utf-16 or utf-32? */
305	p = script;
306	assert(p >= script);
307	while ((size_t)(p-script) < script_size) {
308		p = memchr(p, 0, script_size-(p-script)-2);
309		if (!p) {
310			break;
311		}
312		if (*(p+1) == '\0' && *(p+2) == '\0') {
313			wchar_size = 4;
314			break;
315		}
316
317		/* searching for UTF-32 specific byte orders, so this will do */
318		p += 4;
319	}
320
321	/* BE or LE? */
322	p = script;
323	assert(p >= script);
324	while ((size_t)(p-script) < script_size) {
325		if (*p == '\0' && *(p+wchar_size-1) != '\0') {
326			/* BE */
327			le = 0;
328			break;
329		} else if (*p != '\0' && *(p+wchar_size-1) == '\0') {
330			/* LE* */
331			le = 1;
332			break;
333		}
334		p += wchar_size;
335	}
336
337	if (wchar_size == 2) {
338		return le ? zend_multibyte_encoding_utf16le : zend_multibyte_encoding_utf16be;
339	} else {
340		return le ? zend_multibyte_encoding_utf32le : zend_multibyte_encoding_utf32be;
341	}
342
343	return NULL;
344}
345
346static const zend_encoding* zend_multibyte_detect_unicode(void)
347{
348	const zend_encoding *script_encoding = NULL;
349	int bom_size;
350	unsigned char *pos1, *pos2;
351
352	if (LANG_SCNG(script_org_size) < sizeof(BOM_UTF32_LE)-1) {
353		return NULL;
354	}
355
356	/* check out BOM */
357	if (!memcmp(LANG_SCNG(script_org), BOM_UTF32_BE, sizeof(BOM_UTF32_BE)-1)) {
358		script_encoding = zend_multibyte_encoding_utf32be;
359		bom_size = sizeof(BOM_UTF32_BE)-1;
360	} else if (!memcmp(LANG_SCNG(script_org), BOM_UTF32_LE, sizeof(BOM_UTF32_LE)-1)) {
361		script_encoding = zend_multibyte_encoding_utf32le;
362		bom_size = sizeof(BOM_UTF32_LE)-1;
363	} else if (!memcmp(LANG_SCNG(script_org), BOM_UTF16_BE, sizeof(BOM_UTF16_BE)-1)) {
364		script_encoding = zend_multibyte_encoding_utf16be;
365		bom_size = sizeof(BOM_UTF16_BE)-1;
366	} else if (!memcmp(LANG_SCNG(script_org), BOM_UTF16_LE, sizeof(BOM_UTF16_LE)-1)) {
367		script_encoding = zend_multibyte_encoding_utf16le;
368		bom_size = sizeof(BOM_UTF16_LE)-1;
369	} else if (!memcmp(LANG_SCNG(script_org), BOM_UTF8, sizeof(BOM_UTF8)-1)) {
370		script_encoding = zend_multibyte_encoding_utf8;
371		bom_size = sizeof(BOM_UTF8)-1;
372	}
373
374	if (script_encoding) {
375		/* remove BOM */
376		LANG_SCNG(script_org) += bom_size;
377		LANG_SCNG(script_org_size) -= bom_size;
378
379		return script_encoding;
380	}
381
382	/* script contains NULL bytes -> auto-detection */
383	if ((pos1 = memchr(LANG_SCNG(script_org), 0, LANG_SCNG(script_org_size)))) {
384		/* check if the NULL byte is after the __HALT_COMPILER(); */
385		pos2 = LANG_SCNG(script_org);
386
387		while (pos1 - pos2 >= sizeof("__HALT_COMPILER();")-1) {
388			pos2 = memchr(pos2, '_', pos1 - pos2);
389			if (!pos2) break;
390			pos2++;
391			if (strncasecmp((char*)pos2, "_HALT_COMPILER", sizeof("_HALT_COMPILER")-1) == 0) {
392				pos2 += sizeof("_HALT_COMPILER")-1;
393				while (*pos2 == ' '  ||
394					   *pos2 == '\t' ||
395					   *pos2 == '\r' ||
396					   *pos2 == '\n') {
397					pos2++;
398				}
399				if (*pos2 == '(') {
400					pos2++;
401					while (*pos2 == ' '  ||
402						   *pos2 == '\t' ||
403						   *pos2 == '\r' ||
404						   *pos2 == '\n') {
405						pos2++;
406					}
407					if (*pos2 == ')') {
408						pos2++;
409						while (*pos2 == ' '  ||
410							   *pos2 == '\t' ||
411							   *pos2 == '\r' ||
412							   *pos2 == '\n') {
413							pos2++;
414						}
415						if (*pos2 == ';') {
416							return NULL;
417						}
418					}
419				}
420			}
421		}
422		/* make best effort if BOM is missing */
423		return zend_multibyte_detect_utf_encoding(LANG_SCNG(script_org), LANG_SCNG(script_org_size));
424	}
425
426	return NULL;
427}
428
429static const zend_encoding* zend_multibyte_find_script_encoding(void)
430{
431	const zend_encoding *script_encoding;
432
433	if (CG(detect_unicode)) {
434		/* check out bom(byte order mark) and see if containing wchars */
435		script_encoding = zend_multibyte_detect_unicode();
436		if (script_encoding != NULL) {
437			/* bom or wchar detection is prior to 'script_encoding' option */
438			return script_encoding;
439		}
440	}
441
442	/* if no script_encoding specified, just leave alone */
443	if (!CG(script_encoding_list) || !CG(script_encoding_list_size)) {
444		return NULL;
445	}
446
447	/* if multiple encodings specified, detect automagically */
448	if (CG(script_encoding_list_size) > 1) {
449		return zend_multibyte_encoding_detector(LANG_SCNG(script_org), LANG_SCNG(script_org_size), CG(script_encoding_list), CG(script_encoding_list_size));
450	}
451
452	return CG(script_encoding_list)[0];
453}
454
455ZEND_API int zend_multibyte_set_filter(const zend_encoding *onetime_encoding)
456{
457	const zend_encoding *internal_encoding = zend_multibyte_get_internal_encoding();
458	const zend_encoding *script_encoding = onetime_encoding ? onetime_encoding: zend_multibyte_find_script_encoding();
459
460	if (!script_encoding) {
461		return FAILURE;
462	}
463
464	/* judge input/output filter */
465	LANG_SCNG(script_encoding) = script_encoding;
466	LANG_SCNG(input_filter) = NULL;
467	LANG_SCNG(output_filter) = NULL;
468
469	if (!internal_encoding || LANG_SCNG(script_encoding) == internal_encoding) {
470		if (!zend_multibyte_check_lexer_compatibility(LANG_SCNG(script_encoding))) {
471			/* and if not, work around w/ script_encoding -> utf-8 -> script_encoding conversion */
472			LANG_SCNG(input_filter) = encoding_filter_script_to_intermediate;
473			LANG_SCNG(output_filter) = encoding_filter_intermediate_to_script;
474		} else {
475			LANG_SCNG(input_filter) = NULL;
476			LANG_SCNG(output_filter) = NULL;
477		}
478		return SUCCESS;
479	}
480
481	if (zend_multibyte_check_lexer_compatibility(internal_encoding)) {
482		LANG_SCNG(input_filter) = encoding_filter_script_to_internal;
483		LANG_SCNG(output_filter) = NULL;
484	} else if (zend_multibyte_check_lexer_compatibility(LANG_SCNG(script_encoding))) {
485		LANG_SCNG(input_filter) = NULL;
486		LANG_SCNG(output_filter) = encoding_filter_script_to_internal;
487	} else {
488		/* both script and internal encodings are incompatible w/ flex */
489		LANG_SCNG(input_filter) = encoding_filter_script_to_intermediate;
490		LANG_SCNG(output_filter) = encoding_filter_intermediate_to_internal;
491	}
492
493	return 0;
494}
495
496ZEND_API int open_file_for_scanning(zend_file_handle *file_handle)
497{
498	char *buf;
499	size_t size, offset = 0;
500	zend_string *compiled_filename;
501
502	/* The shebang line was read, get the current position to obtain the buffer start */
503	if (CG(start_lineno) == 2 && file_handle->type == ZEND_HANDLE_FP && file_handle->handle.fp) {
504		if ((offset = ftell(file_handle->handle.fp)) == -1) {
505			offset = 0;
506		}
507	}
508
509	if (zend_stream_fixup(file_handle, &buf, &size) == FAILURE) {
510		return FAILURE;
511	}
512
513	zend_llist_add_element(&CG(open_files), file_handle);
514	if (file_handle->handle.stream.handle >= (void*)file_handle && file_handle->handle.stream.handle <= (void*)(file_handle+1)) {
515		zend_file_handle *fh = (zend_file_handle*)zend_llist_get_last(&CG(open_files));
516		size_t diff = (char*)file_handle->handle.stream.handle - (char*)file_handle;
517		fh->handle.stream.handle = (void*)(((char*)fh) + diff);
518		file_handle->handle.stream.handle = fh->handle.stream.handle;
519	}
520
521	/* Reset the scanner for scanning the new file */
522	SCNG(yy_in) = file_handle;
523	SCNG(yy_start) = NULL;
524
525	if (size != -1) {
526		if (CG(multibyte)) {
527			SCNG(script_org) = (unsigned char*)buf;
528			SCNG(script_org_size) = size;
529			SCNG(script_filtered) = NULL;
530
531			zend_multibyte_set_filter(NULL);
532
533			if (SCNG(input_filter)) {
534				if ((size_t)-1 == SCNG(input_filter)(&SCNG(script_filtered), &SCNG(script_filtered_size), SCNG(script_org), SCNG(script_org_size))) {
535					zend_error_noreturn(E_COMPILE_ERROR, "Could not convert the script from the detected "
536							"encoding \"%s\" to a compatible encoding", zend_multibyte_get_encoding_name(LANG_SCNG(script_encoding)));
537				}
538				buf = (char*)SCNG(script_filtered);
539				size = SCNG(script_filtered_size);
540			}
541		}
542		SCNG(yy_start) = (unsigned char *)buf - offset;
543		yy_scan_buffer(buf, (unsigned int)size);
544	} else {
545		zend_error_noreturn(E_COMPILE_ERROR, "zend_stream_mmap() failed");
546	}
547
548	BEGIN(INITIAL);
549
550	if (file_handle->opened_path) {
551		compiled_filename = zend_string_copy(file_handle->opened_path);
552	} else {
553		compiled_filename = zend_string_init(file_handle->filename, strlen(file_handle->filename), 0);
554	}
555
556	zend_set_compiled_filename(compiled_filename);
557	zend_string_release(compiled_filename);
558
559	if (CG(start_lineno)) {
560		CG(zend_lineno) = CG(start_lineno);
561		CG(start_lineno) = 0;
562	} else {
563		CG(zend_lineno) = 1;
564	}
565
566	RESET_DOC_COMMENT();
567	CG(increment_lineno) = 0;
568	return SUCCESS;
569}
570END_EXTERN_C()
571
572static zend_op_array *zend_compile(int type)
573{
574	zend_op_array *op_array = NULL;
575	zend_bool original_in_compilation = CG(in_compilation);
576
577	CG(in_compilation) = 1;
578	CG(ast) = NULL;
579	CG(ast_arena) = zend_arena_create(1024 * 32);
580
581	if (!zendparse()) {
582		zend_file_context original_file_context;
583		zend_oparray_context original_oparray_context;
584		zend_op_array *original_active_op_array = CG(active_op_array);
585
586		op_array = emalloc(sizeof(zend_op_array));
587		init_op_array(op_array, type, INITIAL_OP_ARRAY_SIZE);
588		CG(active_op_array) = op_array;
589
590		if (zend_ast_process) {
591			zend_ast_process(CG(ast));
592		}
593
594		zend_file_context_begin(&original_file_context);
595		zend_oparray_context_begin(&original_oparray_context);
596		zend_compile_top_stmt(CG(ast));
597		zend_emit_final_return(type == ZEND_USER_FUNCTION);
598		op_array->line_start = 1;
599		op_array->line_end = CG(zend_lineno);
600		pass_two(op_array);
601		zend_oparray_context_end(&original_oparray_context);
602		zend_file_context_end(&original_file_context);
603
604		CG(active_op_array) = original_active_op_array;
605	}
606
607	zend_ast_destroy(CG(ast));
608	zend_arena_destroy(CG(ast_arena));
609
610	CG(in_compilation) = original_in_compilation;
611
612	return op_array;
613}
614
615ZEND_API zend_op_array *compile_file(zend_file_handle *file_handle, int type)
616{
617	zend_lex_state original_lex_state;
618	zend_op_array *op_array = NULL;
619	zend_save_lexical_state(&original_lex_state);
620
621	if (open_file_for_scanning(file_handle)==FAILURE) {
622		if (type==ZEND_REQUIRE) {
623			zend_message_dispatcher(ZMSG_FAILED_REQUIRE_FOPEN, file_handle->filename);
624			zend_bailout();
625		} else {
626			zend_message_dispatcher(ZMSG_FAILED_INCLUDE_FOPEN, file_handle->filename);
627		}
628	} else {
629		op_array = zend_compile(ZEND_USER_FUNCTION);
630	}
631
632	zend_restore_lexical_state(&original_lex_state);
633	return op_array;
634}
635
636
637zend_op_array *compile_filename(int type, zval *filename)
638{
639	zend_file_handle file_handle;
640	zval tmp;
641	zend_op_array *retval;
642	zend_string *opened_path = NULL;
643
644	if (Z_TYPE_P(filename) != IS_STRING) {
645		tmp = *filename;
646		zval_copy_ctor(&tmp);
647		convert_to_string(&tmp);
648		filename = &tmp;
649	}
650	file_handle.filename = Z_STRVAL_P(filename);
651	file_handle.free_filename = 0;
652	file_handle.type = ZEND_HANDLE_FILENAME;
653	file_handle.opened_path = NULL;
654	file_handle.handle.fp = NULL;
655
656	retval = zend_compile_file(&file_handle, type);
657	if (retval && file_handle.handle.stream.handle) {
658		if (!file_handle.opened_path) {
659			file_handle.opened_path = opened_path = zend_string_copy(Z_STR_P(filename));
660		}
661
662		zend_hash_add_empty_element(&EG(included_files), file_handle.opened_path);
663
664		if (opened_path) {
665			zend_string_release(opened_path);
666		}
667	}
668	zend_destroy_file_handle(&file_handle);
669
670	if (filename==&tmp) {
671		zval_dtor(&tmp);
672	}
673	return retval;
674}
675
676ZEND_API int zend_prepare_string_for_scanning(zval *str, char *filename)
677{
678	char *buf;
679	size_t size, old_len;
680	zend_string *new_compiled_filename;
681
682	/* enforce ZEND_MMAP_AHEAD trailing NULLs for flex... */
683	old_len = Z_STRLEN_P(str);
684	Z_STR_P(str) = zend_string_extend(Z_STR_P(str), old_len + ZEND_MMAP_AHEAD, 0);
685	Z_TYPE_INFO_P(str) = IS_STRING_EX;
686	memset(Z_STRVAL_P(str) + old_len, 0, ZEND_MMAP_AHEAD + 1);
687
688	SCNG(yy_in) = NULL;
689	SCNG(yy_start) = NULL;
690
691	buf = Z_STRVAL_P(str);
692	size = old_len;
693
694	if (CG(multibyte)) {
695		SCNG(script_org) = (unsigned char*)buf;
696		SCNG(script_org_size) = size;
697		SCNG(script_filtered) = NULL;
698
699		zend_multibyte_set_filter(zend_multibyte_get_internal_encoding());
700
701		if (SCNG(input_filter)) {
702			if ((size_t)-1 == SCNG(input_filter)(&SCNG(script_filtered), &SCNG(script_filtered_size), SCNG(script_org), SCNG(script_org_size))) {
703				zend_error_noreturn(E_COMPILE_ERROR, "Could not convert the script from the detected "
704						"encoding \"%s\" to a compatible encoding", zend_multibyte_get_encoding_name(LANG_SCNG(script_encoding)));
705			}
706			buf = (char*)SCNG(script_filtered);
707			size = SCNG(script_filtered_size);
708		}
709	}
710
711	yy_scan_buffer(buf, (unsigned int)size);
712
713	new_compiled_filename = zend_string_init(filename, strlen(filename), 0);
714	zend_set_compiled_filename(new_compiled_filename);
715	zend_string_release(new_compiled_filename);
716	CG(zend_lineno) = 1;
717	CG(increment_lineno) = 0;
718	RESET_DOC_COMMENT();
719	return SUCCESS;
720}
721
722
723ZEND_API size_t zend_get_scanned_file_offset(void)
724{
725	size_t offset = SCNG(yy_cursor) - SCNG(yy_start);
726	if (SCNG(input_filter)) {
727		size_t original_offset = offset, length = 0;
728		do {
729			unsigned char *p = NULL;
730			if ((size_t)-1 == SCNG(input_filter)(&p, &length, SCNG(script_org), offset)) {
731				return (size_t)-1;
732			}
733			efree(p);
734			if (length > original_offset) {
735				offset--;
736			} else if (length < original_offset) {
737				offset++;
738			}
739		} while (original_offset != length);
740	}
741	return offset;
742}
743
744zend_op_array *compile_string(zval *source_string, char *filename)
745{
746	zend_lex_state original_lex_state;
747	zend_op_array *op_array = NULL;
748	zval tmp;
749
750	if (Z_STRLEN_P(source_string)==0) {
751		return NULL;
752	}
753
754	ZVAL_DUP(&tmp, source_string);
755	convert_to_string(&tmp);
756	source_string = &tmp;
757
758	zend_save_lexical_state(&original_lex_state);
759	if (zend_prepare_string_for_scanning(source_string, filename) == SUCCESS) {
760		BEGIN(ST_IN_SCRIPTING);
761		op_array = zend_compile(ZEND_EVAL_CODE);
762	}
763
764	zend_restore_lexical_state(&original_lex_state);
765	zval_dtor(&tmp);
766
767	return op_array;
768}
769
770
771BEGIN_EXTERN_C()
772int highlight_file(char *filename, zend_syntax_highlighter_ini *syntax_highlighter_ini)
773{
774	zend_lex_state original_lex_state;
775	zend_file_handle file_handle;
776
777	file_handle.type = ZEND_HANDLE_FILENAME;
778	file_handle.filename = filename;
779	file_handle.free_filename = 0;
780	file_handle.opened_path = NULL;
781	zend_save_lexical_state(&original_lex_state);
782	if (open_file_for_scanning(&file_handle)==FAILURE) {
783		zend_message_dispatcher(ZMSG_FAILED_HIGHLIGHT_FOPEN, filename);
784		zend_restore_lexical_state(&original_lex_state);
785		return FAILURE;
786	}
787	zend_highlight(syntax_highlighter_ini);
788	if (SCNG(script_filtered)) {
789		efree(SCNG(script_filtered));
790		SCNG(script_filtered) = NULL;
791	}
792	zend_destroy_file_handle(&file_handle);
793	zend_restore_lexical_state(&original_lex_state);
794	return SUCCESS;
795}
796
797int highlight_string(zval *str, zend_syntax_highlighter_ini *syntax_highlighter_ini, char *str_name)
798{
799	zend_lex_state original_lex_state;
800	zval tmp = *str;
801
802	str = &tmp;
803	zval_copy_ctor(str);
804	zend_save_lexical_state(&original_lex_state);
805	if (zend_prepare_string_for_scanning(str, str_name)==FAILURE) {
806		zend_restore_lexical_state(&original_lex_state);
807		return FAILURE;
808	}
809	BEGIN(INITIAL);
810	zend_highlight(syntax_highlighter_ini);
811	if (SCNG(script_filtered)) {
812		efree(SCNG(script_filtered));
813		SCNG(script_filtered) = NULL;
814	}
815	zend_restore_lexical_state(&original_lex_state);
816	zval_dtor(str);
817	return SUCCESS;
818}
819
820ZEND_API void zend_multibyte_yyinput_again(zend_encoding_filter old_input_filter, const zend_encoding *old_encoding)
821{
822	size_t length;
823	unsigned char *new_yy_start;
824
825	/* convert and set */
826	if (!SCNG(input_filter)) {
827		if (SCNG(script_filtered)) {
828			efree(SCNG(script_filtered));
829			SCNG(script_filtered) = NULL;
830		}
831		SCNG(script_filtered_size) = 0;
832		length = SCNG(script_org_size);
833		new_yy_start = SCNG(script_org);
834	} else {
835		if ((size_t)-1 == SCNG(input_filter)(&new_yy_start, &length, SCNG(script_org), SCNG(script_org_size))) {
836			zend_error_noreturn(E_COMPILE_ERROR, "Could not convert the script from the detected "
837					"encoding \"%s\" to a compatible encoding", zend_multibyte_get_encoding_name(LANG_SCNG(script_encoding)));
838		}
839		if (SCNG(script_filtered)) {
840			efree(SCNG(script_filtered));
841		}
842		SCNG(script_filtered) = new_yy_start;
843		SCNG(script_filtered_size) = length;
844	}
845
846	SCNG(yy_cursor) = new_yy_start + (SCNG(yy_cursor) - SCNG(yy_start));
847	SCNG(yy_marker) = new_yy_start + (SCNG(yy_marker) - SCNG(yy_start));
848	SCNG(yy_text) = new_yy_start + (SCNG(yy_text) - SCNG(yy_start));
849	SCNG(yy_limit) = new_yy_start + length;
850
851	SCNG(yy_start) = new_yy_start;
852}
853
854
855// TODO: avoid reallocation ???
856# define zend_copy_value(zendlval, yytext, yyleng) \
857	if (SCNG(output_filter)) { \
858		size_t sz = 0; \
859		char *s = NULL; \
860		SCNG(output_filter)((unsigned char **)&s, &sz, (unsigned char *)yytext, (size_t)yyleng); \
861		ZVAL_STRINGL(zendlval, s, sz); \
862		efree(s); \
863	} else { \
864		ZVAL_STRINGL(zendlval, yytext, yyleng); \
865	}
866
867static int zend_scan_escape_string(zval *zendlval, char *str, int len, char quote_type)
868{
869	register char *s, *t;
870	char *end;
871
872	ZVAL_STRINGL(zendlval, str, len);
873
874	/* convert escape sequences */
875	s = t = Z_STRVAL_P(zendlval);
876	end = s+Z_STRLEN_P(zendlval);
877	while (s<end) {
878		if (*s=='\\') {
879			s++;
880			if (s >= end) {
881				*t++ = '\\';
882				break;
883			}
884
885			switch(*s) {
886				case 'n':
887					*t++ = '\n';
888					Z_STRLEN_P(zendlval)--;
889					break;
890				case 'r':
891					*t++ = '\r';
892					Z_STRLEN_P(zendlval)--;
893					break;
894				case 't':
895					*t++ = '\t';
896					Z_STRLEN_P(zendlval)--;
897					break;
898				case 'f':
899					*t++ = '\f';
900					Z_STRLEN_P(zendlval)--;
901					break;
902				case 'v':
903					*t++ = '\v';
904					Z_STRLEN_P(zendlval)--;
905					break;
906				case 'e':
907#ifdef ZEND_WIN32
908					*t++ = VK_ESCAPE;
909#else
910					*t++ = '\e';
911#endif
912					Z_STRLEN_P(zendlval)--;
913					break;
914				case '"':
915				case '`':
916					if (*s != quote_type) {
917						*t++ = '\\';
918						*t++ = *s;
919						break;
920					}
921				case '\\':
922				case '$':
923					*t++ = *s;
924					Z_STRLEN_P(zendlval)--;
925					break;
926				case 'x':
927				case 'X':
928					if (ZEND_IS_HEX(*(s+1))) {
929						char hex_buf[3] = { 0, 0, 0 };
930
931						Z_STRLEN_P(zendlval)--; /* for the 'x' */
932
933						hex_buf[0] = *(++s);
934						Z_STRLEN_P(zendlval)--;
935						if (ZEND_IS_HEX(*(s+1))) {
936							hex_buf[1] = *(++s);
937							Z_STRLEN_P(zendlval)--;
938						}
939						*t++ = (char) ZEND_STRTOL(hex_buf, NULL, 16);
940					} else {
941						*t++ = '\\';
942						*t++ = *s;
943					}
944					break;
945				/* UTF-8 codepoint escape, format: /\\u\{\x+\}/ */
946				case 'u':
947					{
948						/* cache where we started so we can parse after validating */
949						char *start = s + 1;
950						size_t len = 0;
951						zend_bool valid = 1;
952						unsigned long codepoint;
953						size_t byte_len = 0;
954
955						if (*start != '{') {
956							/* we silently let this pass to avoid breaking code
957							 * with JSON in string literals (e.g. "\"\u202e\""
958							 */
959							*t++ = '\\';
960							*t++ = 'u';
961							break;
962						} else {
963							/* on the other hand, invalid \u{blah} errors */
964							s++;
965							len++;
966							s++;
967							while (*s != '}') {
968								if (!ZEND_IS_HEX(*s)) {
969									valid = 0;
970									break;
971								} else {
972									len++;
973								}
974								s++;
975							}
976							if (*s == '}') {
977								valid = 1;
978								len++;
979							}
980						}
981
982						/* \u{} is invalid */
983						if (len <= 2) {
984							valid = 0;
985						}
986
987						if (!valid) {
988							zend_throw_exception(zend_ce_parse_error,
989								"Invalid UTF-8 codepoint escape sequence", 0);
990							zval_ptr_dtor(zendlval);
991							ZVAL_UNDEF(zendlval);
992							return FAILURE;
993						}
994
995						errno = 0;
996						codepoint = strtoul(start + 1, NULL, 16);
997
998						/* per RFC 3629, UTF-8 can only represent 21 bits */
999						if (codepoint > 0x10FFFF || errno) {
1000							zend_throw_exception(zend_ce_parse_error,
1001								"Invalid UTF-8 codepoint escape sequence: Codepoint too large", 0);
1002							zval_ptr_dtor(zendlval);
1003							ZVAL_UNDEF(zendlval);
1004							return FAILURE;
1005						}
1006
1007						/* based on https://en.wikipedia.org/wiki/UTF-8#Sample_code */
1008						if (codepoint < 0x80) {
1009							byte_len = 1;
1010							*t++ = codepoint;
1011						} else if (codepoint <= 0x7FF) {
1012							byte_len = 2;
1013							*t++ = (codepoint >> 6) + 0xC0;
1014							*t++ = (codepoint & 0x3F) + 0x80;
1015						} else if (codepoint <= 0xFFFF) {
1016							byte_len = 3;
1017							*t++ = (codepoint >> 12) + 0xE0;
1018							*t++ = ((codepoint >> 6) & 0x3F) + 0x80;
1019							*t++ = (codepoint & 0x3F) + 0x80;
1020						} else if (codepoint <= 0x10FFFF) {
1021							byte_len = 4;
1022							*t++ = (codepoint >> 18) + 0xF0;
1023							*t++ = ((codepoint >> 12) & 0x3F) + 0x80;
1024							*t++ = ((codepoint >> 6) & 0x3F) + 0x80;
1025							*t++ = (codepoint & 0x3F) + 0x80;
1026						}
1027
1028						Z_STRLEN_P(zendlval) -= 2; /* \u */
1029						Z_STRLEN_P(zendlval) -= (len - byte_len);
1030					}
1031					break;
1032				default:
1033					/* check for an octal */
1034					if (ZEND_IS_OCT(*s)) {
1035						char octal_buf[4] = { 0, 0, 0, 0 };
1036
1037						octal_buf[0] = *s;
1038						Z_STRLEN_P(zendlval)--;
1039						if (ZEND_IS_OCT(*(s+1))) {
1040							octal_buf[1] = *(++s);
1041							Z_STRLEN_P(zendlval)--;
1042							if (ZEND_IS_OCT(*(s+1))) {
1043								octal_buf[2] = *(++s);
1044								Z_STRLEN_P(zendlval)--;
1045							}
1046						}
1047						if (octal_buf[2] &&
1048						    (octal_buf[0] > '3')) {
1049							/* 3 octit values must not overflow 0xFF (\377) */
1050							zend_error(E_COMPILE_WARNING, "Octal escape sequence overflow \\%s is greater than \\377", octal_buf);
1051						}
1052
1053						*t++ = (char) ZEND_STRTOL(octal_buf, NULL, 8);
1054					} else {
1055						*t++ = '\\';
1056						*t++ = *s;
1057					}
1058					break;
1059			}
1060		} else {
1061			*t++ = *s;
1062		}
1063
1064		if (*s == '\n' || (*s == '\r' && (*(s+1) != '\n'))) {
1065			CG(zend_lineno)++;
1066		}
1067		s++;
1068	}
1069	*t = 0;
1070	if (SCNG(output_filter)) {
1071		size_t sz = 0;
1072		unsigned char *str;
1073		// TODO: avoid realocation ???
1074		s = Z_STRVAL_P(zendlval);
1075		SCNG(output_filter)(&str, &sz, (unsigned char *)s, (size_t)Z_STRLEN_P(zendlval));
1076		zval_ptr_dtor(zendlval);
1077		ZVAL_STRINGL(zendlval, (char *) str, sz);
1078		efree(str);
1079	}
1080	return SUCCESS;
1081}
1082
1083static zend_always_inline int emit_token(int token, int token_line)
1084{
1085	if(SCNG(on_event)) SCNG(on_event)(ON_TOKEN, token, token_line);
1086
1087	return token;
1088}
1089
1090#define RETURN_TOKEN(token) return emit_token(token, start_line);
1091
1092int lex_scan(zval *zendlval)
1093{
1094
1095int start_line = CG(zend_lineno);
1096
1097restart:
1098	SCNG(yy_text) = YYCURSOR;
1099
1100/*!re2c
1101re2c:yyfill:check = 0;
1102LNUM	[0-9]+
1103DNUM	([0-9]*"."[0-9]+)|([0-9]+"."[0-9]*)
1104EXPONENT_DNUM	(({LNUM}|{DNUM})[eE][+-]?{LNUM})
1105HNUM	"0x"[0-9a-fA-F]+
1106BNUM	"0b"[01]+
1107LABEL	[a-zA-Z_\x80-\xff][a-zA-Z0-9_\x80-\xff]*
1108WHITESPACE [ \n\r\t]+
1109TABS_AND_SPACES [ \t]*
1110TOKENS [;:,.\[\]()|^&+-/*=%!~$<>?@]
1111ANY_CHAR [^]
1112NEWLINE ("\r"|"\n"|"\r\n")
1113
1114/* compute yyleng before each rule */
1115<!*> := yyleng = YYCURSOR - SCNG(yy_text);
1116
1117<ST_IN_SCRIPTING>"exit" {
1118	RETURN_TOKEN(T_EXIT);
1119}
1120
1121<ST_IN_SCRIPTING>"die" {
1122	RETURN_TOKEN(T_EXIT);
1123}
1124
1125<ST_IN_SCRIPTING>"function" {
1126	RETURN_TOKEN(T_FUNCTION);
1127}
1128
1129<ST_IN_SCRIPTING>"const" {
1130	RETURN_TOKEN(T_CONST);
1131}
1132
1133<ST_IN_SCRIPTING>"return" {
1134	RETURN_TOKEN(T_RETURN);
1135}
1136
1137<ST_IN_SCRIPTING>"yield"{WHITESPACE}"from" {
1138	HANDLE_NEWLINES(yytext, yyleng);
1139	RETURN_TOKEN(T_YIELD_FROM);
1140}
1141
1142<ST_IN_SCRIPTING>"yield" {
1143	RETURN_TOKEN(T_YIELD);
1144}
1145
1146<ST_IN_SCRIPTING>"try" {
1147	RETURN_TOKEN(T_TRY);
1148}
1149
1150<ST_IN_SCRIPTING>"catch" {
1151	RETURN_TOKEN(T_CATCH);
1152}
1153
1154<ST_IN_SCRIPTING>"finally" {
1155	RETURN_TOKEN(T_FINALLY);
1156}
1157
1158<ST_IN_SCRIPTING>"throw" {
1159	RETURN_TOKEN(T_THROW);
1160}
1161
1162<ST_IN_SCRIPTING>"if" {
1163	RETURN_TOKEN(T_IF);
1164}
1165
1166<ST_IN_SCRIPTING>"elseif" {
1167	RETURN_TOKEN(T_ELSEIF);
1168}
1169
1170<ST_IN_SCRIPTING>"endif" {
1171	RETURN_TOKEN(T_ENDIF);
1172}
1173
1174<ST_IN_SCRIPTING>"else" {
1175	RETURN_TOKEN(T_ELSE);
1176}
1177
1178<ST_IN_SCRIPTING>"while" {
1179	RETURN_TOKEN(T_WHILE);
1180}
1181
1182<ST_IN_SCRIPTING>"endwhile" {
1183	RETURN_TOKEN(T_ENDWHILE);
1184}
1185
1186<ST_IN_SCRIPTING>"do" {
1187	RETURN_TOKEN(T_DO);
1188}
1189
1190<ST_IN_SCRIPTING>"for" {
1191	RETURN_TOKEN(T_FOR);
1192}
1193
1194<ST_IN_SCRIPTING>"endfor" {
1195	RETURN_TOKEN(T_ENDFOR);
1196}
1197
1198<ST_IN_SCRIPTING>"foreach" {
1199	RETURN_TOKEN(T_FOREACH);
1200}
1201
1202<ST_IN_SCRIPTING>"endforeach" {
1203	RETURN_TOKEN(T_ENDFOREACH);
1204}
1205
1206<ST_IN_SCRIPTING>"declare" {
1207	RETURN_TOKEN(T_DECLARE);
1208}
1209
1210<ST_IN_SCRIPTING>"enddeclare" {
1211	RETURN_TOKEN(T_ENDDECLARE);
1212}
1213
1214<ST_IN_SCRIPTING>"instanceof" {
1215	RETURN_TOKEN(T_INSTANCEOF);
1216}
1217
1218<ST_IN_SCRIPTING>"as" {
1219	RETURN_TOKEN(T_AS);
1220}
1221
1222<ST_IN_SCRIPTING>"switch" {
1223	RETURN_TOKEN(T_SWITCH);
1224}
1225
1226<ST_IN_SCRIPTING>"endswitch" {
1227	RETURN_TOKEN(T_ENDSWITCH);
1228}
1229
1230<ST_IN_SCRIPTING>"case" {
1231	RETURN_TOKEN(T_CASE);
1232}
1233
1234<ST_IN_SCRIPTING>"default" {
1235	RETURN_TOKEN(T_DEFAULT);
1236}
1237
1238<ST_IN_SCRIPTING>"break" {
1239	RETURN_TOKEN(T_BREAK);
1240}
1241
1242<ST_IN_SCRIPTING>"continue" {
1243	RETURN_TOKEN(T_CONTINUE);
1244}
1245
1246<ST_IN_SCRIPTING>"goto" {
1247	RETURN_TOKEN(T_GOTO);
1248}
1249
1250<ST_IN_SCRIPTING>"echo" {
1251	RETURN_TOKEN(T_ECHO);
1252}
1253
1254<ST_IN_SCRIPTING>"print" {
1255	RETURN_TOKEN(T_PRINT);
1256}
1257
1258<ST_IN_SCRIPTING>"class" {
1259	RETURN_TOKEN(T_CLASS);
1260}
1261
1262<ST_IN_SCRIPTING>"interface" {
1263	RETURN_TOKEN(T_INTERFACE);
1264}
1265
1266<ST_IN_SCRIPTING>"trait" {
1267	RETURN_TOKEN(T_TRAIT);
1268}
1269
1270<ST_IN_SCRIPTING>"extends" {
1271	RETURN_TOKEN(T_EXTENDS);
1272}
1273
1274<ST_IN_SCRIPTING>"implements" {
1275	RETURN_TOKEN(T_IMPLEMENTS);
1276}
1277
1278<ST_IN_SCRIPTING>"->" {
1279	yy_push_state(ST_LOOKING_FOR_PROPERTY);
1280	RETURN_TOKEN(T_OBJECT_OPERATOR);
1281}
1282
1283<ST_IN_SCRIPTING,ST_LOOKING_FOR_PROPERTY>{WHITESPACE}+ {
1284	HANDLE_NEWLINES(yytext, yyleng);
1285	RETURN_TOKEN(T_WHITESPACE);
1286}
1287
1288<ST_LOOKING_FOR_PROPERTY>"->" {
1289	RETURN_TOKEN(T_OBJECT_OPERATOR);
1290}
1291
1292<ST_LOOKING_FOR_PROPERTY>{LABEL} {
1293	yy_pop_state();
1294	zend_copy_value(zendlval, yytext, yyleng);
1295	RETURN_TOKEN(T_STRING);
1296}
1297
1298<ST_LOOKING_FOR_PROPERTY>{ANY_CHAR} {
1299	yyless(0);
1300	yy_pop_state();
1301	goto restart;
1302}
1303
1304<ST_IN_SCRIPTING>"::" {
1305	RETURN_TOKEN(T_PAAMAYIM_NEKUDOTAYIM);
1306}
1307
1308<ST_IN_SCRIPTING>"\\" {
1309	RETURN_TOKEN(T_NS_SEPARATOR);
1310}
1311
1312<ST_IN_SCRIPTING>"..." {
1313	RETURN_TOKEN(T_ELLIPSIS);
1314}
1315
1316<ST_IN_SCRIPTING>"??" {
1317	RETURN_TOKEN(T_COALESCE);
1318}
1319
1320<ST_IN_SCRIPTING>"new" {
1321	RETURN_TOKEN(T_NEW);
1322}
1323
1324<ST_IN_SCRIPTING>"clone" {
1325	RETURN_TOKEN(T_CLONE);
1326}
1327
1328<ST_IN_SCRIPTING>"var" {
1329	RETURN_TOKEN(T_VAR);
1330}
1331
1332<ST_IN_SCRIPTING>"("{TABS_AND_SPACES}("int"|"integer"){TABS_AND_SPACES}")" {
1333	RETURN_TOKEN(T_INT_CAST);
1334}
1335
1336<ST_IN_SCRIPTING>"("{TABS_AND_SPACES}("real"|"double"|"float"){TABS_AND_SPACES}")" {
1337	RETURN_TOKEN(T_DOUBLE_CAST);
1338}
1339
1340<ST_IN_SCRIPTING>"("{TABS_AND_SPACES}("string"|"binary"){TABS_AND_SPACES}")" {
1341	RETURN_TOKEN(T_STRING_CAST);
1342}
1343
1344<ST_IN_SCRIPTING>"("{TABS_AND_SPACES}"array"{TABS_AND_SPACES}")" {
1345	RETURN_TOKEN(T_ARRAY_CAST);
1346}
1347
1348<ST_IN_SCRIPTING>"("{TABS_AND_SPACES}"object"{TABS_AND_SPACES}")" {
1349	RETURN_TOKEN(T_OBJECT_CAST);
1350}
1351
1352<ST_IN_SCRIPTING>"("{TABS_AND_SPACES}("bool"|"boolean"){TABS_AND_SPACES}")" {
1353	RETURN_TOKEN(T_BOOL_CAST);
1354}
1355
1356<ST_IN_SCRIPTING>"("{TABS_AND_SPACES}("unset"){TABS_AND_SPACES}")" {
1357	RETURN_TOKEN(T_UNSET_CAST);
1358}
1359
1360<ST_IN_SCRIPTING>"eval" {
1361	RETURN_TOKEN(T_EVAL);
1362}
1363
1364<ST_IN_SCRIPTING>"include" {
1365	RETURN_TOKEN(T_INCLUDE);
1366}
1367
1368<ST_IN_SCRIPTING>"include_once" {
1369	RETURN_TOKEN(T_INCLUDE_ONCE);
1370}
1371
1372<ST_IN_SCRIPTING>"require" {
1373	RETURN_TOKEN(T_REQUIRE);
1374}
1375
1376<ST_IN_SCRIPTING>"require_once" {
1377	RETURN_TOKEN(T_REQUIRE_ONCE);
1378}
1379
1380<ST_IN_SCRIPTING>"namespace" {
1381	RETURN_TOKEN(T_NAMESPACE);
1382}
1383
1384<ST_IN_SCRIPTING>"use" {
1385	RETURN_TOKEN(T_USE);
1386}
1387
1388<ST_IN_SCRIPTING>"insteadof" {
1389    RETURN_TOKEN(T_INSTEADOF);
1390}
1391
1392<ST_IN_SCRIPTING>"global" {
1393	RETURN_TOKEN(T_GLOBAL);
1394}
1395
1396<ST_IN_SCRIPTING>"isset" {
1397	RETURN_TOKEN(T_ISSET);
1398}
1399
1400<ST_IN_SCRIPTING>"empty" {
1401	RETURN_TOKEN(T_EMPTY);
1402}
1403
1404<ST_IN_SCRIPTING>"__halt_compiler" {
1405	RETURN_TOKEN(T_HALT_COMPILER);
1406}
1407
1408<ST_IN_SCRIPTING>"static" {
1409	RETURN_TOKEN(T_STATIC);
1410}
1411
1412<ST_IN_SCRIPTING>"abstract" {
1413	RETURN_TOKEN(T_ABSTRACT);
1414}
1415
1416<ST_IN_SCRIPTING>"final" {
1417	RETURN_TOKEN(T_FINAL);
1418}
1419
1420<ST_IN_SCRIPTING>"private" {
1421	RETURN_TOKEN(T_PRIVATE);
1422}
1423
1424<ST_IN_SCRIPTING>"protected" {
1425	RETURN_TOKEN(T_PROTECTED);
1426}
1427
1428<ST_IN_SCRIPTING>"public" {
1429	RETURN_TOKEN(T_PUBLIC);
1430}
1431
1432<ST_IN_SCRIPTING>"unset" {
1433	RETURN_TOKEN(T_UNSET);
1434}
1435
1436<ST_IN_SCRIPTING>"=>" {
1437	RETURN_TOKEN(T_DOUBLE_ARROW);
1438}
1439
1440<ST_IN_SCRIPTING>"list" {
1441	RETURN_TOKEN(T_LIST);
1442}
1443
1444<ST_IN_SCRIPTING>"array" {
1445	RETURN_TOKEN(T_ARRAY);
1446}
1447
1448<ST_IN_SCRIPTING>"callable" {
1449	RETURN_TOKEN(T_CALLABLE);
1450}
1451
1452<ST_IN_SCRIPTING>"++" {
1453	RETURN_TOKEN(T_INC);
1454}
1455
1456<ST_IN_SCRIPTING>"--" {
1457	RETURN_TOKEN(T_DEC);
1458}
1459
1460<ST_IN_SCRIPTING>"===" {
1461	RETURN_TOKEN(T_IS_IDENTICAL);
1462}
1463
1464<ST_IN_SCRIPTING>"!==" {
1465	RETURN_TOKEN(T_IS_NOT_IDENTICAL);
1466}
1467
1468<ST_IN_SCRIPTING>"==" {
1469	RETURN_TOKEN(T_IS_EQUAL);
1470}
1471
1472<ST_IN_SCRIPTING>"!="|"<>" {
1473	RETURN_TOKEN(T_IS_NOT_EQUAL);
1474}
1475
1476<ST_IN_SCRIPTING>"<=>" {
1477	RETURN_TOKEN(T_SPACESHIP);
1478}
1479
1480<ST_IN_SCRIPTING>"<=" {
1481	RETURN_TOKEN(T_IS_SMALLER_OR_EQUAL);
1482}
1483
1484<ST_IN_SCRIPTING>">=" {
1485	RETURN_TOKEN(T_IS_GREATER_OR_EQUAL);
1486}
1487
1488<ST_IN_SCRIPTING>"+=" {
1489	RETURN_TOKEN(T_PLUS_EQUAL);
1490}
1491
1492<ST_IN_SCRIPTING>"-=" {
1493	RETURN_TOKEN(T_MINUS_EQUAL);
1494}
1495
1496<ST_IN_SCRIPTING>"*=" {
1497	RETURN_TOKEN(T_MUL_EQUAL);
1498}
1499
1500<ST_IN_SCRIPTING>"*\*" {
1501	RETURN_TOKEN(T_POW);
1502}
1503
1504<ST_IN_SCRIPTING>"*\*=" {
1505	RETURN_TOKEN(T_POW_EQUAL);
1506}
1507
1508<ST_IN_SCRIPTING>"/=" {
1509	RETURN_TOKEN(T_DIV_EQUAL);
1510}
1511
1512<ST_IN_SCRIPTING>".=" {
1513	RETURN_TOKEN(T_CONCAT_EQUAL);
1514}
1515
1516<ST_IN_SCRIPTING>"%=" {
1517	RETURN_TOKEN(T_MOD_EQUAL);
1518}
1519
1520<ST_IN_SCRIPTING>"<<=" {
1521	RETURN_TOKEN(T_SL_EQUAL);
1522}
1523
1524<ST_IN_SCRIPTING>">>=" {
1525	RETURN_TOKEN(T_SR_EQUAL);
1526}
1527
1528<ST_IN_SCRIPTING>"&=" {
1529	RETURN_TOKEN(T_AND_EQUAL);
1530}
1531
1532<ST_IN_SCRIPTING>"|=" {
1533	RETURN_TOKEN(T_OR_EQUAL);
1534}
1535
1536<ST_IN_SCRIPTING>"^=" {
1537	RETURN_TOKEN(T_XOR_EQUAL);
1538}
1539
1540<ST_IN_SCRIPTING>"||" {
1541	RETURN_TOKEN(T_BOOLEAN_OR);
1542}
1543
1544<ST_IN_SCRIPTING>"&&" {
1545	RETURN_TOKEN(T_BOOLEAN_AND);
1546}
1547
1548<ST_IN_SCRIPTING>"OR" {
1549	RETURN_TOKEN(T_LOGICAL_OR);
1550}
1551
1552<ST_IN_SCRIPTING>"AND" {
1553	RETURN_TOKEN(T_LOGICAL_AND);
1554}
1555
1556<ST_IN_SCRIPTING>"XOR" {
1557	RETURN_TOKEN(T_LOGICAL_XOR);
1558}
1559
1560<ST_IN_SCRIPTING>"<<" {
1561	RETURN_TOKEN(T_SL);
1562}
1563
1564<ST_IN_SCRIPTING>">>" {
1565	RETURN_TOKEN(T_SR);
1566}
1567
1568<ST_IN_SCRIPTING>{TOKENS} {
1569	RETURN_TOKEN(yytext[0]);
1570}
1571
1572
1573<ST_IN_SCRIPTING>"{" {
1574	yy_push_state(ST_IN_SCRIPTING);
1575	RETURN_TOKEN('{');
1576}
1577
1578
1579<ST_DOUBLE_QUOTES,ST_BACKQUOTE,ST_HEREDOC>"${" {
1580	yy_push_state(ST_LOOKING_FOR_VARNAME);
1581	RETURN_TOKEN(T_DOLLAR_OPEN_CURLY_BRACES);
1582}
1583
1584
1585<ST_IN_SCRIPTING>"}" {
1586	RESET_DOC_COMMENT();
1587	if (!zend_stack_is_empty(&SCNG(state_stack))) {
1588		yy_pop_state();
1589	}
1590	RETURN_TOKEN('}');
1591}
1592
1593
1594<ST_LOOKING_FOR_VARNAME>{LABEL}[[}] {
1595	yyless(yyleng - 1);
1596	zend_copy_value(zendlval, yytext, yyleng);
1597	yy_pop_state();
1598	yy_push_state(ST_IN_SCRIPTING);
1599	RETURN_TOKEN(T_STRING_VARNAME);
1600}
1601
1602
1603<ST_LOOKING_FOR_VARNAME>{ANY_CHAR} {
1604	yyless(0);
1605	yy_pop_state();
1606	yy_push_state(ST_IN_SCRIPTING);
1607	goto restart;
1608}
1609
1610<ST_IN_SCRIPTING>{BNUM} {
1611	char *bin = yytext + 2; /* Skip "0b" */
1612	int len = yyleng - 2;
1613	char *end;
1614
1615	/* Skip any leading 0s */
1616	while (*bin == '0') {
1617		++bin;
1618		--len;
1619	}
1620
1621	if (len < SIZEOF_ZEND_LONG * 8) {
1622		if (len == 0) {
1623			ZVAL_LONG(zendlval, 0);
1624		} else {
1625			errno = 0;
1626			ZVAL_LONG(zendlval, ZEND_STRTOL(bin, &end, 2));
1627			ZEND_ASSERT(!errno && end == yytext + yyleng);
1628		}
1629		RETURN_TOKEN(T_LNUMBER);
1630	} else {
1631		ZVAL_DOUBLE(zendlval, zend_bin_strtod(bin, (const char **)&end));
1632		/* errno isn't checked since we allow HUGE_VAL/INF overflow */
1633		ZEND_ASSERT(end == yytext + yyleng);
1634		RETURN_TOKEN(T_DNUMBER);
1635	}
1636}
1637
1638<ST_IN_SCRIPTING>{LNUM} {
1639	char *end;
1640	if (yyleng < MAX_LENGTH_OF_LONG - 1) { /* Won't overflow */
1641		errno = 0;
1642		ZVAL_LONG(zendlval, ZEND_STRTOL(yytext, &end, 0));
1643		/* This isn't an assert, we need to ensure 019 isn't valid octal
1644		 * Because the lexing itself doesn't do that for us
1645		 */
1646		if (end != yytext + yyleng) {
1647			zend_throw_exception(zend_ce_parse_error, "Invalid numeric literal", 0);
1648			ZVAL_UNDEF(zendlval);
1649			RETURN_TOKEN(T_LNUMBER);
1650		}
1651	} else {
1652		errno = 0;
1653		ZVAL_LONG(zendlval, ZEND_STRTOL(yytext, &end, 0));
1654		if (errno == ERANGE) { /* Overflow */
1655			errno = 0;
1656			if (yytext[0] == '0') { /* octal overflow */
1657				errno = 0;
1658				ZVAL_DOUBLE(zendlval, zend_oct_strtod(yytext, (const char **)&end));
1659			} else {
1660				ZVAL_DOUBLE(zendlval, zend_strtod(yytext, (const char **)&end));
1661			}
1662			/* Also not an assert for the same reason */
1663			if (end != yytext + yyleng) {
1664				zend_throw_exception(zend_ce_parse_error,
1665					"Invalid numeric literal", 0);
1666				ZVAL_UNDEF(zendlval);
1667				RETURN_TOKEN(T_DNUMBER);
1668			}
1669			ZEND_ASSERT(!errno);
1670			RETURN_TOKEN(T_DNUMBER);
1671		}
1672		/* Also not an assert for the same reason */
1673		if (end != yytext + yyleng) {
1674			zend_throw_exception(zend_ce_parse_error, "Invalid numeric literal", 0);
1675			ZVAL_UNDEF(zendlval);
1676			RETURN_TOKEN(T_DNUMBER);
1677		}
1678	}
1679	ZEND_ASSERT(!errno);
1680	RETURN_TOKEN(T_LNUMBER);
1681}
1682
1683<ST_IN_SCRIPTING>{HNUM} {
1684	char *hex = yytext + 2; /* Skip "0x" */
1685	int len = yyleng - 2;
1686	char *end;
1687
1688	/* Skip any leading 0s */
1689	while (*hex == '0') {
1690		hex++;
1691		len--;
1692	}
1693
1694	if (len < SIZEOF_ZEND_LONG * 2 || (len == SIZEOF_ZEND_LONG * 2 && *hex <= '7')) {
1695		if (len == 0) {
1696			ZVAL_LONG(zendlval, 0);
1697		} else {
1698			errno = 0;
1699			ZVAL_LONG(zendlval, ZEND_STRTOL(hex, &end, 16));
1700			ZEND_ASSERT(!errno && end == hex + len);
1701		}
1702		RETURN_TOKEN(T_LNUMBER);
1703	} else {
1704		ZVAL_DOUBLE(zendlval, zend_hex_strtod(hex, (const char **)&end));
1705		/* errno isn't checked since we allow HUGE_VAL/INF overflow */
1706		ZEND_ASSERT(end == hex + len);
1707		RETURN_TOKEN(T_DNUMBER);
1708	}
1709}
1710
1711<ST_VAR_OFFSET>[0]|([1-9][0-9]*) { /* Offset could be treated as a long */
1712	if (yyleng < MAX_LENGTH_OF_LONG - 1 || (yyleng == MAX_LENGTH_OF_LONG - 1 && strcmp(yytext, long_min_digits) < 0)) {
1713		char *end;
1714		errno = 0;
1715		ZVAL_LONG(zendlval, ZEND_STRTOL(yytext, &end, 10));
1716		if (errno == ERANGE) {
1717			goto string;
1718		}
1719		ZEND_ASSERT(end == yytext + yyleng);
1720	} else {
1721string:
1722		ZVAL_STRINGL(zendlval, yytext, yyleng);
1723	}
1724	RETURN_TOKEN(T_NUM_STRING);
1725}
1726
1727<ST_VAR_OFFSET>{LNUM}|{HNUM}|{BNUM} { /* Offset must be treated as a string */
1728	ZVAL_STRINGL(zendlval, yytext, yyleng);
1729	RETURN_TOKEN(T_NUM_STRING);
1730}
1731
1732<ST_IN_SCRIPTING>{DNUM}|{EXPONENT_DNUM} {
1733	const char *end;
1734
1735	ZVAL_DOUBLE(zendlval, zend_strtod(yytext, &end));
1736	/* errno isn't checked since we allow HUGE_VAL/INF overflow */
1737	ZEND_ASSERT(end == yytext + yyleng);
1738	RETURN_TOKEN(T_DNUMBER);
1739}
1740
1741<ST_IN_SCRIPTING>"__CLASS__" {
1742	RETURN_TOKEN(T_CLASS_C);
1743}
1744
1745<ST_IN_SCRIPTING>"__TRAIT__" {
1746	RETURN_TOKEN(T_TRAIT_C);
1747}
1748
1749<ST_IN_SCRIPTING>"__FUNCTION__" {
1750	RETURN_TOKEN(T_FUNC_C);
1751}
1752
1753<ST_IN_SCRIPTING>"__METHOD__" {
1754	RETURN_TOKEN(T_METHOD_C);
1755}
1756
1757<ST_IN_SCRIPTING>"__LINE__" {
1758	RETURN_TOKEN(T_LINE);
1759}
1760
1761<ST_IN_SCRIPTING>"__FILE__" {
1762	RETURN_TOKEN(T_FILE);
1763}
1764
1765<ST_IN_SCRIPTING>"__DIR__" {
1766	RETURN_TOKEN(T_DIR);
1767}
1768
1769<ST_IN_SCRIPTING>"__NAMESPACE__" {
1770	RETURN_TOKEN(T_NS_C);
1771}
1772
1773
1774<INITIAL>"<?=" {
1775	BEGIN(ST_IN_SCRIPTING);
1776	RETURN_TOKEN(T_OPEN_TAG_WITH_ECHO);
1777}
1778
1779
1780<INITIAL>"<?php"([ \t]|{NEWLINE}) {
1781	HANDLE_NEWLINE(yytext[yyleng-1]);
1782	BEGIN(ST_IN_SCRIPTING);
1783	RETURN_TOKEN(T_OPEN_TAG);
1784}
1785
1786
1787<INITIAL>"<?" {
1788	if (CG(short_tags)) {
1789		BEGIN(ST_IN_SCRIPTING);
1790		RETURN_TOKEN(T_OPEN_TAG);
1791	} else {
1792		goto inline_char_handler;
1793	}
1794}
1795
1796<INITIAL>{ANY_CHAR} {
1797	if (YYCURSOR > YYLIMIT) {
1798		RETURN_TOKEN(END);
1799	}
1800
1801inline_char_handler:
1802
1803	while (1) {
1804		YYCTYPE *ptr = memchr(YYCURSOR, '<', YYLIMIT - YYCURSOR);
1805
1806		YYCURSOR = ptr ? ptr + 1 : YYLIMIT;
1807
1808		if (YYCURSOR >= YYLIMIT) {
1809			break;
1810		}
1811
1812		if (*YYCURSOR == '?') {
1813			if (CG(short_tags) || !strncasecmp((char*)YYCURSOR + 1, "php", 3) || (*(YYCURSOR + 1) == '=')) { /* Assume [ \t\n\r] follows "php" */
1814
1815				YYCURSOR--;
1816				break;
1817			}
1818		}
1819	}
1820
1821	yyleng = YYCURSOR - SCNG(yy_text);
1822
1823	if (SCNG(output_filter)) {
1824		size_t readsize;
1825		char *s = NULL;
1826		size_t sz = 0;
1827		// TODO: avoid reallocation ???
1828		readsize = SCNG(output_filter)((unsigned char **)&s, &sz, (unsigned char *)yytext, (size_t)yyleng);
1829		ZVAL_STRINGL(zendlval, s, sz);
1830		efree(s);
1831		if (readsize < yyleng) {
1832			yyless(readsize);
1833		}
1834	} else {
1835	  ZVAL_STRINGL(zendlval, yytext, yyleng);
1836	}
1837	HANDLE_NEWLINES(yytext, yyleng);
1838	RETURN_TOKEN(T_INLINE_HTML);
1839}
1840
1841
1842/* Make sure a label character follows "->", otherwise there is no property
1843 * and "->" will be taken literally
1844 */
1845<ST_DOUBLE_QUOTES,ST_HEREDOC,ST_BACKQUOTE>"$"{LABEL}"->"[a-zA-Z_\x80-\xff] {
1846	yyless(yyleng - 3);
1847	yy_push_state(ST_LOOKING_FOR_PROPERTY);
1848	zend_copy_value(zendlval, (yytext+1), (yyleng-1));
1849	RETURN_TOKEN(T_VARIABLE);
1850}
1851
1852/* A [ always designates a variable offset, regardless of what follows
1853 */
1854<ST_DOUBLE_QUOTES,ST_HEREDOC,ST_BACKQUOTE>"$"{LABEL}"[" {
1855	yyless(yyleng - 1);
1856	yy_push_state(ST_VAR_OFFSET);
1857	zend_copy_value(zendlval, (yytext+1), (yyleng-1));
1858	RETURN_TOKEN(T_VARIABLE);
1859}
1860
1861<ST_IN_SCRIPTING,ST_DOUBLE_QUOTES,ST_HEREDOC,ST_BACKQUOTE,ST_VAR_OFFSET>"$"{LABEL} {
1862	zend_copy_value(zendlval, (yytext+1), (yyleng-1));
1863	RETURN_TOKEN(T_VARIABLE);
1864}
1865
1866<ST_VAR_OFFSET>"]" {
1867	yy_pop_state();
1868	RETURN_TOKEN(']');
1869}
1870
1871<ST_VAR_OFFSET>{TOKENS}|[{}"`] {
1872	/* Only '[' can be valid, but returning other tokens will allow a more explicit parse error */
1873	RETURN_TOKEN(yytext[0]);
1874}
1875
1876<ST_VAR_OFFSET>[ \n\r\t\\'#] {
1877	/* Invalid rule to return a more explicit parse error with proper line number */
1878	yyless(0);
1879	yy_pop_state();
1880	ZVAL_NULL(zendlval);
1881	RETURN_TOKEN(T_ENCAPSED_AND_WHITESPACE);
1882}
1883
1884<ST_IN_SCRIPTING,ST_VAR_OFFSET>{LABEL} {
1885	zend_copy_value(zendlval, yytext, yyleng);
1886	RETURN_TOKEN(T_STRING);
1887}
1888
1889
1890<ST_IN_SCRIPTING>"#"|"//" {
1891	while (YYCURSOR < YYLIMIT) {
1892		switch (*YYCURSOR++) {
1893			case '\r':
1894				if (*YYCURSOR == '\n') {
1895					YYCURSOR++;
1896				}
1897				/* fall through */
1898			case '\n':
1899				CG(zend_lineno)++;
1900				break;
1901			case '?':
1902				if (*YYCURSOR == '>') {
1903					YYCURSOR--;
1904					break;
1905				}
1906				/* fall through */
1907			default:
1908				continue;
1909		}
1910
1911		break;
1912	}
1913
1914	yyleng = YYCURSOR - SCNG(yy_text);
1915
1916	RETURN_TOKEN(T_COMMENT);
1917}
1918
1919<ST_IN_SCRIPTING>"/*"|"/**"{WHITESPACE} {
1920	int doc_com;
1921
1922	if (yyleng > 2) {
1923		doc_com = 1;
1924		RESET_DOC_COMMENT();
1925	} else {
1926		doc_com = 0;
1927	}
1928
1929	while (YYCURSOR < YYLIMIT) {
1930		if (*YYCURSOR++ == '*' && *YYCURSOR == '/') {
1931			break;
1932		}
1933	}
1934
1935	if (YYCURSOR < YYLIMIT) {
1936		YYCURSOR++;
1937	} else {
1938		zend_error(E_COMPILE_WARNING, "Unterminated comment starting line %d", CG(zend_lineno));
1939	}
1940
1941	yyleng = YYCURSOR - SCNG(yy_text);
1942	HANDLE_NEWLINES(yytext, yyleng);
1943
1944	if (doc_com) {
1945		CG(doc_comment) = zend_string_init(yytext, yyleng, 0);
1946		RETURN_TOKEN(T_DOC_COMMENT);
1947	}
1948
1949	RETURN_TOKEN(T_COMMENT);
1950}
1951
1952<ST_IN_SCRIPTING>"?>"{NEWLINE}? {
1953	BEGIN(INITIAL);
1954	RETURN_TOKEN(T_CLOSE_TAG);  /* implicit ';' at php-end tag */
1955}
1956
1957
1958<ST_IN_SCRIPTING>b?['] {
1959	register char *s, *t;
1960	char *end;
1961	int bprefix = (yytext[0] != '\'') ? 1 : 0;
1962
1963	while (1) {
1964		if (YYCURSOR < YYLIMIT) {
1965			if (*YYCURSOR == '\'') {
1966				YYCURSOR++;
1967				yyleng = YYCURSOR - SCNG(yy_text);
1968
1969				break;
1970			} else if (*YYCURSOR++ == '\\' && YYCURSOR < YYLIMIT) {
1971				YYCURSOR++;
1972			}
1973		} else {
1974			yyleng = YYLIMIT - SCNG(yy_text);
1975
1976			/* Unclosed single quotes; treat similar to double quotes, but without a separate token
1977			 * for ' (unrecognized by parser), instead of old flex fallback to "Unexpected character..."
1978			 * rule, which continued in ST_IN_SCRIPTING state after the quote */
1979			ZVAL_NULL(zendlval);
1980			RETURN_TOKEN(T_ENCAPSED_AND_WHITESPACE);
1981		}
1982	}
1983
1984	ZVAL_STRINGL(zendlval, yytext+bprefix+1, yyleng-bprefix-2);
1985
1986	/* convert escape sequences */
1987	s = t = Z_STRVAL_P(zendlval);
1988	end = s+Z_STRLEN_P(zendlval);
1989	while (s<end) {
1990		if (*s=='\\') {
1991			s++;
1992
1993			switch(*s) {
1994				case '\\':
1995				case '\'':
1996					*t++ = *s;
1997					Z_STRLEN_P(zendlval)--;
1998					break;
1999				default:
2000					*t++ = '\\';
2001					*t++ = *s;
2002					break;
2003			}
2004		} else {
2005			*t++ = *s;
2006		}
2007
2008		if (*s == '\n' || (*s == '\r' && (*(s+1) != '\n'))) {
2009			CG(zend_lineno)++;
2010		}
2011		s++;
2012	}
2013	*t = 0;
2014
2015	if (SCNG(output_filter)) {
2016		size_t sz = 0;
2017		char *str = NULL;
2018		s = Z_STRVAL_P(zendlval);
2019		// TODO: avoid reallocation ???
2020		SCNG(output_filter)((unsigned char **)&str, &sz, (unsigned char *)s, (size_t)Z_STRLEN_P(zendlval));
2021		ZVAL_STRINGL(zendlval, str, sz);
2022	}
2023	RETURN_TOKEN(T_CONSTANT_ENCAPSED_STRING);
2024}
2025
2026
2027<ST_IN_SCRIPTING>b?["] {
2028	int bprefix = (yytext[0] != '"') ? 1 : 0;
2029
2030	while (YYCURSOR < YYLIMIT) {
2031		switch (*YYCURSOR++) {
2032			case '"':
2033				yyleng = YYCURSOR - SCNG(yy_text);
2034				zend_scan_escape_string(zendlval, yytext+bprefix+1, yyleng-bprefix-2, '"');
2035				RETURN_TOKEN(T_CONSTANT_ENCAPSED_STRING);
2036			case '$':
2037				if (IS_LABEL_START(*YYCURSOR) || *YYCURSOR == '{') {
2038					break;
2039				}
2040				continue;
2041			case '{':
2042				if (*YYCURSOR == '$') {
2043					break;
2044				}
2045				continue;
2046			case '\\':
2047				if (YYCURSOR < YYLIMIT) {
2048					YYCURSOR++;
2049				}
2050				/* fall through */
2051			default:
2052				continue;
2053		}
2054
2055		YYCURSOR--;
2056		break;
2057	}
2058
2059	/* Remember how much was scanned to save rescanning */
2060	SET_DOUBLE_QUOTES_SCANNED_LENGTH(YYCURSOR - SCNG(yy_text) - yyleng);
2061
2062	YYCURSOR = SCNG(yy_text) + yyleng;
2063
2064	BEGIN(ST_DOUBLE_QUOTES);
2065	RETURN_TOKEN('"');
2066}
2067
2068
2069<ST_IN_SCRIPTING>b?"<<<"{TABS_AND_SPACES}({LABEL}|([']{LABEL}['])|(["]{LABEL}["])){NEWLINE} {
2070	char *s;
2071	int bprefix = (yytext[0] != '<') ? 1 : 0;
2072	zend_heredoc_label *heredoc_label = emalloc(sizeof(zend_heredoc_label));
2073
2074	CG(zend_lineno)++;
2075	heredoc_label->length = yyleng-bprefix-3-1-(yytext[yyleng-2]=='\r'?1:0);
2076	s = yytext+bprefix+3;
2077	while ((*s == ' ') || (*s == '\t')) {
2078		s++;
2079		heredoc_label->length--;
2080	}
2081
2082	if (*s == '\'') {
2083		s++;
2084		heredoc_label->length -= 2;
2085
2086		BEGIN(ST_NOWDOC);
2087	} else {
2088		if (*s == '"') {
2089			s++;
2090			heredoc_label->length -= 2;
2091		}
2092
2093		BEGIN(ST_HEREDOC);
2094	}
2095
2096	heredoc_label->label = estrndup(s, heredoc_label->length);
2097
2098	/* Check for ending label on the next line */
2099	if (heredoc_label->length < YYLIMIT - YYCURSOR && !memcmp(YYCURSOR, s, heredoc_label->length)) {
2100		YYCTYPE *end = YYCURSOR + heredoc_label->length;
2101
2102		if (*end == ';') {
2103			end++;
2104		}
2105
2106		if (*end == '\n' || *end == '\r') {
2107			BEGIN(ST_END_HEREDOC);
2108		}
2109	}
2110
2111	zend_ptr_stack_push(&SCNG(heredoc_label_stack), (void *) heredoc_label);
2112
2113	RETURN_TOKEN(T_START_HEREDOC);
2114}
2115
2116
2117<ST_IN_SCRIPTING>[`] {
2118	BEGIN(ST_BACKQUOTE);
2119	RETURN_TOKEN('`');
2120}
2121
2122
2123<ST_END_HEREDOC>{ANY_CHAR} {
2124	zend_heredoc_label *heredoc_label = zend_ptr_stack_pop(&SCNG(heredoc_label_stack));
2125
2126	YYCURSOR += heredoc_label->length - 1;
2127	yyleng = heredoc_label->length;
2128
2129	heredoc_label_dtor(heredoc_label);
2130	efree(heredoc_label);
2131
2132	BEGIN(ST_IN_SCRIPTING);
2133	RETURN_TOKEN(T_END_HEREDOC);
2134}
2135
2136
2137<ST_DOUBLE_QUOTES,ST_BACKQUOTE,ST_HEREDOC>"{$" {
2138	Z_LVAL_P(zendlval) = (zend_long) '{';
2139	yy_push_state(ST_IN_SCRIPTING);
2140	yyless(1);
2141	RETURN_TOKEN(T_CURLY_OPEN);
2142}
2143
2144
2145<ST_DOUBLE_QUOTES>["] {
2146	BEGIN(ST_IN_SCRIPTING);
2147	RETURN_TOKEN('"');
2148}
2149
2150<ST_BACKQUOTE>[`] {
2151	BEGIN(ST_IN_SCRIPTING);
2152	RETURN_TOKEN('`');
2153}
2154
2155
2156<ST_DOUBLE_QUOTES>{ANY_CHAR} {
2157	if (GET_DOUBLE_QUOTES_SCANNED_LENGTH()) {
2158		YYCURSOR += GET_DOUBLE_QUOTES_SCANNED_LENGTH() - 1;
2159		SET_DOUBLE_QUOTES_SCANNED_LENGTH(0);
2160
2161		goto double_quotes_scan_done;
2162	}
2163
2164	if (YYCURSOR > YYLIMIT) {
2165		RETURN_TOKEN(END);
2166	}
2167	if (yytext[0] == '\\' && YYCURSOR < YYLIMIT) {
2168		YYCURSOR++;
2169	}
2170
2171	while (YYCURSOR < YYLIMIT) {
2172		switch (*YYCURSOR++) {
2173			case '"':
2174				break;
2175			case '$':
2176				if (IS_LABEL_START(*YYCURSOR) || *YYCURSOR == '{') {
2177					break;
2178				}
2179				continue;
2180			case '{':
2181				if (*YYCURSOR == '$') {
2182					break;
2183				}
2184				continue;
2185			case '\\':
2186				if (YYCURSOR < YYLIMIT) {
2187					YYCURSOR++;
2188				}
2189				/* fall through */
2190			default:
2191				continue;
2192		}
2193
2194		YYCURSOR--;
2195		break;
2196	}
2197
2198double_quotes_scan_done:
2199	yyleng = YYCURSOR - SCNG(yy_text);
2200
2201	zend_scan_escape_string(zendlval, yytext, yyleng, '"');
2202	RETURN_TOKEN(T_ENCAPSED_AND_WHITESPACE);
2203}
2204
2205
2206<ST_BACKQUOTE>{ANY_CHAR} {
2207	if (YYCURSOR > YYLIMIT) {
2208		RETURN_TOKEN(END);
2209	}
2210	if (yytext[0] == '\\' && YYCURSOR < YYLIMIT) {
2211		YYCURSOR++;
2212	}
2213
2214	while (YYCURSOR < YYLIMIT) {
2215		switch (*YYCURSOR++) {
2216			case '`':
2217				break;
2218			case '$':
2219				if (IS_LABEL_START(*YYCURSOR) || *YYCURSOR == '{') {
2220					break;
2221				}
2222				continue;
2223			case '{':
2224				if (*YYCURSOR == '$') {
2225					break;
2226				}
2227				continue;
2228			case '\\':
2229				if (YYCURSOR < YYLIMIT) {
2230					YYCURSOR++;
2231				}
2232				/* fall through */
2233			default:
2234				continue;
2235		}
2236
2237		YYCURSOR--;
2238		break;
2239	}
2240
2241	yyleng = YYCURSOR - SCNG(yy_text);
2242
2243	zend_scan_escape_string(zendlval, yytext, yyleng, '`');
2244	RETURN_TOKEN(T_ENCAPSED_AND_WHITESPACE);
2245}
2246
2247
2248<ST_HEREDOC>{ANY_CHAR} {
2249	int newline = 0;
2250
2251	zend_heredoc_label *heredoc_label = zend_ptr_stack_top(&SCNG(heredoc_label_stack));
2252
2253	if (YYCURSOR > YYLIMIT) {
2254		RETURN_TOKEN(END);
2255	}
2256
2257	YYCURSOR--;
2258
2259	while (YYCURSOR < YYLIMIT) {
2260		switch (*YYCURSOR++) {
2261			case '\r':
2262				if (*YYCURSOR == '\n') {
2263					YYCURSOR++;
2264				}
2265				/* fall through */
2266			case '\n':
2267				/* Check for ending label on the next line */
2268				if (IS_LABEL_START(*YYCURSOR) && heredoc_label->length < YYLIMIT - YYCURSOR && !memcmp(YYCURSOR, heredoc_label->label, heredoc_label->length)) {
2269					YYCTYPE *end = YYCURSOR + heredoc_label->length;
2270
2271					if (*end == ';') {
2272						end++;
2273					}
2274
2275					if (*end == '\n' || *end == '\r') {
2276						/* newline before label will be subtracted from returned text, but
2277						 * yyleng/yytext will include it, for zend_highlight/strip, tokenizer, etc. */
2278						if (YYCURSOR[-2] == '\r' && YYCURSOR[-1] == '\n') {
2279							newline = 2; /* Windows newline */
2280						} else {
2281							newline = 1;
2282						}
2283
2284						CG(increment_lineno) = 1; /* For newline before label */
2285						BEGIN(ST_END_HEREDOC);
2286
2287						goto heredoc_scan_done;
2288					}
2289				}
2290				continue;
2291			case '$':
2292				if (IS_LABEL_START(*YYCURSOR) || *YYCURSOR == '{') {
2293					break;
2294				}
2295				continue;
2296			case '{':
2297				if (*YYCURSOR == '$') {
2298					break;
2299				}
2300				continue;
2301			case '\\':
2302				if (YYCURSOR < YYLIMIT && *YYCURSOR != '\n' && *YYCURSOR != '\r') {
2303					YYCURSOR++;
2304				}
2305				/* fall through */
2306			default:
2307				continue;
2308		}
2309
2310		YYCURSOR--;
2311		break;
2312	}
2313
2314heredoc_scan_done:
2315	yyleng = YYCURSOR - SCNG(yy_text);
2316
2317	zend_scan_escape_string(zendlval, yytext, yyleng - newline, 0);
2318	RETURN_TOKEN(T_ENCAPSED_AND_WHITESPACE);
2319}
2320
2321
2322<ST_NOWDOC>{ANY_CHAR} {
2323	int newline = 0;
2324
2325	zend_heredoc_label *heredoc_label = zend_ptr_stack_top(&SCNG(heredoc_label_stack));
2326
2327	if (YYCURSOR > YYLIMIT) {
2328		RETURN_TOKEN(END);
2329	}
2330
2331	YYCURSOR--;
2332
2333	while (YYCURSOR < YYLIMIT) {
2334		switch (*YYCURSOR++) {
2335			case '\r':
2336				if (*YYCURSOR == '\n') {
2337					YYCURSOR++;
2338				}
2339				/* fall through */
2340			case '\n':
2341				/* Check for ending label on the next line */
2342				if (IS_LABEL_START(*YYCURSOR) && heredoc_label->length < YYLIMIT - YYCURSOR && !memcmp(YYCURSOR, heredoc_label->label, heredoc_label->length)) {
2343					YYCTYPE *end = YYCURSOR + heredoc_label->length;
2344
2345					if (*end == ';') {
2346						end++;
2347					}
2348
2349					if (*end == '\n' || *end == '\r') {
2350						/* newline before label will be subtracted from returned text, but
2351						 * yyleng/yytext will include it, for zend_highlight/strip, tokenizer, etc. */
2352						if (YYCURSOR[-2] == '\r' && YYCURSOR[-1] == '\n') {
2353							newline = 2; /* Windows newline */
2354						} else {
2355							newline = 1;
2356						}
2357
2358						CG(increment_lineno) = 1; /* For newline before label */
2359						BEGIN(ST_END_HEREDOC);
2360
2361						goto nowdoc_scan_done;
2362					}
2363				}
2364				/* fall through */
2365			default:
2366				continue;
2367		}
2368	}
2369
2370nowdoc_scan_done:
2371	yyleng = YYCURSOR - SCNG(yy_text);
2372
2373	zend_copy_value(zendlval, yytext, yyleng - newline);
2374	HANDLE_NEWLINES(yytext, yyleng - newline);
2375	RETURN_TOKEN(T_ENCAPSED_AND_WHITESPACE);
2376}
2377
2378
2379<ST_IN_SCRIPTING,ST_VAR_OFFSET>{ANY_CHAR} {
2380	if (YYCURSOR > YYLIMIT) {
2381		RETURN_TOKEN(END);
2382	}
2383
2384	zend_error(E_COMPILE_WARNING,"Unexpected character in input:  '%c' (ASCII=%d) state=%d", yytext[0], yytext[0], YYSTATE);
2385	goto restart;
2386}
2387
2388*/
2389}
2390