Navigation

← Previous Changeset
Next Changeset →

Changeset 581

Timestamp:

07/08/08 02:52:22 (5 months ago)

Author:

moenicke

Message:

* ported string and character code to the ICU unicode library

Files:

trunk/rphp/compiler/parser/phplexer.cpp (modified) (24 diffs)
trunk/rphp/compiler/parser/phplexer.h (modified) (4 diffs)

Legend:

: Unmodified
: Added
: Removed
: Modified
: Copied
: Moved

trunk/rphp/compiler/parser/phplexer.cpp

r580	r581
21	21	#include "phplexer.h"
22	22
23		#include "~~php~~parser.h"
	23	#include "rphp_parser.h"
24	24
25	25	#include <kdev-pg-location-table.h>
26	26	#include <kdev-pg-token-stream.h>
27	27
28		~~#include <QtCore/QString>~~
29		~~#include <QtCore/QStringList>~~
30		~~#include <QtCore/QRegExp>~~
31		~~#include <QtCore/QDebug>~~
32
33	28	#include <iostream>
34
35		namespace Php
36		{
37
38		Lexer::Lexer( Parser* _parser, const QString& content ):
	29	#include <string>
	30	#include "unicode/utypes.h"
	31
	32	// TODO check cursor position after returning from a method and if they change the former state, or not!
	33	// TODO pos and m_curpos the same?
	34
	35	namespace rphp
	36	{
	37
	38	Lexer::Lexer( parser* _parser, const UnicodeString& content ):
39	39	m_content( content ), m_parser( _parser ),
40		m_curpos( 0 ), m_contentSize( m_content.~~size~~() ),
	40	m_curpos( 0 ), m_contentSize( m_content.length() ),
41	41	m_tokenBegin( 0 ), m_tokenEnd( 0 ), m_haltCompiler( 0 )
42	42	{
…	…
47	47	int Lexer::state(int deepness) const
48	48	{
49		return m_state.at(m_state.size() - deepness - 1);
	49	#ifdef PENDING_THOMAS
	50	// TODO pending we need to look at other positions than top
	51	return m_state[ m_state.size() - deepness - 1 ];
	52	#endif
50	53	}
51	54	void Lexer::printState()
…	…
53	56	int s = state();
54	57	if (s == ErrorState)
55		~~qDebug()~~ << "ErrorState";
	58	std::cout << "ErrorState";
56	59	else if (s == HtmlState)
57		~~qDebug()~~ << "HtmlState";
	60	std::cout << "HtmlState";
58	61	else if (s == DefaultState)
59		~~qDebug()~~ << "DefaultState";
	62	std::cout << "DefaultState";
60	63	else if (s == String)
61		~~qDebug()~~ << "String";
	64	std::cout << "String";
62	65	else if (s == StringVariable)
63		~~qDebug()~~ << "StringVariable";
	66	std::cout << "StringVariable";
64	67	else if (s == StringVariableBracket)
65		~~qDebug()~~ << "StringVariableBracket";
	68	std::cout << "StringVariableBracket";
66	69	else if (s == StringVariableObjectOperator)
67		~~qDebug()~~ << "StringVariableObjectOperator";
	70	std::cout << "StringVariableObjectOperator";
68	71	else if (s == StringVariableCurly)
69		~~qDebug()~~ << "StringVariableCurly";
	72	std::cout << "StringVariableCurly";
70	73	else if (s == StringVarname)
71		~~qDebug()~~ << "StringVarname";
	74	std::cout << "StringVarname";
72	75	else if (s == StringHeredoc)
73		~~qDebug()~~ << "StringHeredoc";
	76	std::cout << "StringHeredoc";
74	77	else if (s == StringBacktick)
75		~~qDebug()~~ << "StringBacktick";
	78	std::cout << "StringBacktick";
76	79	}
77	80
…	…
88	91	int Lexer::nextTokenKind()
89	92	{
90		int token = Parser::Token_INVALID;
	93	int token = parser::Token_INVALID;
91	94	if ( m_curpos >= m_contentSize )
92	95	{
…	…
95	98	return 0;
96	99	}
97		QChar* it = m_content.data();
98		it += m_curpos;
	100	int pos = m_curpos;
	101	#ifdef THOMAS_TEMP_DISABLED
	102	// TODO temp. disabled code, tokenBegin was not in use
99	103	m_tokenBegin = m_curpos;
	104	#endif
100	105	switch ( state() )
101	106	{
102	107	case HtmlState:
103		if (it->unicode() == '<' && (it+1)->unicode() == '?') {
104		token = Parser::Token_OPEN_TAG;
105		if ((it+2)->unicode() == '=')
106		{
107		token = Parser::Token_OPEN_TAG_WITH_ECHO;
108		m_curpos++;
109		it++;
110		}
111		else if ((it+2)->toLower().unicode() == 'p'
112		&& (it+3)->toLower().unicode() == 'h'
113		&& (it+4)->toLower().unicode() == 'p'
114		&& (it+5)->isSpace())
	108	if (lookAt( pos ) == '<' && lookAt( pos + 1 ) == '?') {
	109	token = parser::Token_OPEN_TAG;
	110	if ( lookAt( pos + 2 ) == '=')
	111	{
	112	token = parser::Token_OPEN_TAG_WITH_ECHO;
	113	m_curpos++;
	114	pos++;
	115	}
	116	// TODO toLower, isSpace
	117	else if (
	118	(lookAt( pos + 2 ) == 'p' \|\| lookAt( pos + 2 ) == 'P')
	119	&& (lookAt( pos + 3 ) == 'h' \|\| lookAt( pos + 3 ) == 'H')
	120	&& (lookAt( pos + 4 ) == 'p' \|\| lookAt( pos + 4 ) == 'P')
	121	&& lookAt( pos + 5 ) == ' ')
115	122	{
116	123	m_curpos += 4;
…	…
119	126	pushState(DefaultState);
120	127	} else {
121		token = Parser::Token_INLINE_HTML;
	128	token = parser::Token_INLINE_HTML;
122	129	while( m_curpos < m_contentSize )
123	130	{
124		if (~~it->unicode(~~) == '\n') createNewline(m_curpos);
125		if (~~(it+1)->unicode() == '<' && (it+2)->unicode(~~) == '?') {
	131	if (lookAt( pos ) == '\n') createNewline(m_curpos);
	132	if ( lookAt( pos + 1 ) == '<' && lookAt( pos + 2 ) == '?') {
126	133	break;
127	134	}
128		it++;
	135	pos++;
129	136	m_curpos++;
130	137	}
…	…
134	141	case StringVariableCurly:
135	142	{
136		if (it->isSpace())
137		{
138		token = Parser::Token_WHITESPACE;
139		while (m_curpos < m_contentSize && it->isSpace()) {
140		if (it->unicode() == '\n') createNewline(m_curpos);
141		it++;
	143	// isSpace
	144	if ( lookAt( pos ) == ' ')
	145	{
	146	token = parser::Token_WHITESPACE;
	147	while (m_curpos < m_contentSize && lookAt( pos ) == ' ') {
	148	if ( lookAt( pos ) == '\n') createNewline(m_curpos);
	149	pos; // weiterspringen!!!!
142	150	m_curpos++;
143	151	}
144	152	m_curpos--;
145	153	}
146		else if (it->isDigit() \|\| (it->unicode() == '.' && (it+1)->isDigit()))
147		{
148		QString num;bool hasPoint = false;
	154	else if (u_isdigit( lookAt( pos ) )
	155	\|\| ( lookAt( pos ) == '.' && u_isdigit( lookAt( pos + 1 ) ) ))
	156	{
	157	UnicodeString num;bool hasPoint = false;
149	158	bool hex = false;
150		if (~~it->unicode() == '0' && (it+1)->unicode(~~) == 'x') {
151		it += 2;
	159	if ( lookAt( pos ) == '0' && ( lookAt( pos + 1 )) == 'x') {
	160	pos += 2;
152	161	m_curpos += 2;
153	162	hex = true;
154	163	}
155	164	while (m_curpos < m_contentSize && (
156		~~it->isDigit(~~)
157		\|\| (!hex && !hasPoint && ~~it->unicode(~~) == '.')
158		\|\| (hex && (~~it->toLower() == 'a' \|\| it->toLower(~~) == 'b' \|\|
159		~~it->toLower() == 'c' \|\| it->toLower(~~) == 'd' \|\|
160		~~it->toLower() == 'e' \|\| it->toLower(~~) == 'f'))))
161		{
162		if (~~it->unicode(~~) == '.') hasPoint = true;
163		num.append(*it);
164		it++;
165		m_curpos++;
166		}
167		if (!hex && ~~it->toLower(~~) == 'e' &&
168		(~~(it+1)->isDigit(~~) \|\|
169		((~~(it+1)->unicode() == '-' \|\| (it+1)->unicode() == '+') && (it+2)->isDigit()~~)))
	165	u_isdigit( lookAt( pos ) )
	166	\|\| (!hex && !hasPoint && lookAt( pos ) == '.')
	167	\|\| (hex && (u_tolower( lookAt( pos ) ) == 'a' \|\| u_tolower( lookAt( pos ) ) == 'b' \|\|
	168	u_tolower( lookAt( pos ) ) == 'c' \|\| u_tolower( lookAt( pos ) ) == 'd' \|\|
	169	u_tolower( lookAt( pos ) ) == 'e' \|\| u_tolower( lookAt( pos ) ) == 'f'))))
	170	{
	171	if (lookAt( pos ) == '.') hasPoint = true;
	172	num.append(lookAt( pos ));
	173	pos++;
	174	m_curpos++;
	175	}
	176	if (!hex && u_tolower( lookAt( pos ) ) == 'e' &&
	177	(u_isdigit( lookAt( pos ) ) \|\|
	178	((lookAt( pos + 1 ) == '-' \|\| lookAt( pos + 1 ) == '+') && u_isdigit( lookAt( pos + 2 ) ) )))
170	179	{
171	180	//exponential number
172		token = Parser::Token_DNUMBER;
173		m_curpos++;
174		it++;
175		if (~~it->unicode() == '-' \|\| it->unicode(~~) == '+') {
176		it++;
	181	token = parser::Token_DNUMBER;
	182	m_curpos++;
	183	pos++;
	184	if (lookAt( pos ) == '-' \|\| lookAt( pos ) == '+') {
	185	pos++;
177	186	m_curpos++;
178	187	}
179		while (m_curpos < m_contentSize && (~~it->isDigit()~~)) {
180		it++;
	188	while (m_curpos < m_contentSize && ( u_isdigit( lookAt( pos ) ) )) {
	189	pos++;
181	190	m_curpos++;
182	191	}
…	…
185	194	m_curpos--;
186	195	if (hasPoint) {
187		token = Parser::Token_DNUMBER;
	196	token = parser::Token_DNUMBER;
188	197	} else {
	198	#ifdef THOMAS_TEMP_DISABLED
	199	// TODO temp. disabled code, toLong()
189	200	bool ok;
190	201	//check if string can be converted to long
…	…
192	203	num.toLong(&ok, hex ? 16 : 10);
193	204	if (ok) {
194		token = Parser::Token_LNUMBER;
	205	token = parser::Token_LNUMBER;
195	206	} else {
196		token = Parser::Token_DNUMBER;
	207	token = parser::Token_DNUMBER;
197	208	}
198		}
199		}
200
201		}
202		else if (processVariable(it))
203		{
204		token = Parser::Token_VARIABLE;
205		}
206		else if (it->unicode() == '$')
207		{
208		//when it was not recognized as variable
209		token = Parser::Token_DOLLAR;
210		}
211		else if (it->unicode() == '}')
212		{
213		token = Parser::Token_RBRACE;
	209	#endif
	210	}
	211	}
	212
	213	}
	214	else if (processVariable( lookAt( pos ) ))
	215	{
	216	token = parser::Token_VARIABLE;
	217	}
	218	else if (lookAt( pos ) == '$')
	219	{
	220	//when *it was not recognized as variable
	221	token = parser::Token_DOLLAR;
	222	}
	223	else if (lookAt( pos ) == '}')
	224	{
	225	token = parser::Token_RBRACE;
214	226	if (state() == StringVariableCurly) {
215	227	popState();
216	228	}
217	229	}
218		else if (~~it->unicode(~~) == '{')
219		{
220		token = Parser::Token_LBRACE;
	230	else if (lookAt( pos ) == '{')
	231	{
	232	token = parser::Token_LBRACE;
221	233	if (state() == StringVariableCurly) {
222	234	pushState(StringVariableCurly);
223	235	}
224	236	}
225		else if (~~it->unicode(~~) == ')')
226		{
227		token = Parser::Token_RPAREN;
228		}
229		else if (~~it->unicode(~~) == '(')
230		{
231		it++;
232		int pos = m_curpos + 1;
233		while (~~pos < m_contentSize && it->isSpace()~~)
234		{
235		it++;
236		pos++;
237		}
238		QString name;
239		while (~~pos < m_contentSize && it->isLetter(~~))
240		{
241		name.append(*it);
242		it++;
243		pos++;
244		}
245		while (~~pos < m_contentSize && it->isSpace()~~)
246		{
247		it++;
248		pos++;
	237	else if (lookAt( pos ) == ')')
	238	{
	239	token = parser::Token_RPAREN;
	240	}
	241	else if (lookAt( pos ) == '(')
	242	{
	243	pos++;
	244	int _pos = m_curpos + 1;
	245	while (_pos < m_contentSize && lookAt( pos ) == ' ')
	246	{
	247	pos++;
	248	_pos++;
	249	}
	250	UnicodeString name;
	251	while (_pos < m_contentSize && u_isalnum( lookAt( pos ) ))
	252	{
	253	name.append( lookAt( pos ) );
	254	pos++;
	255	_pos++;
	256	}
	257	while (_pos < m_contentSize && lookAt( pos ) == ' ')
	258	{
	259	pos++;
	260	_pos++;
249	261	}
250	262	name = name.toLower();
251		if (~~it->unicode(~~) == ')')
	263	if (lookAt( pos ) == ')')
252	264	{
253	265	if (name == "int" \|\| name == "integer")
254	266	{
255		token = Parser::Token_INT_CAST;
	267	token = parser::Token_INT_CAST;
256	268	}
257	269	else if (name == "real" \|\| name == "double" \|\| name == "float")
258	270	{
259		token = Parser::Token_DOUBLE_CAST;
	271	token = parser::Token_DOUBLE_CAST;
260	272	}
261	273	else if (name == "string")
262	274	{
263		token = Parser::Token_STRING_CAST;
	275	token = parser::Token_STRING_CAST;
264	276	}
265	277	else if (name == "binary")
266	278	{
267	279	//as in php
268		token = Parser::Token_STRING_CAST;
	280	token = parser::Token_STRING_CAST;
269	281	}
270	282	else if (name == "array")
271	283	{
272		token = Parser::Token_ARRAY_CAST;
	284	token = parser::Token_ARRAY_CAST;
273	285	}
274	286	else if (name == "object")
275	287	{
276		token = Parser::Token_OBJECT_CAST;
	288	token = parser::Token_OBJECT_CAST;
277	289	}
278	290	else if (name == "bool" \|\| name == "boolean")
279	291	{
280		token = Parser::Token_BOOL_CAST;
	292	token = parser::Token_BOOL_CAST;
281	293	}
282	294	else if (name == "unset")
283	295	{
284		token = Parser::Token_UNSET_CAST;
	296	token = parser::Token_UNSET_CAST;
285	297	}
286	298	else
287	299	{
288		token = Parser::Token_LPAREN;
289		}
290
291		if (token != Parser::Token_LPAREN)
292		{
293		m_curpos = pos;
	300	token = parser::Token_LPAREN;
	301	}
	302
	303	if (token != parser::Token_LPAREN)
	304	{
	305	m_curpos = _pos;
294	306	}
295	307	}
296	308	else
297	309	{
298		token = Parser::Token_LPAREN;
299		}
300		}
301		else if (~~it->unicode(~~) == ']')
302		{
303		token = Parser::Token_RBRACKET;
304		}
305		else if (~~it->unicode(~~) == '[')
306		{
307		token = Parser::Token_LBRACKET;
308		}
309		else if (~~it->unicode(~~) == ',')
310		{
311		token = Parser::Token_COMMA;
312		}
313		else if (~~it->unicode(~~) == '@')
314		{
315		token = Parser::Token_AT;
316		}
317		else if (~~it->unicode(~~) == '!')
318		{
319		if (~~(it+1)->unicode(~~) == '=')
320		{
321		m_curpos++;
322		if (~~(it+2)->unicode(~~) == '=')
	310	token = parser::Token_LPAREN;
	311	}
	312	}
	313	else if (lookAt( pos ) == ']')
	314	{
	315	token = parser::Token_RBRACKET;
	316	}
	317	else if (lookAt( pos ) == '[')
	318	{
	319	token = parser::Token_LBRACKET;
	320	}
	321	else if (lookAt( pos ) == ',')
	322	{
	323	token = parser::Token_COMMA;
	324	}
	325	else if (lookAt( pos ) == '@')
	326	{
	327	token = parser::Token_AT;
	328	}
	329	else if (lookAt( pos ) == '!')
	330	{
	331	if (lookAt( pos + 1 ) == '=')
	332	{
	333	m_curpos++;
	334	if (lookAt( pos + 2 ) == '=')
323	335	{
324	336	m_curpos++;
325		token = Parser::Token_IS_NOT_IDENTICAL;
	337	token = parser::Token_IS_NOT_IDENTICAL;
326	338	}
327	339	else
328	340	{
329		token = Parser::Token_IS_NOT_EQUAL;
	341	token = parser::Token_IS_NOT_EQUAL;
330	342	}
331	343	}
332	344	else
333	345	{
334		token = Parser::Token_BANG;
335		}
336		}
337		else if (~~it->unicode(~~) == '<')
338		{
339		if (~~(it+1)->unicode(~~) == '<')
340		{
341		m_curpos++;
342		if (~~(it+2)->unicode(~~) == '<' && state() != StringVariableCurly)
	346	token = parser::Token_BANG;
	347	}
	348	}
	349	else if (lookAt( pos ) == '<')
	350	{
	351	if (lookAt( pos + 1 ) == '<')
	352	{
	353	m_curpos++;
	354	if (lookAt( pos + 2 ) == '<' && state() != StringVariableCurly)
343	355	{
344	356	//HEREDOC string (<<< EOD\nfoo\nEOD;\n)
345		int pos = 3;
346		while (m_curpos+pos < m_contentSize &&
347		(~~(it+pos)->unicode() == ' ' \|\| (it+pos)->unicode(~~) == '\t'))
	357	int _pos = 3;
	358	while (m_curpos+_pos < m_contentSize &&
	359	( u_isalnum( lookAt( pos + _pos ) ) == ' ' \|\| u_isalnum( lookAt( pos + _pos ) ) == '\t'))
348	360	{
349		pos++;
	361	_pos++;
350	362	}
351		if (~~(it+pos)->isLetter() \|\| (it+pos)->unicode(~~) == '_') //identifier must start with a letter
	363	if ( u_isalnum( lookAt( pos + _pos ) ) \|\| lookAt( pos + _pos ) == '_') //identifier must start with a letter
352	364	{
353		m_heredocIdentifier.~~clear~~();
354		while (m_curpos+pos < m_contentSize &&
355		(~~(it+pos)->isDigit() \|\| (it+pos)->isLetter() \|\| (it+pos)->unicode(~~) == '_'))
	365	m_heredocIdentifier.remove();
	366	while (m_curpos+_pos < m_contentSize &&
	367	( u_isdigit( lookAt( pos + _pos ) ) \|\| u_isalnum( lookAt( pos + _pos ) ) \|\| lookAt( pos + _pos ) == '_'))
356	368	{
357		m_heredocIdentifier.append(*(it+pos));
358		pos++;
	369	m_heredocIdentifier.append( lookAt( pos + _pos ) );
	370	_pos++;
359	371	}
360		if (~~(it+pos)->unicode(~~) == '\n') {
	372	if ( lookAt( pos + _pos ) == '\n') {
361	373	//identifier must be followed by newline, newline is part of HEREDOC token
362		token = Parser::Token_START_HEREDOC;
	374	token = parser::Token_START_HEREDOC;
363	375	pushState(StringHeredoc);
364		m_curpos += pos-1;
	376	m_curpos += _pos-1;
365	377	}
366	378	}
367	379	}
368	380
369		if (token != Parser::Token_START_HEREDOC)
370		{
371		if (~~(it+2)->unicode(~~) == '=')
	381	if (token != parser::Token_START_HEREDOC)
	382	{
	383	if (lookAt( pos + 2 ) == '=')
372	384	{
373	385	m_curpos++;
374		token = Parser::Token_SL_ASSIGN;
	386	token = parser::Token_SL_ASSIGN;
375	387	}
376	388	else
377	389	{
378		token = Parser::Token_SL;
	390	token = parser::Token_SL;
379	391	}
380	392	}
381	393	}
382		else if (~~(it+1)->unicode(~~) == '=')
383		{
384		m_curpos++;
385		token = Parser::Token_IS_SMALLER_OR_EQUAL;
386		}
387		else if (~~(it+1)->unicode(~~) == '>')
388		{
389		m_curpos++;
390		token = Parser::Token_IS_NOT_EQUAL;
	394	else if (lookAt( pos + 1 ) == '=')
	395	{
	396	m_curpos++;
	397	token = parser::Token_IS_SMALLER_OR_EQUAL;
	398	}
	399	else if (lookAt( pos + 1 ) == '>')
	400	{
	401	m_curpos++;
	402	token = parser::Token_IS_NOT_EQUAL;
391	403	}
392	404	else
393	405	{
394		token = Parser::Token_IS_SMALLER;
395		}
396		}
397		else if (~~it->unicode(~~) == '>')
398		{
399		if (~~(it+1)->unicode(~~) == '>')
400		{
401		m_curpos++;
402		if (~~(it+2)->unicode(~~) == '=')
	406	token = parser::Token_IS_SMALLER;
	407	}
	408	}
	409	else if (lookAt( pos ) == '>')
	410	{
	411	if (lookAt( pos + 1 ) == '>')
	412	{
	413	m_curpos++;
	414	if (lookAt( pos + 2 ) == '=')
403	415	{
404	416	m_curpos++;
405		token = Parser::Token_SR_ASSIGN;
	417	token = parser::Token_SR_ASSIGN;
406	418	}
407	419	else
408	420	{
409		token = Parser::Token_SR;
410		}
411		}
412		else if (~~(it+1)->unicode(~~) == '=')
413		{
414		m_curpos++;
415		token = Parser::Token_IS_GREATER_OR_EQUAL;
	421	token = parser::Token_SR;
	422	}
	423	}
	424	else if (lookAt( pos + 1 ) == '=')
	425	{
	426	m_curpos++;
	427	token = parser::Token_IS_GREATER_OR_EQUAL;
416	428	}
417	429	else
418	430	{
419		token = Parser::Token_IS_GREATER;
420		}
421		}
422		else if (~~it->unicode(~~) == '~')
423		{
424		token = Parser::Token_TILDE;
425		}
426		else if (~~it->unicode(~~) == ':')
427		{
428		if (~~(it+1)->unicode(~~) == ':') {
429		m_curpos++;
430		token = Parser::Token_PAAMAYIM_NEKUDOTAYIM;
431		} else {
432		token = Parser::Token_COLON;
433		}
434		}
435		else if (~~it->unicode(~~) == '?')
436		{
437		if (~~(it+1)->unicode(~~) == '>')
	431	token = parser::Token_IS_GREATER;
	432	}
	433	}
	434	else if (lookAt( pos ) == '~')
	435	{
	436	token = parser::Token_TILDE;
	437	}
	438	else if (lookAt( pos ) == ':')
	439	{
	440	if (lookAt( pos + 1 ) == ':') {
	441	m_curpos++;
	442	token = parser::Token_PAAMAYIM_NEKUDOTAYIM;
	443	} else {
	444	token = parser::Token_COLON;
	445	}
	446	}
	447	else if (lookAt( pos ) == '?')
	448	{
	449	if (lookAt( pos + 1 ) == '>')
438	450	{
439	451	//accept CLOSE_TAG inside StringVariableCurly too, as php does
440		token = Parser::Token_CLOSE_TAG;
441		m_curpos++;
442		if (~~(it+2)->unicode(~~) == '\n') m_curpos++;
	452	token = parser::Token_CLOSE_TAG;
	453	m_curpos++;
	454	if (lookAt( pos + 2 ) == '\n') m_curpos++;
443	455	while (state() != HtmlState) popState();
444	456	}
445	457	else
446	458	{
447		token = Parser::Token_QUESTION;
448		}
449		}
450		else if (~~it->unicode() == '-' && (it+1)->unicode(~~) == '>')
	459	token = parser::Token_QUESTION;
	460	}
	461	}
	462	else if (lookAt( pos ) == '-' && lookAt( pos + 1 ) == '>')
451	463	{
452	464	m_curpos++;
453		token = Parser::Token_OBJECT_OPERATOR;
454		if (isValidVariableIdentifier(~~it+2)~~) {
	465	token = parser::Token_OBJECT_OPERATOR;
	466	if (isValidVariableIdentifier( lookAt( pos + 2 ) ) ) {
455	467	pushState(StringVariableObjectOperator);
456	468	}
457	469	}
458		else if (~~it->unicode(~~) == '%')
459		{
460		if (~~(it+1)->unicode(~~) == '=') {
461		m_curpos++;
462		token = Parser::Token_MOD_ASSIGN;
463		} else {
464		token = Parser::Token_MOD;
465		}
466		}
467		else if (~~it->unicode(~~) == '/')
468		{
469		if (~~(it+1)->unicode(~~) == '=')
470		{
471		m_curpos++;
472		token = Parser::Token_DIV_ASSIGN;
473		}
474		else if (~~(it+1)->unicode(~~) == '/')
	470	else if (lookAt( pos ) == '%')
	471	{
	472	if (lookAt( pos + 1 ) == '=') {
	473	m_curpos++;
	474	token = parser::Token_MOD_ASSIGN;
	475	} else {
	476	token = parser::Token_MOD;
	477	}
	478	}
	479	else if (lookAt( pos ) == '/')
	480	{
	481	if (lookAt( pos + 1 ) == '=')
	482	{
	483	m_curpos++;
	484	token = parser::Token_DIV_ASSIGN;
	485	}
	486	else if (lookAt( pos + 1 ) == '/')
475	487	{
476	488	//accept COMMENT inside StringVariableCurly too, as php does
477		token = Parser::Token_COMMENT;
478		while (m_curpos < m_contentSize && ~~it->unicode(~~) != '\n' &&
479		!(~~(it+1)->unicode() == '?' && (it+2)->unicode(~~) == '>'))
480		{
481		it++;
	489	token = parser::Token_COMMENT;
	490	while (m_curpos < m_contentSize && lookAt( pos ) != '\n' &&
	491	!(lookAt( pos + 1 ) == '?' && lookAt( pos + 2 ) == '>'))
	492	{
	493	pos++;
482	494	m_curpos++;
483	495	}
484	496	}
485		else if (~~(it+1)->unicode(~~) == '*')
	497	else if (lookAt( pos + 1 ) == '*')
486	498	{
487	499	//accept COMMENT inside StringVariableCurly too, as php does
488		if (~~(it+2)->unicode() == '*' && (it+3)->isSpace()~~)
489		{
490		token = Parser::Token_DOC_COMMENT;
	500	if (lookAt( pos + 2 ) == '*' && lookAt( pos + 3 ) == ' ')
	501	{
	502	token = parser::Token_DOC_COMMENT;
491	503	}
492	504	else
493	505	{
494		token = Parser::Token_COMMENT;
495		}
496		it += 2;
	506	token = parser::Token_COMMENT;
	507	}
	508	pos += 2;
497	509	m_curpos += 2;
498		while (m_curpos < m_contentSize && !(~~it->unicode() == '*' && (it+1)->unicode(~~) == '/'))
499		{
500		it++;
	510	while (m_curpos < m_contentSize && !(lookAt( pos ) == '*' && lookAt( pos + 1 ) == '/'))
	511	{
	512	pos++;
501	513	m_curpos++;
502	514	}
503	515	m_curpos++;
504	516	} else {
505		token = Parser::Token_DIV;
506		}
507		}
508		else if (~~it->unicode(~~) == '#')
	517	token = parser::Token_DIV;
	518	}
	519	}
	520	else if (lookAt( pos ) == '#')
509	521	{
510	522	//accept COMMENT inside StringVariableCurly too, as php does
511		token = Parser::Token_COMMENT;
512		while (m_curpos < m_contentSize && ~~it->unicode(~~) != '\n')
513		{
514		it++;
515		m_curpos++;
516		}
517		}
518		else if (~~it->unicode(~~) == '^')
519		{
520		if (~~(it+1)->unicode(~~) == '=') {
521		m_curpos++;
522		token = Parser::Token_XOR_ASSIGN;
523		} else {
524		token = Parser::Token_BIT_XOR;
525		}
526		}
527		else if (~~it->unicode(~~) == '*')
528		{
529		if (~~(it+1)->unicode(~~) == '=') {
530		m_curpos++;
531		token = Parser::Token_MUL_ASSIGN;
532		} else {
533		token = Parser::Token_MUL;
534		}
535		}
536		else if (~~it->unicode(~~) == '\|')
537		{
538		if (~~(it+1)->unicode(~~) == '\|') {
539		m_curpos++;
540