/****************************************************************************
	Kalkulon - A programmable calculator for programmers

	This file is part of the Kalkulon core.
  
	If you find this source code useful, your are encouraged to use it in your own projects.
	Please give me credit!
	
	COPYRIGHT by Juergen Holetzeck 2003 - 2008
	e-mail:		contact@kalkulon.de
	homepage:	www.kalkulon.de
****************************************************************************/


// ArithParser.cpp: Implementierung der Klasse CArithParser.
//
//////////////////////////////////////////////////////////////////////

#include "stdafx.h"

#include <cstdlib>
#include <algorithm>

#include "DataObject.h"
#include "Token.h"
#include "ArithParser.h"
#include "ErrBase.h"

using namespace Kalkulon;

//////////////////////////////////////////////////////////////////////
// Konstruktion/Destruktion
//////////////////////////////////////////////////////////////////////

CArithParser::CArithParser()
{
	
}


CArithParser::~CArithParser()
{
	
}

/////////////////////////////////////////////////////////////////////
// ArithParser
int CArithParser::parse(std::istream* in, RPNStack_t* prpn_stack_)
{
	uid = 0;
	instream = in;
	prpn_stack = prpn_stack_;
	next_token();
	int num_expr=expr_list();
	if(tok.tid!=Token::END) throw ErrENDExpr(tok);	// END expected
	return num_expr;
}


int CArithParser::expr_list ()
{
	int num_expr=1;		// count number of expressions
	for(;;)
	{
		arg_list();
		switch(tok.tid)
		{
		case Token::DELIM:
			{
				tok.uid = uid;
				prpn_stack->push_back(tok);
				next_token();
				if(tok.tid==Token::END) return num_expr;
				++num_expr;
				break;
			}
		case Token::END:
			prpn_stack->push_back(tok);
			// no break;
		default:
			return num_expr;
		}
	}
}


int CArithParser::arg_list (bool function)
{
	int num_expr=1;		// count number of arguments
	for(;;)
	{
		expression();
		switch(tok.tid)
		{
		case Token::SEP:
			{
				if(!function) prpn_stack->push_back(tok);
				next_token();
				++num_expr;
				break;
			}
		default:
			return num_expr;
		}
	}
}


void CArithParser::expression()
{
	logical_or_expr();
}


void CArithParser::logical_or_expr()
{
	logical_and_expr();
	for(;;)
		switch(tok.tid)
	{
			case Token::LOGICAL_OR:
				{
					// logical AND/OR defines sequence point! -> expr1 OR expr2 BLOCKEND
					int postmp=tok.pos;				
					tok.uid = ++uid;
					prpn_stack->push_back(tok);
					next_token();
					logical_and_expr();
					// add additional BLOCKEND token
					Token tok_tmp(Token::BLOCKEND, postmp, uid--);
					prpn_stack->push_back(tok_tmp);
					break;
					
				}
			default:
				return;
	}
}


void CArithParser::logical_and_expr()
{
	or_expr();
	for(;;)
		switch(tok.tid)
	{
			case Token::LOGICAL_AND:
				{
					// logical AND/OR defines sequence point! -> expr1 AND expr2 BLOCKEND
					int postmp=tok.pos;				
					tok.uid = ++uid;
					prpn_stack->push_back(tok);
					next_token();
					or_expr();
					// add additional BLOCKEND token
					Token tok_tmp(Token::BLOCKEND, postmp, uid--);
					prpn_stack->push_back(tok_tmp);
					break;
				}
			default:
				return;
	}
}


void CArithParser::or_expr()
{
	xor_expr();
	for(;;)
		switch(tok.tid)
	{
			case Token::OR:
				{
					Token tok_post=tok;
					next_token();
					xor_expr();
					prpn_stack->push_back(tok_post);
					break;
				}
			default:
				return;
	}
}


void CArithParser::xor_expr()
{
	and_expr();
	for(;;)
		switch(tok.tid)
	{
			case Token::XOR:
				{
					Token tok_post=tok;
					next_token();
					and_expr();
					prpn_stack->push_back(tok_post);
					break;
				}
			default:
				return;
	}
}


void CArithParser::and_expr()
{
	eq_expr();
	for(;;)
		switch(tok.tid)
	{
			case Token::AND:
				{
					Token tok_post=tok;
					next_token();
					eq_expr();
					prpn_stack->push_back(tok_post);
					break;
				}
			default:
				return;
	}
}


void CArithParser::eq_expr()
{
	rel_expr();
	for(;;)
		switch(tok.tid)
	{
			case Token::EQUAL:
			case Token::NOT_EQUAL:
				{
					Token tok_post=tok;
					next_token();
					rel_expr();
					prpn_stack->push_back(tok_post);
					break;
				}
			default:
				return;
	}
}


void CArithParser::rel_expr()
{
	shift_expr();
	for(;;)
		switch(tok.tid)
	{
			case Token::GREATER:
			case Token::GREATER_EQUAL:
			case Token::LESSER:
			case Token::LESSER_EQUAL:
				{
					Token tok_post=tok;
					next_token();
					shift_expr();
					prpn_stack->push_back(tok_post);
					break;
				}
			default:
				return;
	}
}


void CArithParser::shift_expr()
{
	add_expr();
	for(;;)
		switch(tok.tid)
	{
			case Token::LSHIFT:
			case Token::RSHIFT:
				{
					Token tok_post=tok;
					next_token();
					add_expr();
					prpn_stack->push_back(tok_post);
					break;
				}
			default:
				return;
	}
}


void CArithParser::add_expr  ()
{
	mult_expr();
	for(;;)
		switch(tok.tid)
	{
			case Token::PLUS:
			case Token::MINUS:
				{
					Token tok_post=tok;
					next_token();
					mult_expr();
					prpn_stack->push_back(tok_post);
					break;
				}
			default:
				return;
	}
}


void CArithParser::mult_expr ()
{
	pow_expr();
	for(;;)
		switch(tok.tid)
	{
			case Token::MUL:
			case Token::DIV:
			case Token::MOD:
				{
					Token tok_post=tok;
					next_token();
					pow_expr();
					prpn_stack->push_back(tok_post);
					break;
				}
			default:
				return;
	}
}


void CArithParser::pow_expr ()
{
	unary_expr();
	switch(tok.tid)
	{
	case Token::POW:
		{
			Token tok_post=tok;
			next_token();
			pow_expr();
			prpn_stack->push_back(tok_post);
			break;
		}
	default:
		return;
	}
}


void CArithParser::unary_expr()
{
	switch(tok.tid)
	{
	case Token::CAST_U08:
	case Token::CAST_U16:
	case Token::CAST_U32:
	case Token::CAST_S08:
	case Token::CAST_S16:
	case Token::CAST_S32:
	case Token::CAST_STRING:
		
	case Token::MINUS:
	case Token::NOT:
	case Token::LOGICAL_NOT:
		{
			if(tok.tid == Token::MINUS) tok.tid = Token::NEG;
			Token tok_post=tok;
			next_token();
			unary_expr();
			prpn_stack->push_back(tok_post);
			break;
		}
	default:
		postfix_expr();
		break;
	}
}


void CArithParser::postfix_expr()
{
	prim_expr();
	for(;;)
		switch(tok.tid)
	{
			case Token::INDEXBEGIN:
				{	// index notation: [1, 2, ..., n] -> 1 2 ... n INDEXEND
					next_token();
					tok.token_int = arg_list(true);	// remember size of list
					if(tok.tid != Token::INDEXEND) throw ErrRBExpr(tok);	// right brackets "]" expected
					prpn_stack->push_back(tok);
					next_token();					
					break;
				}
			case Token::EVALPF:
				{
					Token tok_post=tok;		// EVALPF
					next_token();			// look ahead
					if(tok.tid != Token::LP) throw ErrLPExpr(tok);	// left parenthesis "(" expected
					next_token();
					if(tok.tid != Token::RP)
					{
						// more than 0 arguments
						tok_post.token_int = arg_list(true);
						if(tok.tid != Token::RP) throw ErrRPExpr(tok);	// right parenthesis ")" expected
					}
					else
					{
						// 0 arguments
						tok_post.token_int = 0;
					}
					prpn_stack->push_back(tok_post);
					next_token();	// skip ")"
					break;
				}
			default:
				return;
	}	
}


void CArithParser::prim_expr ()
{
	switch(tok.tid)
	{
	case Token::DATAOBJECT:
		prpn_stack->push_back(tok);
		next_token();
		break;
	case Token::INC:
	case Token::DEC:
		{
			// pre increment
			Token tok_post=tok;
			next_token();	// look ahead
			if(tok.tid == Token::SCOPE)
			{
				// handle identifier with global scope, set FLAG_GLOBALSCOPE
				next_token();	// look ahead
				tok.token_flags = Token::FLAG_GLOBALSCOPE;
			}
			
			// next token must be an identifier
			if(tok.tid == Token::NAME)
			{
				(tok_post.tid==Token::INC) ? tok.tid=Token::PREINC : tok.tid=Token::PREDEC;
				tok_post=tok; // PREINC or PREDEC
				next_token();
			}
			else
				throw ErrIdExpected(tok);

			if(tok.tid == Token::INDEXBEGIN)
			{
				// indexed preinc: ++id[1,2,..n] -> 1 2..n PREINC(id)
				// set FLAG_INDEXED
				tok_post.token_flags = (Token::TokenFlags) (tok_post.token_flags | Token::FLAG_INDEXED);
				next_token();
				tok_post.token_int = arg_list(true);	// remember size of list
				if(tok.tid != Token::INDEXEND) throw ErrRBExpr(tok);	// right brackets "]" expected
				next_token();					
			}
			prpn_stack->push_back(tok_post);
		}
		break;
	case Token::SCOPE:
		{
			// handle identifier with global scope, set FLAG_GLOBALSCOPE
			// next token must be an identifier
			next_token();
			if(tok.tid != Token::NAME) throw ErrIdExpected(tok); 
			tok.token_flags = Token::FLAG_GLOBALSCOPE;
		}
		// no break !!! SCOPE belongs to NAME
	case Token::NAME:
		{
			Token tok_post=tok;					// NAME
			next_token();						// look ahead
			switch (tok.tid)
			{
			case Token::INDEXBEGIN:
				{	// indexed name: id[1,2,..n]        -> 1 2..n NAME(id)
					// assignment  : id[1,2,..n] = expr -> 1 2..n expr LSNAME(id) 
					// set FLAG_INDEXED
					tok_post.token_flags = (Token::TokenFlags) (tok_post.token_flags | Token::FLAG_INDEXED);
					next_token();
					tok_post.token_int = arg_list(true);	// remember size of list
					if(tok.tid != Token::INDEXEND) throw ErrRBExpr(tok);	// right brackets "]" expected
					next_token();					
					// no break !!! for op "[]=" "[]++" etc.
				}					
			case Token::ASSIGN:
			case Token::INC:
			case Token::DEC:
				{
					// BEWARE: new switch because of "no break" case from above !!!
					switch(tok.tid)
					{
					case Token::ASSIGN:
						// assignment
						tok_post.tid=Token::LSNAME;
						tok_post.sid=tok.sid;	// possible "compound assign" like '+=', '*=' etc
						next_token();
						expression();
						prpn_stack->push_back(tok_post);
						break;
					case Token::INC:
						// post increment
						tok_post.tid=Token::POSTINC;
						prpn_stack->push_back(tok_post);
						next_token();
						break;
					case Token::DEC:
						// post decrement
						tok_post.tid=Token::POSTDEC;
						prpn_stack->push_back(tok_post);
						next_token();
						break;
					default:
						// pure identifier
						prpn_stack->push_back(tok_post);
						break;
					}
					break;
				}
			case Token::LP:
				{
					// function call or function definition
					
					// perhaps function definition, if not this token will be erased later (see below)
					tok_post.tid = Token::LSFUNC;
					tok_post.uid = ++uid;
					
					prpn_stack->push_back(tok_post);
					RPNStack_t::iterator itFuncBegin=--prpn_stack->end();	// remember position
					
					next_token();
					if(tok.tid != Token::RP)
					{
						// more than 0 arguments
						tok_post.token_int = itFuncBegin->token_int = arg_list(true);
						if(tok.tid != Token::RP) throw ErrRPExpr(tok);	// right parenthesis ")" expected
					}
					else
					{
						// 0 arguments
						tok_post.token_int = itFuncBegin->token_int = 0;
					}
					next_token();	// skip ")"
					
					if(tok.tid==Token::ASSIGN)
					{
						// function definition
						// LSFUNC already stored at begin, store ASSIGN to recognize end of parameter list
						prpn_stack->push_back(tok);
						next_token();
						expression();
						// add additional BLOCKEND token
						// set position to start of function definition
						Token tok_tmp(Token::BLOCKEND, tok_post.pos, uid--);
						prpn_stack->push_back(tok_tmp);
					}
					else
					{
						// function call
						
						// erase previous LSFUNC token
						prpn_stack->erase(itFuncBegin);
						--uid; // correct UID
						
						// and store FUNC
						tok_post.tid=Token::FUNC;
						prpn_stack->push_back(tok_post);
					}
				}
				break;
			default:
				// pure identifier
				prpn_stack->push_back(tok_post);
				break;
			}				
		}
		break;
		case Token::LP:
			next_token();
			arg_list();
			if(tok.tid != Token::RP) throw ErrRPExpr(tok);	// right parenthesis ")" expected
			next_token();
			break;
			
		case Token::PUREFUNC:
			{
				int pos=tok.pos;
				tok.uid = ++uid;
				prpn_stack->push_back(tok);
				next_token();
				arg_list();
				if(tok.tid != Token::PUREFUNC) throw ErrRQExpr(tok);	// right quote "'" expected
				// add additional BLOCKEND token
				Token tok_tmp(Token::BLOCKEND, pos, uid--);
				prpn_stack->push_back(tok_tmp);
				prpn_stack->push_back(tok);
				next_token();
				break;
			}
		case Token::LISTBEGIN:
			{	// list notation: {1, 2, ..., n} -> 1 2 ... n LISTEND
				next_token();
				if(tok.tid != Token::LISTEND)
				{
					tok.token_int = arg_list(true);	// remember size of list
					if(tok.tid != Token::LISTEND) throw ErrRCBExpr(tok);	// right curly barces "}" expected
				}
				else
				{
					// list of zero size
					tok.token_int = 0;
				}
				prpn_stack->push_back(tok);
				next_token();
				break;
			}
		case Token::IF:
		case Token::WHILE:
		case Token::DO:
		case Token::FOR:
			{	// statements have delayed argument evaluation: STATEMENTuid, ARG1, ..., ARGn, BLOCKENDuid
				int num_arg_min=0, num_arg_max=0;
				
				if(tok.tid==Token::IF)
					num_arg_min = 2, num_arg_max = 3;	
				else if(tok.tid==Token::WHILE || tok.tid==Token::DO) 
					num_arg_min = num_arg_max = 2;
				else if(tok.tid==Token::FOR)
					num_arg_min = num_arg_max = 4;
				
				Token tok_cpy=tok;				
				tok.uid = ++uid;
				prpn_stack->push_back(tok);
				RPNStack_t::iterator itbeg = --prpn_stack->end();				// remember position
				next_token();
				if(tok.tid != Token::LP) throw ErrLPExpr(tok);					// left parenthesis "(" expected
				next_token();
				int num_expr=expr_list();
				if(num_expr < num_arg_min || num_expr > num_arg_max) throw ErrStatement(tok_cpy);	// number of 'args' incorrect
				itbeg->token_int = num_expr;
				if(tok.tid != Token::RP) throw ErrRPExpr(tok);					// right parenthesis ")" expected
				// add additional BLOCKEND token
				Token tok_tmp(Token::BLOCKEND, tok_cpy.pos, uid--);
				prpn_stack->push_back(tok_tmp);
				
				next_token();
				break;
			}
		default:
			throw ErrPrimExpr(tok);	// primary expr expected
	}
}


void CArithParser::next_token()
{
	tok = get_token();
}


/////////////////////////////////////////////////////////////////////
// lexer
Token CArithParser::get_token()
{
	bool bComment;
	Token t;	
	
	do{
		bComment = false;
		char ch=0;
		
		int startpos = (int) instream->tellg();	// remember position before token 
		*instream >> ch;
		t.pos = (int) instream->tellg();			// remember position of token
		
		switch(ch)
		{
		case 0: 
			t.tid = Token::END;
			break;
			
		case ';':
		case ',':
		case ')':
		case '{':
		case '}':
		case '[':
		case ']':
		case '\'':
		case '@':
		case '~':
			t.tid = Token::TokenId(ch);
			break;
		case '/':
			{
				// test for comment
				instream->get(ch);
				switch(ch)
				{
					// test for '//'
					case '/':
						// start of line comment
						while(instream->get(ch) && ch && ch!='\n') ;
						bComment =true;
						break;
					// test for '/*'
					case '*':
						// start of block comment
						while(instream->get(ch))
						{
							if(ch == '*')
							{
								if(!instream->get(ch) || ch == '/') break;
								instream->putback(ch);
							}
						}
						bComment =true;
						break;
					// test for '/='
					case '=':
						t.tid = Token::ASSIGN;
						t.sid = Token::DIV;
						break;
					default:
						// only simple '/' DIV
						instream->putback(ch);
						t.tid = Token::DIV;
				}
				break;
			}
		case '(':
			{
				// could be normal left parenthesis or begin of cast expression
				int pos=instream->tellg();
				*instream >> ch;
				if(isalpha(ch))
				{
					t.token_str = ch;
					while(instream->get(ch) && isalnum(ch)) t.token_str += ch;
					instream->putback(ch);
					*instream >> ch;
					if(ch==')' && iscast(t))
					{
						// cast expression detected
						return t;
					}
				}
				// just left parenthesis, so put stream pointer back
				instream->seekg(pos);
				t.tid=Token::LP;
				break;
			}
		case '+':
			{
				instream->get(ch);
				switch(ch)
				{
					// test for '++' INC
					case '+':
						t.tid=Token::INC;
						break;					
					// test for '+='
					case '=':
						t.tid = Token::ASSIGN;
						t.sid = Token::PLUS;
						break;
					default:
						// only simple '+' PLUS
						instream->putback(ch);
						t.tid = Token::PLUS;
				}
				break;
			}
		case '-':
			{
				instream->get(ch);
				switch(ch)
				{
					// test for '--' DEC
					case '-':
						t.tid=Token::DEC;
						break;
					// test for '-='
					case '=':
						t.tid = Token::ASSIGN;
						t.sid = Token::MINUS;
						break;
					default:
						// only simple '-' MINUS
						instream->putback(ch);
						t.tid = Token::MINUS;
				}
				break;
			}
		case '*':
			{
				instream->get(ch);
				switch(ch)
				{
					// test for '**' POW
					case '*':
						// test for '**='
						instream->get(ch);
						if(ch == '=')
						{
							t.tid=Token::ASSIGN;
							t.sid=Token::POW;
						}
						else
						{
							// only simple '**' POW
							instream->putback(ch);
							t.tid=Token::POW;
						}
						break;
					// test for '*='
					case '=':
						t.tid=Token::ASSIGN;
						t.sid=Token::MUL;
						break;
					default:
						// only simple '*' MUL
						instream->putback(ch);
						t.tid = Token::MUL;
				}
				break;
			}
		case '=':
			{
				// test for '=='
				instream->get(ch);
				if(ch=='=')
					t.tid=Token::EQUAL;
				else
				{
					// only simple '='
					instream->putback(ch);
					t.tid=Token::ASSIGN;
				}				
				break;
			}
		case '!':
			{
				// test for '!='
				instream->get(ch);
				if(ch=='=')
					t.tid=Token::NOT_EQUAL;
				else
				{
					// only simple '!'
					instream->putback(ch);
					t.tid=Token::LOGICAL_NOT;
				}				
				break;
			}
		case '<':
			{
				instream->get(ch);
				switch(ch)
				{
					// test for '<<'
					case '<':
						// test for '<<='
						instream->get(ch);
						if(ch == '=')
						{
							t.tid=Token::ASSIGN;
							t.sid=Token::LSHIFT;
						}
						else
						{
							// only simple '<<' LSHIFT
							instream->putback(ch);
							t.tid=Token::LSHIFT;
						}
						break;
					// test for '<='
					case '=':
						t.tid=Token::LESSER_EQUAL;
						break;					
					default:
						// only simple '<'
						instream->putback(ch);
						t.tid=Token::LESSER;
				}
				break;
			}
		case '>':
			{
				instream->get(ch);
				switch(ch)
				{
					// test for '>>'
					case '>':
						// test for '>>='
						instream->get(ch);
						if(ch == '=')
						{
							t.tid=Token::ASSIGN;
							t.sid=Token::RSHIFT;
						}
						else
						{
							// only simple '>>' RSHIFT
							instream->putback(ch);
							t.tid=Token::RSHIFT;
						}
						break;
					// test for '>='
					case '=':
						t.tid=Token::GREATER_EQUAL;
						break;					
					default:
						// only simple '>'
						instream->putback(ch);
						t.tid=Token::GREATER;
				}
				break;
			}
		case '|':
			{
				instream->get(ch);
				switch(ch)
				{
					// test for '||'
					case '|':
						t.tid=Token::LOGICAL_OR;
						break;
					// test for '|='
					case '=':
						t.tid=Token::ASSIGN;
						t.sid=Token::OR;
						break;
					default:
						// only simple '|'
						instream->putback(ch);
						t.tid=Token::OR;
				}
				break;
			}
		case '&':
			{
				instream->get(ch);
				switch(ch)
				{
					// test for '&&'
					case '&':
						t.tid=Token::LOGICAL_AND;
						break;
					// test for '&='
					case '=':
						t.tid=Token::ASSIGN;
						t.sid=Token::AND;
						break;
					default:
						// only simple '&'
						instream->putback(ch);
						t.tid=Token::AND;
				}
				break;
			}
		case '%':
			{
				instream->get(ch);
				switch(ch)
				{
					// test for '%='
					case '=':
						t.tid=Token::ASSIGN;
						t.sid=Token::MOD;
						break;
					default:
						// only simple '%'
						instream->putback(ch);
						t.tid=Token::MOD;
				}
				break;
			}
		case '^':
			{
				instream->get(ch);
				switch(ch)
				{
					// test for '^='
					case '=':
						t.tid=Token::ASSIGN;
						t.sid=Token::XOR;
						break;
					default:
						// only simple '^'
						instream->putback(ch);
						t.tid=Token::XOR;
				}
				break;
			}
		case '"':
			{
				// string
				t.tid = Token::DATAOBJECT;
				DataObject dob(DataObject::STRING);
				while(instream->get(ch))
				{
					if(ch == '\\')
					{
						// escape sequence
						if(instream->get(ch))
						switch(ch)
						{
							case 'n': // newline
								*dob.getPtrStrValue() += '\n';
								break;
							case 't': // tab
								*dob.getPtrStrValue() += '\t';
								break;
							default:
								// un-escaped character
								*dob.getPtrStrValue() += ch;
						}
					}
					else
						if(ch == '"')
						{
							break; // end while loop
						}
						else
						{
							// normal character
							*dob.getPtrStrValue() += ch;
						}	
				}
				t.data = dob;
				break;
			}
		case ':':
			// test for scope operator '::'
			instream->get(ch);
			if(ch==':')
				t.tid=Token::SCOPE;
			else
			{
				// only simple ':', unknown token
				t.tid = Token::UNKNOWN;
				t.token_str = ':';
				throw ErrWrongToken(t);	
			}				
			break;
		default:
			if(isalpha(ch) || ch=='#')
			{
				// name found
				
				t.tid = Token::NAME;
				t.token_str = ch;
				while(instream->get(ch) && (isalnum(ch) || ch=='_')) t.token_str += ch;
				instream->putback(ch);
				
				isstatement(t);
				return t;
			}
			else
				if(isdigit(ch) || ch =='.')
				{
					// number found
					char base;
					std::string num_str;
					
					if(   instream->get(base) 
						&& ch=='0'
                        && (base == 'x' || base == 'X' || base == 'b' || base == 'B' || base == 'o' || base == 'O')
						)
					{
						// long integer number
						t.tid = Token::DATAOBJECT;
						unsigned long i=0;
						switch(base)
						{
						case 'x':	// hex number
						case 'X':
							while(instream->get(ch) && isxdigit(ch)) num_str+=ch;
							instream->putback(ch);
							i = strtoul(num_str.c_str(),NULL,16);	
							break;
						case 'b': // binary number
						case 'B':
							while(instream->get(ch) && (ch=='0' || ch=='1')) num_str+=ch;
							instream->putback(ch);
							i = strtoul(num_str.c_str(),NULL,2);	
							break;
                        case 'o': // octal number
                        case 'O':
                            while(instream->get(ch) && (ch>='0' && ch<='7')) num_str+=ch;
                            instream->putback(ch);
                            i = strtoul(num_str.c_str(),NULL,8);
                            break;
                            //default:  not used, see if
						}
						t.data = DataObject(i);
					}
					else
					{
						// float number
						t.tid = Token::DATAOBJECT;
						double d;
						if(!instream->eof())
						{
							instream->seekg(startpos);
							*instream >> d;
						}
						else
						{
							// eof reached, so last read character can't be 'putback'ed
							num_str=ch;
							d = strtoul(num_str.c_str(),NULL,10);
						}
						t.data = DataObject(d);
					}
					return t;
				}
				// unknown token found
				t.tid = Token::UNKNOWN;
				t.token_str = ch;
				t.pos = instream->tellg();
				throw ErrWrongToken(t);	
			}
		} while(bComment);
		return t;
}


/////////////////////////////////////////////////////////////////////
// help functions

bool CArithParser::iscast(Token &tok)
{
	std::string s=tok.token_str;
	std::transform(s.begin(),s.end(),s.begin(),tolower);
	
	if(	s=="u8"		||
		s=="u08"	||
		s=="ub"		||
		s=="byte")
	{
		tok.tid = Token::CAST_U08;
		return true;
	}
	
	if(	s=="u16"	||
		s=="uw"		||
		s=="word")
	{
		tok.tid = Token::CAST_U16;
		return true;
	}
	
	if(	s=="u32"	||
		s=="ul")
	{
		tok.tid = Token::CAST_U32;
		return true;
	}
	
	if(	s=="s8"		||
		s=="s08"	||
		s=="sb"		||
		s=="char")
	{
		tok.tid = Token::CAST_S08;
		return true;
	}
	
	if(	s=="s16"	||
		s=="sw")
	{
		tok.tid = Token::CAST_S16;
		return true;
	}
	
	if(	s=="s32"	||
		s=="sl")
	{
		tok.tid = Token::CAST_S32;
		return true;
	}
	
	if(	s=="string")
	{
		tok.tid = Token::CAST_STRING;
		return true;
	}
	
	return false;
}


bool CArithParser::isstatement(Token &tok)
{
	
	if(tok.token_str == "while")
	{
		tok.tid = Token::WHILE;
		return true;
	}
	
	if(tok.token_str == "do")
	{
		tok.tid = Token::DO;
		return true;
	}
	
	if(tok.token_str == "if")
	{
		tok.tid = Token::IF;
		return true;
	}
	
	if(tok.token_str == "for")
	{
		tok.tid = Token::FOR;
		return true;
	}
	return false;
}
