var tzr_delimiters = [',','(',')','{','}','[',']',';','.',':','?'];
var tzr_operators = ['%','^','-','+','*','/','=','~','!','&','|','<','>',
'->','++','--','>>','<<','>>=','<<=','+=','-=','/=','*=','>=','<=','^=','$=','!=','==','/*','*/','//'];
var tzr_keywords = ['auto','bool','break','case','catch','char','cerr','cin','class','const','continue',
'cout','default','delete','do','double','else','enum','explicit','extern','float','for','friend',
'goto','if','inline','int','long','namespace','new','operator','private','protected','public',
'register','return','short','signed','sizeof','static','struct','switch','template','this','throw',
'try','typedef','union','unsigned','virtual','void','volatile','while','__asm','__fastcall','__based',
'__cdecl','__pascal','__inline','__multiple_inheritance','__single_inheritance','__virtual_inheritance'];
var tzr_directives = ['#define','#error','#include','#elif','#if','#line','#else','#ifdef',
'#pragma','#endif','#ifndef','#undef','#if','#else','#endif'];

function tzr_isIn(el,array){
	for (var i=0; i<array.length; i++)
		if (el==array[i]) return true;
	return false;
}

function tzr_isDelimiter(token){ return tzr_isIn(token,tzr_delimiters); }
function tzr_isOperator(token){ return tzr_isIn(token,tzr_operators); }
function tzr_isKeyword(token){ return tzr_isIn(token,tzr_keywords); }
function tzr_isDirective(token){ return tzr_isIn(token,tzr_directives); }

function tzr_getTokenClass(token){
	if (tzr_isDelimiter(token)) return "DELIMITER";
	if (tzr_isOperator(token)) return "OPERATOR";
	if (tzr_isKeyword(token)) return "KEYWORD";
	if (tzr_isDirective(token)) return "DIRECTIVE";
	return null;
}

var tzr_look, tzr_dot, tzr_input_char, tzr_output;

function Token(){
	this.token = "";
	this.token_class = "";
	this.tzr_line_number = 0;
	this.tzr_char_number = 0;
}

function tzr_report(){ return "Tokenizer report: " + ((tzr_output=="")?("Success<br>"):("<br>"+tzr_output)); }
function tzr_addReport(x){ tzr_output += "line: "+tzr_line_number+" char: "+tzr_char_number+" msg: "+ x +"<br>"; }

function Tokenizer(txt){
	this.tokenize = tzr_tokenize;
	this.report = tzr_report;
	tzr_input_char = txt;
	tzr_look = tzr_input_char.charAt(tzr_dot=0);
	tzr_output = "";
	tzr_line_number = tzr_char_number = 1;
	tzr_skipWhite();
}

function tzr_nextChar(){
	tzr_look = (tzr_dot+1 < tzr_input_char.length)? tzr_input_char.charAt(++tzr_dot) : null;
	tzr_char_number++;
}

function tzr_isLetter(){ if (tzr_look==null) return false; return (('a'<=tzr_look && tzr_look<='z') || ('A'<=tzr_look && tzr_look<='Z')); }
function tzr_isDigit(){ if (tzr_look==null) return false; return ('0'<=tzr_look && tzr_look<='9'); }
function tzr_isQuote1(){ if (tzr_look==null) return false; return (tzr_look=='\''); }
function tzr_isQuote2(){ if (tzr_look==null) return false; return (tzr_look=='"'); }
function tzr_isNewline(){ if (tzr_look==null) return false; return (tzr_look=='\n'); }
function tzr_isCarriage(){ if (tzr_look==null) return false; return (tzr_look=='\r'); }
function tzr_isUnderline(){ if (tzr_look==null) return false; return (tzr_look=='_'); }
function tzr_isWhite(){ if (tzr_look==null) return false; return (tzr_look==' ' || tzr_look=='\t'); }
function tzr_skipWhite(){ while (tzr_isWhite()) tzr_nextChar(); }

function tzr_getName(){
	if (tzr_look==null || !tzr_isLetter()){ tzr_addReport("expected a letter"); return null; }
	var name = tzr_look+"";
	tzr_nextChar();
	while (tzr_isLetter() || tzr_isDigit() || tzr_isUnderline()){
		name += tzr_look;
		tzr_nextChar();
	}
	tzr_skipWhite();
	return name;
}

function tzr_getNumber(){
	if (tzr_look==null || !tzr_isDigit()){ tzr_addReport("expected a digit"); return null; }
	var value = tzr_look+"";
	tzr_nextChar();
	while (tzr_isDigit()){
		value += tzr_look;
		tzr_nextChar();
	}
	tzr_skipWhite();
	return value;
}

function tzr_tokenize(){
	var list = new ArrayList();

	while (tzr_look!=null){
		var t = new Token();

		if (tzr_isLetter()){
			t.token = tzr_getName();
			t.token_class = tzr_getTokenClass(t.token);
			if (t.token_class==null)
				t.token_class = "IDENTIFIER";
		}
		else if (tzr_isDigit()){
			t.token = tzr_getNumber();
			t.token_class = "NUMBER";
		}
		else if (tzr_isDelimiter(tzr_look)){
			t.token = tzr_look;
			t.token_class = "DELIMITER";
			tzr_nextChar();
			tzr_skipWhite();
		}
		else if (tzr_isOperator(tzr_look)){
			var op = tzr_getOperator();

			if (op=="//"){
				while (tzr_look!=null && !tzr_isNewline()){
					op += tzr_look;
					tzr_nextChar();
				}
				t.token = op;
				t.token_class = "LINE_CMT";
			}
			else if (op=="/*"){
				var temp = tzr_look;
				while (tzr_look!=null && (temp!='*' || tzr_look!='/')){
					if (tzr_isNewline()) { tzr_line_number++; tzr_char_number = 0; tzr_look = "<br>"; }
					if (tzr_look==' ') tzr_look = "&nbsp;";
					if (tzr_look=='\t') tzr_look = "&nbsp; ";
					if (tzr_look=='<') tzr_look = "&lt;";
					op += tzr_look;
					temp = tzr_look;
					tzr_nextChar();
				}
				if (tzr_look!=null) op += tzr_look;
				t.token = op;
				t.token_class = "BLK_CMT";
				tzr_nextChar();
				tzr_skipWhite();    
			}
			else {
				t.token = op;
				t.token_class = "OPERATOR";
			}
		}
		else if (tzr_isNewline() || tzr_isCarriage()){
			if (tzr_isNewline()){
				tzr_line_number++;
				tzr_char_number = 0; 
			}
			tzr_nextChar();
			tzr_skipWhite();
			t.token = "\n";
			t.token_class = "NEWLINE";
		}
		else if (tzr_isQuote1()){
			t = tzr_quote1();
		}
		else if (tzr_isQuote2()){
			t = tzr_quote2();
		}
		else if (tzr_look=='#'){
			tzr_nextChar();
			tzr_skipWhite();
			t.token = "#"+tzr_getName();
			t.token_class = tzr_getTokenClass(t.token);
			t.line_number = tzr_line_number;
			t.char_number = tzr_char_number;

			list.add(t);

			t = new Token();
			var dummy = "";
			while (tzr_look!=null && !tzr_isNewline()){
				dummy += tzr_look;
				tzr_nextChar();
			}
			t.token = dummy;
			t.token_class = "IDENTIFIER";
		}
		else if (tzr_look=='&'){
			t.token = "&";
			t.token_class = tzr_getTokenClass(t.token);
			tzr_nextChar();
			tzr_skipWhite();
		}
		else if (tzr_look==' '){
			tzr_addReport("error token! there should be no space here");
			tzr_nextChar();
			tzr_skipWhite();
		}
		else {
			tzr_addReport("unrecognized token = "+tzr_look);
			tzr_nextChar();
			tzr_skipWhite();
		}

		t.line_number = tzr_line_number;
		t.char_number = tzr_char_number;

		if (t.token==null){
			tzr_addReport("error : null token");
			return list;
		}

		// tzr_output += t.token + " - " + t.token_class + "<br>";
		list.add(t);
	}

	return list.list;
}

function tzr_getOperator(){
	if (tzr_look==null || !tzr_isOperator(tzr_look)){ tzr_addReport("expected an operator"); return null; }
	var op = tzr_look+"";
	tzr_nextChar();
	while (tzr_isOperator(op+tzr_look)){ op += tzr_look; tzr_nextChar(); }
	tzr_skipWhite();
	return op;
}

function tzr_quote1(){
	var t = new Token();
	var temp = tzr_look;

	tzr_nextChar();
	while (tzr_look!=null && !tzr_isQuote1()){
		if (tzr_look=='\\'){
			temp += tzr_look;
			tzr_nextChar();
		}
		temp += tzr_look;
		tzr_nextChar();
	}
	if (tzr_look!=null) temp += tzr_look;
	tzr_nextChar();
	tzr_skipWhite();

	t.token = temp;
	t.token_class = "QUOTE1";

	return t;
}

function tzr_quote2(){
	var t = new Token();
	var temp = tzr_look;

	tzr_nextChar();
	while (tzr_look!=null && !tzr_isQuote2()){
		if (tzr_look=='\\'){
			temp += tzr_look;
			tzr_nextChar();
		}
		temp += tzr_look;
		tzr_nextChar();
	}
	if (tzr_look!=null) temp += tzr_look;
	tzr_nextChar();
	tzr_skipWhite();

	t.token = temp;
	t.token_class = "QUOTE2";

	return t;
}