/*********************************************************************
 * This file is part of the cpplibs suite.
 *
 * Copyright (C) 2001 Topi Mäenpää
 * All rights reserved.
 *
 * This program is free software. You can redistribute and/or modify
 * it under the terms of the free software licence found in the
 * accompanying file "COPYING". The licence terms must always be
 * redistributed with this source file. The above copyright notice
 * must be reproduced in all modified and unmodified copies of this
 * source file.
 *
 * $Revision: 1.13 $
 *********************************************************************/

#include "StreamTokenizer.h"
#include <stdlib.h>

#define readCharThrow { input.read(&chr,1); if (!input) throw IOException("StreamTokenizer::getNextToken(): cannot read from stream"); }

using namespace std;
using namespace util::io;

namespace util
{
	StreamTokenizer::StreamTokenizer(Syntax syntax)
	{
		_cDecimalSeparator = '.';
		_iMaxStartLength = 0;
		setDecimalSeparator(_cDecimalSeparator);
		if (syntax == SYNTAX_C)
			setCSyntax();
		else
			setXMLSyntax();
	}

	StreamTokenizer::~StreamTokenizer()
	{
		clearGeneratedSections();
	}

	void StreamTokenizer::addSection(Section* section)
	{
		unsigned char startChar = section->start()[0];
		_lstSections += section;
		_iaAttributeMap[startChar] |= TOKEN_SECTION;
		if (section->start().length() > _iMaxStartLength)
			_iMaxStartLength = section->start().length();
	}
	
	void StreamTokenizer::removeSection(Section* section)
	{
		unsigned char startChar = section->start()[0];
		_lstSections -= section;
		_iMaxStartLength = 0;
		//Check if the last section for this start character was deleted
		bool match = false;
		for (int i=_lstSections.getLength();i--;)
			{
				if (_lstSections[i]->start()[0] == startChar)
					match = true;
				if (_lstSections[i]->start().length() > _iMaxStartLength)
					_iMaxStartLength = _lstSections[i]->start().length();
			}
		if (!match)
			_iaAttributeMap[startChar] &= ~TOKEN_SECTION;
	}

	void StreamTokenizer::setXMLSyntax(void)
	{
		for (int i=0;i<128;i++)
			{
				_iaAttributeMap[i] = 0;
				if (isalnum(i))
					_iaAttributeMap[i] |= TOKEN_WORD;
				if (isdigit(i))
					_iaAttributeMap[i] |= TOKEN_NUMBER;
				if (isspace(i))
					_iaAttributeMap[i] |= TOKEN_SPACE;
				if (!_iaAttributeMap[i])
					_iaAttributeMap[i] = TOKEN_NORMAL;
			}
		for (int i=128;i<256;i++)
			_iaAttributeMap[i] = TOKEN_WORD;
		
		setAttribute('.', TOKEN_NORMAL | TOKEN_NUMBER);
		setAttribute('-', TOKEN_NORMAL | TOKEN_NUMBER);

		EndDelimitedSection *stringSection = new EndDelimitedSection("string","\"","\"");
		stringSection->addReplacement("&lt;","<");
		stringSection->addReplacement("&gt;",">");
		stringSection->addReplacement("&amp;","&");
		stringSection->addReplacement("&apos;","'");
		stringSection->addReplacement("&quot;","\"");
		stringSection->addReplacement("\r\n"," ");
		stringSection->addReplacement("\n"," ");
		stringSection->addReplacement("\t"," ");

		EndDelimitedSection *stringSection2 = new EndDelimitedSection("string","'","'");
		stringSection2->addReplacement("&lt;","<");
		stringSection2->addReplacement("&gt;",">");
		stringSection2->addReplacement("&amp;","&");
		stringSection2->addReplacement("&apos;","'");
		stringSection2->addReplacement("&quot;","\"");
		stringSection->addReplacement("\r\n"," ");
		stringSection->addReplacement("\n"," ");
		stringSection->addReplacement("\t"," ");

		EndDelimitedSection *comment = new EndDelimitedSection("comment","<!--","-->");
		EndDelimitedSection *instruction = new EndDelimitedSection("instruction","<?","?>");
		EndDelimitedSection *declaration = new EndDelimitedSection("declaration","<!",">");
		EndDelimitedSection *cdata = new EndDelimitedSection("cdata","<![CDATA[","]]>");
		EndDelimitedSection *tag = new EndDelimitedSection("tag","<",">");

		clearGeneratedSections();
		
		addSection(stringSection);
		addSection(stringSection2);
		addSection(comment);
		addSection(instruction);
		addSection(declaration);
		addSection(cdata);
		addSection(tag);

		_lstGeneratedSections += stringSection;
		_lstGeneratedSections += stringSection2;
		_lstGeneratedSections += comment;
		_lstGeneratedSections += instruction;
		_lstGeneratedSections += declaration;
		_lstGeneratedSections += cdata;
		_lstGeneratedSections += tag;
	}
	
	void StreamTokenizer::setCSyntax()
	{
		for (int i=0;i<128;i++)
			{
				_iaAttributeMap[i] = 0;
				if (isalnum(i))
					_iaAttributeMap[i] |= TOKEN_WORD;
				if (isdigit(i))
					_iaAttributeMap[i] |= TOKEN_NUMBER;
				if (isspace(i))
					_iaAttributeMap[i] |= TOKEN_SPACE;
				if (!_iaAttributeMap[i])
					_iaAttributeMap[i] = TOKEN_NORMAL;
			}
		for (int i=128;i<256;i++)
			_iaAttributeMap[i] = TOKEN_WORD;
		
		setAttribute('.', TOKEN_NORMAL | TOKEN_NUMBER);
		setAttribute('-', TOKEN_NORMAL | TOKEN_NUMBER);

		EndDelimitedSection *stringSection = new EndDelimitedSection("string","\"","\"");
		stringSection->addReplacement("\\n","\n");
		stringSection->addReplacement("\\r","\r");
		stringSection->addReplacement("\\t","\t");
		stringSection->addReplacement("\\\"","\"");
		stringSection->addReplacement("\\\'","\'");
		stringSection->addReplacement("\\\\","\\");

		EndDelimitedSection *slashStarComment = new EndDelimitedSection("comment","/*","*/");
		EOLSection *slashSlashComment = new EOLSection("comment","//");

		clearGeneratedSections();
		
		addSection(stringSection);
		addSection(slashStarComment);
		addSection(slashSlashComment);

		_lstGeneratedSections += stringSection;
		_lstGeneratedSections += slashStarComment;
		_lstGeneratedSections += slashSlashComment;
	}

	void StreamTokenizer::clearGeneratedSections(void)
	{
		for (int i=_lstGeneratedSections.getLength();i--;)
			delete _lstGeneratedSections[i];
		_lstGeneratedSections.clear();
	}

	void StreamTokenizer::setDecimalSeparator(unsigned char separator)
	{
		//_iaAttributeMap[_cDecimalSeparator] &= ~TOKEN_NUMBER;
		_cDecimalSeparator = separator;
		//_iaAttributeMap[_cDecimalSeparator] |= TOKEN_NUMBER;
	}
	
	void StreamTokenizer::setAttribute(unsigned char character, int attr)
	{
		_iaAttributeMap[character] = attr;
	}

	StreamTokenizer::Token* StreamTokenizer::getNextToken(istream& input) throw (IOException&)
	{
		unsigned char chr;

		//Discard all spaces
		do
			{
				input.read((char*)&chr,1);
				//cerr << "Read " << chr << endl;
				if (!input)
					throw EOFException("StreamTokenizer::getNextToken(): no more tokens");
			} while (_iaAttributeMap[chr] & TOKEN_SPACE);

		//We have encountered a section start character
		if (_iaAttributeMap[chr] & TOKEN_SECTION)
			{
				//cerr << "This starts a section." << endl;
				//Let's see what kind of sections this character can start
				List<Section*> lst;
				for (int i=_lstSections.getLength();i--;)
					if (_lstSections[i]->start()[0] == chr)
						lst += _lstSections[i];

				if (lst.getLength())
					{
						//cerr << "Found " << lst.getLength() << " sections." << endl;
						//Create a string for the section start comparison
						string sectionStart(1,chr);

						//Read as many characters as there can be in a sequence start string
						int maxLen = _iMaxStartLength;
						bool eof = false;
						while (--maxLen)
							{
								input.read((char*)&chr,1);
								if (!input)
									{
										eof = true;
										break;
									}
								sectionStart += chr;
							}

						//cerr << "Read " << _iMaxStartLength << " characters." << endl;

						//Find the longest match for the sequence start string
						int maxMatchLen = 0, matchIndex = -1;
						for (int i=0;i<lst.getLength();i++)
							{
								int len = lst[i]->start().length();
								if (len >= maxMatchLen &&
										!lst[i]->compare(lst[i]->start().data(),sectionStart.data(),len))
									{
										maxMatchLen = len;
										matchIndex = i;
									}
							}
						//Return unused characters to the stream
						input.clear();
						for (int i=sectionStart.size()-1;i>=maxMatchLen;i--)
							input.putback(sectionStart[i]);

						//If we found a match
						if (matchIndex != -1)
							{
								//cerr << "Match found. Reading section." << endl;
								//Let the section read itself
								return new SectionToken(lst[matchIndex],
																				lst[matchIndex]->readSection(input));
							}
						input.read((char*)&chr,1);
					}
				//There were no sequences starting with this character (should not happen)
				else
					input.putback(chr);
			}

		//The character was neither a space nor a section start.
		//Next, see if it is "normal", i.e. does not constitute a word
		//or number.
		if (_iaAttributeMap[chr] & TOKEN_NORMAL)
			{
				//cerr << "This has no special meaning." << endl;
				bool number = false;
				if (_iaAttributeMap[chr] & TOKEN_NUMBER) //if it could start a number, too
					{
						unsigned char tmp;
						input.read((char*)&tmp,1);
						if (input)
							{
								if (_iaAttributeMap[tmp] & TOKEN_NUMBER &&
										!(_iaAttributeMap[tmp] & TOKEN_NORMAL))
									number = true;
								input.putback(tmp);
							}
					}
				if (!number)
					return new NormalToken(chr);
			}

		CharList lst(24);

		//Read a number
		if (_iaAttributeMap[chr] & TOKEN_NUMBER)
			{
				double number;
				input.putback(chr);
				//store number start position
				int start = input.rdbuf()->pubseekoff(0,input.cur);
				input >> number; //read it
				//store number end position
				int end = input.rdbuf()->pubseekoff(0,input.cur);
				int len = end-start;
				//read the number as a string from the read buffer
				input.rdbuf()->pubseekoff(-len,input.cur);
				char bfr[len+1];
				input.rdbuf()->sgetn(bfr,len);
				bfr[len] = 0;
				if (!input)
					input.clear();
				else
					return new NumberToken(bfr,number);
			}
		//Finally, if nothing else applies, this is a word.

		if (_iaAttributeMap[chr] & TOKEN_WORD)
			{
				//cerr << "This starts a word." << endl;
				do
					{
						lst += chr;
						input.read((char*)&chr,1);
						if (!input)
							break;
						//cerr << "Read " << chr << endl;
					} while (_iaAttributeMap[chr] & TOKEN_WORD);
				if (input) input.putback(chr);
				lst += 0;
				return new WordToken(string(lst.getData()));
			}
		return new NormalToken(chr);
	}

	
	Section::Section(const string& name,
									 const string& start)
	{
		_strName = name;
		_strStart = start;
		setCaseSensitive(true);
		for (int i=0;i<256;i++)
			_baReplace[i] = false;
	}

	void Section::setCaseSensitive(bool sensitive)
	{
		_bCaseSensitive = sensitive;
		//_cmpFunc = (sensitive)? strncmp : strncasecmp;
	}

	void Section::addReplacement(const string& from, const string& to)
	{
		unsigned char start = (unsigned char)from[0];
		_baReplace[start] = true;
		Pair<string,string> replacement(from,to);
		_lstReplacements += replacement;
		if (from.length() > _iMaxReplacementLength)
			_iMaxReplacementLength = from.length();
	}

	bool Section::checkReplace(unsigned char& chr, istream& input, string& result)
	{
		unsigned char tmp;
		input.read((char*)&chr,1);
		if (!input)
			return false;
		if (!_baReplace[chr])
			return false;
		tmp = chr;
		string str;
		str += tmp;
		for (int i=_iMaxReplacementLength-1;i--;)
			{
				input.read((char*)&tmp,1);
				if (!input)
					{
						input.clear();
						break;
					}
				str += tmp;
			}
		for (unsigned int i=str.size()-1;i>=1;i--)
			{
				//cerr << str << endl;
				for (int j=_lstReplacements.getLength();j--;)
					if (_lstReplacements[j].first().length() == i && !strncmp(str.data(),_lstReplacements[j].first().data(),i))
						{
							result += _lstReplacements[j].second();
							return true;
						}

				input.putback(str[i]);
			}
		return false;
	}


	string EOLSection::readSection(istream& input) throw (IOException&)
	{
		unsigned char chr;
		string result;

		do
			{
				if (!checkReplace(chr,input,result) && input)
					{
						if (chr == '\r')
							{
								input.read((char*)&chr,1);
								if (!input || chr == '\n')
									break;
							}
						else if (chr == '\n')
							break;
						result += chr;
					}
			} while(input);

		return result;
	}

	EndDelimitedSection::EndDelimitedSection(const string& name,
																					 const string& start,
																					 const string& end) : Section(name,start)
	{
		_strEnd = end;
	}

	string EndDelimitedSection::readSection(istream& input) throw (IOException&)
	{
		int len = _strEnd.size()-1;
		unsigned char stop = (unsigned char)_strEnd[0];
		string result;
		while(true)
			{
				do
					{
						unsigned char tmp;
						bool replaced = checkReplace(tmp,input,result);
						//cerr << "Read " << tmp << ", ";
						//cerr << "input: " << (input?"true":"false") << " replaced: " << (replaced?"true":"false") << endl;
						if (input && !replaced)
							{
								if (tmp != stop)
									result += tmp;
								else
									break;
							}
					} while (input);
				if (!input)
					throw EOFException("EndDelimitedSection::readSection(): unexcepted EOF while reading section contents.");
				if (!len) //only one stop character -> no need to construct comparison string
					break;
				string cmp;
				cmp += stop;
				for (int i=len;i--;)
					{
						unsigned char tmp;
						input.read((char*)&tmp,1);
						if (!input)
							throw EOFException("EndDelimitedSection::readSection(): unexcepted EOF while reading section end mark.");
						cmp += tmp;
					}
				if (_strEnd == cmp)
					break;

				result += cmp;
			}
		return result;
	}
}