uses data.String

data ParseToken {
	const int TYPE_PARTICLE			= 0
	const int TYPE_LITERAL_STRING 	= 1
	const int TYPE_LITERAL_NUMBER 	= 2
	const int TYPE_LINE_COMMENT 	= 3
	const int TYPE_BLOCK_COMMENT 	= 4
	const int TYPE_TOKEN 			= 5
	
	int type
	char content[]
	
	int sourceStart
	int sourceLength
	}

/*
{"description" : "This interface is used to tokenise a text document, using a given set of tokens, and supporting both line and block comments.", "@deprecated" : "true", "@deprecatedBy" : "parsing.Tokeniser", "@deprecatedOn" : "2020/01/01"}
*/

interface Tokeniser {
	/*
	{"@description" : "Initialise a tokeniser instance with the given set of tokens.",
		"tokens" : "The tokens that will be used by the tokeniser to separate text."}
	*/
	Tokeniser(store String tokens[])
	
	/*
	{"@description" : "Set the string that indicates the remainder of a line is a comment.",
		"lineComment" : "A string that indicates the remainder of a line is a comment."}
	*/
	void setLineComment(char lineComment[])
	
	/*
	{"@description" : "Set the starting and ending string that indicate the enclosed text is a comment.",
		"start" : "A string that indicates the start of a block comment.",
		"end" : "A string that indicates the end of a block comment."}
	*/
	void setBlockComment(char start[], char end[])
	
	/*
	{"@description" : "Run the tokeniser, returning an array of all tokens.",
		"content" : "The text to split into tokens.",
		"@return" : "An array of tokens."}
	*/
	ParseToken[] tokenise(char content[])
	}