civquest.parser
Class Tokenizer

java.lang.Object
  civquest.parser.Tokenizer

public abstract class Tokenizer

extends java.lang.Object

A common tokenizers for parsers in the CivQuest project.

Field Summary

protected int	col
protected java.lang.String	curLine
protected boolean	eof
protected int	line
protected java.util.ArrayList	lines
protected Token	prevToken
protected boolean	pushedBack

Constructor Summary

Tokenizer()

Method Summary

protected boolean	eol()
int	getCol()
protected java.lang.String	getIdentifier()
int	getLine()
protected java.lang.String	getNumericValue()
protected java.lang.String	getQuotedValue()
protected char	nextChar()
protected void	nextLine()
Token	nextToken()
Token	nextToken(boolean goto_next)
char	peek()
void	pushBack() Make the previous token the current one (if client read one too far)
protected void	rollback(int amount)
protected void	skipComment()
protected char	thisChar()

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

lines

protected java.util.ArrayList lines

col

protected int col

line

protected int line

curLine

protected java.lang.String curLine

pushedBack

protected boolean pushedBack

prevToken

protected Token prevToken

eof

protected boolean eof

Constructor Detail

Tokenizer

public Tokenizer()

Method Detail

getLine

public int getLine()

getCol

public int getCol()

nextLine

protected void nextLine()

nextChar

protected char nextChar()

thisChar

protected char thisChar()

eol

protected boolean eol()

rollback

protected void rollback(int amount)

pushBack

public void pushBack()

Make the previous token the current one (if client read one too far)

nextToken

public Token nextToken()

Returns:

the next token

nextToken

public Token nextToken(boolean goto_next)

skipComment

protected void skipComment()

getQuotedValue

protected java.lang.String getQuotedValue()

getIdentifier

protected java.lang.String getIdentifier()

getNumericValue

protected java.lang.String getNumericValue()

peek

public char peek()

Returns:

the next character in the input stream, without consuming it