public abstract class AbstractTokenizer extends java.lang.Object implements Tokenizer
Modifier and Type | Field and Description |
---|---|
protected java.util.List<java.lang.String> |
ignorableCharacter |
protected java.util.List<java.lang.String> |
ignorableStmt |
protected char |
ONE_LINE_COMMENT_CHAR |
protected boolean |
spanMultipleLinesString |
protected java.util.List<java.lang.String> |
stringToken |
Constructor and Description |
---|
AbstractTokenizer() |
Modifier and Type | Method and Description |
---|---|
void |
tokenize(SourceCode tokens,
Tokens tokenEntries) |
protected java.util.List<java.lang.String> stringToken
protected java.util.List<java.lang.String> ignorableCharacter
protected java.util.List<java.lang.String> ignorableStmt
protected char ONE_LINE_COMMENT_CHAR
protected boolean spanMultipleLinesString
public void tokenize(SourceCode tokens, Tokens tokenEntries)