User: dsundstrom
Date: 01/08/06 17:18:31
Added: src/main/org/jboss/ejb/plugins/cmp/ejbql
ApproximateNumericLiteral.java
EJBQLNumberState.java EJBQLQuoteState.java
EJBQLWordState.java ExactNumericLiteral.java
StringLiteral.java SymbolState.java
SymbolToken.java Token.java Tokenizer.java
TokenizerState.java WhitespaceState.java
WordToken.java
Log:
Initial revision of a tokenizer designed for EJB-QL.
Revision Changes Path
1.1
jboss/src/main/org/jboss/ejb/plugins/cmp/ejbql/ApproximateNumericLiteral.java
Index: ApproximateNumericLiteral.java
===================================================================
package org.jboss.ejb.plugins.cmp.ejbql;
public class ApproximateNumericLiteral implements Token {
private final double value;
public ApproximateNumericLiteral(double value) {
this.value = value;
}
public double getValue() {
return value;
}
public int hashCode() {
long l = Double.doubleToLongBits(value);
return 37 * 17 + (int)(l ^ (l >>> 32));
}
public boolean equals(Object o) {
if(o instanceof ApproximateNumericLiteral) {
return ((ApproximateNumericLiteral)o).value == value;
}
return false;
}
public String toString() {
return "" + value;
}
}
1.1
jboss/src/main/org/jboss/ejb/plugins/cmp/ejbql/EJBQLNumberState.java
Index: EJBQLNumberState.java
===================================================================
package org.jboss.ejb.plugins.cmp.ejbql;
import java.io.PushbackReader;
import java.io.IOException;
import java.io.CharArrayWriter;
public class EJBQLNumberState implements TokenizerState {
private TokenizerState symbolState = new SymbolState();
public EJBQLNumberState() {
}
public Token nextToken(PushbackReader in, char c, Tokenizer tokenizer)
throws IOException {
if(!Character.isDigit(c) && c != '.') {
throw new IllegalArgumentException("EJBQLWordState must begin
with a digit or a '.': c="+c);
}
// do we just have a peroid
if(c == '.') {
int peek = peekChar(in);
if(!Character.isDigit((char)peek)) {
// deligate to the symbolState
return symbolState.nextToken(in, c, tokenizer);
}
}
//
// Now we are definately working on a number
//
// put the first characte back on, makes the code easier to write
in.unread(c);
// output buffer
CharArrayWriter out = new CharArrayWriter(16);
// whole number part
readWholeNumberPart(in, out);
// fractional part
readFractionalPart(in, out);
// exponent part
readExponentPart(in, out);
// exponent part
readSuffix(in, out);
String number = out.toString().toLowerCase();
System.out.println("number is ["+number+"]");
if(isExactNumeric(number)) {
return createExactNumericLiteral(number);
} else {
return createApproximateNumericLiteral(number);
}
}
private void readWholeNumberPart(PushbackReader in, CharArrayWriter out)
throws IOException {
int first = peekChar(in);
if(Character.isDigit((char)first)) {
// read the first digit off the stream and write it out
out.write(in.read());
// is it a hex number
int second = peekChar(in);
System.out.println("HEX? " + (char)first + (char)second);
if(first == '0' && (second == 'x' || second == 'X')) {
// read the x off the stream and write it out
out.write(in.read());
readNumber(in, out, 16);
} else {
// can't check for octal yet, because we don't
// know if this is a float number yet, but dec will
work for now
readNumber(in, out, 10);
}
}
}
private void readFractionalPart(PushbackReader in, CharArrayWriter out) throws
IOException {
int peek = peekChar(in);
if(peek == '.') {
// read the peroid off the stream and write it out
out.write(in.read());
// get all the decimal digits
readNumber(in, out, 10);
System.out.println("Read fractional part numebr is
["+out.toString()+"]");
}
}
private void readExponentPart(PushbackReader in, CharArrayWriter out) throws
IOException {
int peek = peekChar(in);
if(peek == 'e' || peek == 'E') {
// read the e off the stream and write it out
out.write(in.read());
// check for a sign in the exponent
peek = peekChar(in);
if(peek == '+' || peek == '-') {
// read the sign off the stream and write it out
out.write(in.read());
}
// read the integer
readNumber(in, out, 10);
}
}
private void readSuffix(PushbackReader in, CharArrayWriter out) throws
IOException {
int peek = peekChar(in);
if(peek == 'l' || peek == 'L' ||
peek == 'f' || peek == 'F' ||
peek == 'd' || peek == 'D') {
// read the suffix off the stream and write it out
out.write(in.read());
}
}
private void readNumber(PushbackReader in, CharArrayWriter out, int radix)
throws IOException {
// which the read character is a digit in the specified radix
int c = in.read();
while(Character.digit((char)c, radix) != -1) {
out.write(c);
c = in.read();
}
// unread that last character because it is not a digit
if(c != -1) {
in.unread(c);
}
}
private boolean isExactNumeric(String number) {
// is this a hexadecimal number
if(number.startsWith("0x") || number.startsWith("0X")){
return true;
}
// does it contain a peroid
if(number.indexOf('.')>=0) {
return false;
}
// does it contain an exponent
if(number.indexOf('e')>=0 || number.indexOf('E')>=0) {
return false;
}
// does it end with an f
if(number.endsWith("f") || number.endsWith("F")) {
return false;
}
// does it end with a d
if(number.endsWith("d") || number.endsWith("D")) {
return false;
}
return true;
}
/**
* This function is broken. It does not support bit field style
* integers and longs 0xffffffff
*/
private ExactNumericLiteral createExactNumericLiteral(String number) throws
IOException {
// long suffix
if(number.endsWith("l") || number.endsWith("L")) {
// chop off the suffix
number = number.substring(0, number.length()-1);
System.out.println("decode long number ["+number+"]");
return new
ExactNumericLiteral(Long.decode(number).longValue());
} else {
return new
ExactNumericLiteral(Integer.decode(number.toUpperCase()).intValue());
}
}
private ApproximateNumericLiteral createApproximateNumericLiteral(String
number) throws IOException {
// float suffix
if(number.endsWith("f") || number.endsWith("F")) {
// chop off the suffix
number = number.substring(0, number.length()-1);
return new ApproximateNumericLiteral(Float.parseFloat(number));
}
// ends with a d suffix, chop it off
if(number.endsWith("d") || number.endsWith("D")) {
number = number.substring(0, number.length()-1);
}
// regular double
return new ApproximateNumericLiteral(Double.parseDouble(number));
}
private int peekChar(PushbackReader in) throws IOException {
int nextChar = in.read();
if(nextChar != -1) {
in.unread(nextChar);
}
return nextChar;
}
}
1.1
jboss/src/main/org/jboss/ejb/plugins/cmp/ejbql/EJBQLQuoteState.java
Index: EJBQLQuoteState.java
===================================================================
package org.jboss.ejb.plugins.cmp.ejbql;
import java.io.PushbackReader;
import java.io.IOException;
import java.io.CharArrayWriter;
public class EJBQLQuoteState implements TokenizerState {
private static final char SINGLE_QUOTE = '\'';
public EJBQLQuoteState() {
}
public Token nextToken(PushbackReader in, char character, Tokenizer tokenizer)
throws IOException {
if(character != SINGLE_QUOTE) {
throw new IllegalArgumentException("EJBQLQuoteState can only
tokenize single quoted strings");
}
if(in == null) {
throw new IllegalArgumentException("in is null");
}
CharArrayWriter out = new CharArrayWriter(16);
// the returned token is quoted
out.write(SINGLE_QUOTE);
for(int c = in.read(); c != -1; c = in.read()) {
// check for bad characters
if(c == '\n' || c == '\r') {
throw new IOException("Error: Line terminator in
string literal");
}
//
// Should we check for unicode encoded characters here?
//
out.write(c);
// check for end of string
if(c == SINGLE_QUOTE) {
// we have a possible end, check if the next character
is
// a single quote. If it is, we just have an escaped
single quote.
int nextChar = in.read();
if(nextChar != SINGLE_QUOTE) {
// unread next char if it is not an end of
stream
if(nextChar != -1) {
in.unread(nextChar);
}
return new StringLiteral(out.toString());
}
}
}
throw new IOException("Error: Unterminated string literal");
}
}
1.1
jboss/src/main/org/jboss/ejb/plugins/cmp/ejbql/EJBQLWordState.java
Index: EJBQLWordState.java
===================================================================
package org.jboss.ejb.plugins.cmp.ejbql;
import java.io.PushbackReader;
import java.io.IOException;
import java.io.CharArrayWriter;
public class EJBQLWordState implements TokenizerState {
public EJBQLWordState() {
}
public Token nextToken(PushbackReader in, char character, Tokenizer tokenizer)
throws IOException {
if(!isEJBQLIdentifierStart(character)) {
throw new IllegalArgumentException("EJBQLWordState must begin
with a valid identified start: c="+character);
}
if(in == null) {
throw new IllegalArgumentException("in is null");
}
CharArrayWriter out = new CharArrayWriter(16);
out.write(character);
// read all chars in identifier
int c = in.read();
while(isEJBQLIdentifierPart((char)c)) {
out.write(c);
c = in.read();
}
// unread last char as it is not part of the identifier
if(c != -1) {
in.unread(c);
}
return new WordToken(out.toString());
}
private boolean isEJBQLIdentifierStart(char c) {
return Character.isJavaIdentifierStart(c) || c=='?';
}
private boolean isEJBQLIdentifierPart(char c) {
return Character.isJavaIdentifierPart(c) && c!='?';
}
}
1.1
jboss/src/main/org/jboss/ejb/plugins/cmp/ejbql/ExactNumericLiteral.java
Index: ExactNumericLiteral.java
===================================================================
package org.jboss.ejb.plugins.cmp.ejbql;
public class ExactNumericLiteral implements Token {
private final long value;
public ExactNumericLiteral(long value) {
this.value = value;
}
public long getValue() {
return value;
}
public int hashCode() {
return 37 * 17 + (int)(value ^ (value >>> 32));
}
public boolean equals(Object o) {
if(o instanceof ExactNumericLiteral) {
return ((ExactNumericLiteral)o).value == value;
}
return false;
}
public String toString() {
return "" + value;
}
}
1.1
jboss/src/main/org/jboss/ejb/plugins/cmp/ejbql/StringLiteral.java
Index: StringLiteral.java
===================================================================
package org.jboss.ejb.plugins.cmp.ejbql;
public final class StringLiteral implements Token {
private final String value;
public StringLiteral(String value) {
this.value = value;
}
public int hashCode() {
return 37 * 17 + value.hashCode();
}
public boolean equals(Object o) {
if(o instanceof StringLiteral) {
return ((StringLiteral)o).value.equals(value);
}
return false;
}
public String toString() {
return value;
}
}
1.1 jboss/src/main/org/jboss/ejb/plugins/cmp/ejbql/SymbolState.java
Index: SymbolState.java
===================================================================
package org.jboss.ejb.plugins.cmp.ejbql;
import java.io.PushbackReader;
import java.io.IOException;
/**
* Note: this is not a robust implementation of a SymbolState and only works because
* the ejb-ql symbols are simple
*
*/
public class SymbolState implements TokenizerState {
public SymbolState() {
}
public Token nextToken(PushbackReader in, char character, Tokenizer tokenizer)
throws IOException {
if(character == '<') {
// check for <> or <=
int nextChar = in.read();
if(nextChar == '=') {
return new SymbolToken("<=");
} else if(nextChar == '>') {
return new SymbolToken("<>");
} else {
// nope, unread nextChar
if(nextChar != -1) {
in.unread(nextChar);
}
}
} else if(character == '>') {
// check for >=
int nextChar = in.read();
if(nextChar == '=') {
return new SymbolToken(">=");
} else {
// nope, unread nextChar
if(nextChar != -1) {
in.unread(nextChar);
}
}
}
return new SymbolToken(""+character);
}
}
1.1 jboss/src/main/org/jboss/ejb/plugins/cmp/ejbql/SymbolToken.java
Index: SymbolToken.java
===================================================================
package org.jboss.ejb.plugins.cmp.ejbql;
public final class SymbolToken implements Token {
private final String value;
public SymbolToken(String value) {
this.value = value;
}
public int hashCode() {
return 37 * 17 + value.hashCode();
}
public boolean equals(Object o) {
if(o instanceof SymbolToken) {
return ((SymbolToken)o).value.equals(value);
}
return false;
}
public String toString() {
return value;
}
}
1.1 jboss/src/main/org/jboss/ejb/plugins/cmp/ejbql/Token.java
Index: Token.java
===================================================================
package org.jboss.ejb.plugins.cmp.ejbql;
public interface Token {
}
1.1 jboss/src/main/org/jboss/ejb/plugins/cmp/ejbql/Tokenizer.java
Index: Tokenizer.java
===================================================================
package org.jboss.ejb.plugins.cmp.ejbql;
import java.io.Reader;
import java.io.PushbackReader;
import java.io.IOException;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.NoSuchElementException;
public class Tokenizer implements Iterator {
private static final TokenizerState NUMBER_STATE = new EJBQLNumberState();
private static final TokenizerState QUOTE_STATE = new EJBQLQuoteState();
private static final TokenizerState SYMBOL_STATE = new SymbolState();
private static final TokenizerState WHITESPACE_STATE = new WhitespaceState();
private static final TokenizerState WORD_STATE = new EJBQLWordState();
private Map tokenizerStates = new HashMap(256);
private TokenizerState defaultState;
private PushbackReader reader;
private Token peekedToken;
public Tokenizer() {
initDefaultStates();
}
public TokenizerState getCharacterState(char character) {
TokenizerState s = (TokenizerState)tokenizerStates.get(new
Character(character));
if(s != null) {
return s;
}
return SYMBOL_STATE;
}
public void setCharacterState(char character, TokenizerState state) {
tokenizerStates.put(new Character(character), state);
}
public Reader getReader() {
return reader;
}
public void setReader(Reader reader){
this.reader = new PushbackReader(reader, 32);
}
public boolean hasNext() {
return peekToken() != null;
}
public Object next() {
return nextToken();
}
public Token nextToken() {
Token n = peekToken();
if(n == null) {
throw new NoSuchElementException();
}
peekedToken = null;
return n;
}
private Token peekToken() {
if(peekedToken == null) {
try{
int c = reader.read();
if(c == -1) {
return null;
}
TokenizerState state = getCharacterState((char)c);
peekedToken = state.nextToken(reader, (char)c, this);
} catch(IOException e) {
throw new RuntimeException(e.getMessage());
}
}
return peekedToken;
}
public void remove() {
throw new UnsupportedOperationException("Tokenier does not support
remove");
}
protected void initDefaultStates() {
for(int i=0; i<(int)Character.MAX_VALUE; i++) {
if(Character.isJavaIdentifierStart((char)i)) {
tokenizerStates.put(new Character((char)i),
WORD_STATE);
}
if(Character.isWhitespace((char)i)) {
tokenizerStates.put(new Character((char)i),
WHITESPACE_STATE);
}
if(Character.isDigit((char)i)) {
tokenizerStates.put(new Character((char)i),
NUMBER_STATE);
}
}
tokenizerStates.put(new Character('\''), QUOTE_STATE);
tokenizerStates.put(new Character('?'), WORD_STATE);
tokenizerStates.put(new Character('.'), NUMBER_STATE);
}
}
1.1
jboss/src/main/org/jboss/ejb/plugins/cmp/ejbql/TokenizerState.java
Index: TokenizerState.java
===================================================================
package org.jboss.ejb.plugins.cmp.ejbql;
import java.io.PushbackReader;
import java.io.IOException;
public interface TokenizerState {
public Token nextToken(PushbackReader reader, char character, Tokenizer
tokenizer)
throws IOException;
}
1.1
jboss/src/main/org/jboss/ejb/plugins/cmp/ejbql/WhitespaceState.java
Index: WhitespaceState.java
===================================================================
package org.jboss.ejb.plugins.cmp.ejbql;
import java.io.PushbackReader;
import java.io.IOException;
public class WhitespaceState implements TokenizerState {
public WhitespaceState() {
}
public Token nextToken(PushbackReader in, char character, Tokenizer tokenizer)
throws IOException {
for(int c = in.read(); c != -1; c = in.read()) {
// get the state for the next character
TokenizerState state = tokenizer.getCharacterState((char)c);
// if we are NOT the state for the next character, call the
// next state, else loop
if(state != this) {
return state.nextToken(in, (char)c, tokenizer);
}
}
// white space to the end of the stream
return null;
}
}
1.1 jboss/src/main/org/jboss/ejb/plugins/cmp/ejbql/WordToken.java
Index: WordToken.java
===================================================================
package org.jboss.ejb.plugins.cmp.ejbql;
public final class WordToken implements Token {
private final String value;
public WordToken(String value) {
this.value = value;
}
public int hashCode() {
return 37 * 17 + value.hashCode();
}
public boolean equals(Object o) {
if(o instanceof WordToken) {
return ((WordToken)o).value.equals(value);
}
return false;
}
public String toString() {
return value;
}
}
_______________________________________________
Jboss-development mailing list
[EMAIL PROTECTED]
http://lists.sourceforge.net/lists/listinfo/jboss-development
