- Revision
- 580
- Author
- rfscholte
- Date
- 2009-03-10 17:05:36 -0500 (Tue, 10 Mar 2009)
Log Message
qdox-140 bugfix: unicode support
Modified Paths
Diff
Modified: trunk/qdox/src/grammar/lexer.flex (579 => 580)
--- trunk/qdox/src/grammar/lexer.flex 2009-03-09 20:20:51 UTC (rev 579) +++ trunk/qdox/src/grammar/lexer.flex 2009-03-10 22:05:36 UTC (rev 580) @@ -86,7 +86,8 @@ ( "." [0-9]+ ({Exponent})? [dD] ) | ( ([0-9])+ {Exponent} [dD] ) | ( ([0-9])+ ({Exponent})? [dD] ) -Id = [:jletter:] [:jletterdigit:]* +UnicodeChar = \\u[a-fA-F0-9]{4} +Id = ([:jletter:]|{UnicodeChar}) ([:jletterdigit:]|{UnicodeChar})* %state JAVADOC CODEBLOCK PARENBLOCK ASSIGNMENT STRING CHAR SINGLELINECOMMENT MULTILINECOMMENT ANNOTATION ANNOSTRING ANNOCHAR ENUM
Modified: trunk/qdox/src/test/com/thoughtworks/qdox/parser/LexerTest.java (579 => 580)
--- trunk/qdox/src/test/com/thoughtworks/qdox/parser/LexerTest.java 2009-03-09 20:20:51 UTC (rev 579) +++ trunk/qdox/src/test/com/thoughtworks/qdox/parser/LexerTest.java 2009-03-10 22:05:36 UTC (rev 580) @@ -653,7 +653,7 @@ assertEquals(expectedText, lexer.text()); } - // TODO make this work .... QDOX-140 + // for QDOX-140 public void testNonAsciiMethodNameDoesNotCrashLexerButChewsUpUnicodeEscapedSequencesBadly() throws Exception { String in = "" + "interface X { " @@ -665,34 +665,11 @@ assertLex(Parser.IDENTIFIER, lexer); assertLex(Parser.BRACEOPEN, lexer); assertLex(Parser.IDENTIFIER, "void", lexer); - assertLex(Parser.IDENTIFIER, "paramWithNonAsciis", lexer); - assertLex(Parser.IDENTIFIER, "u00E4", lexer); - assertLex(Parser.IDENTIFIER, "u00F6", lexer); - assertLex(Parser.IDENTIFIER, "u00FC", lexer); - assertLex(Parser.IDENTIFIER, "u00DF", lexer); + assertLex(Parser.IDENTIFIER, "paramWithNonAsciis\\u00E4\\u00F6\\u00FC\\u00DF", lexer); assertLex(Parser.PARENOPEN, lexer); assertLex(Parser.PARENCLOSE, lexer); assertLex(Parser.SEMI, lexer); assertLex(Parser.BRACECLOSE, lexer); assertLex(0, lexer); } - - // for QDOX-140 - public void todo_testReadFileWithUnicode() throws Exception { - URL url = "" - - //this line is copied from JavaDocBuilder.addSource(URL), because that part immediately starts parsing - Reader reader = new InputStreamReader(url.openStream(),System.getProperty("file.encoding")); - - StringBuffer buffer = new StringBuffer(); - int ch; - while ((ch = reader.read()) > -1) { - buffer.append((char)ch); - } - reader.close(); - String result = buffer.toString(); - assertEquals(false, result.indexOf("\\u00E4") > -1); - assertEquals(true, result.indexOf("\u00E4") > -1); - } - }
To unsubscribe from this list please visit:
