Author: sebb
Date: Wed Mar 21 19:04:32 2012
New Revision: 1303505
URL: http://svn.apache.org/viewvc?rev=1303505&view=rev
Log:
Move Token into separate file for more flexibility
Modified:
commons/proper/csv/trunk/src/main/java/org/apache/commons/csv/CSVLexer.java
commons/proper/csv/trunk/src/main/java/org/apache/commons/csv/CSVParser.java
commons/proper/csv/trunk/src/test/java/org/apache/commons/csv/CSVLexerTest.java
Modified:
commons/proper/csv/trunk/src/main/java/org/apache/commons/csv/CSVLexer.java
URL:
http://svn.apache.org/viewvc/commons/proper/csv/trunk/src/main/java/org/apache/commons/csv/CSVLexer.java?rev=1303505&r1=1303504&r2=1303505&view=diff
==============================================================================
--- commons/proper/csv/trunk/src/main/java/org/apache/commons/csv/CSVLexer.java
(original)
+++ commons/proper/csv/trunk/src/main/java/org/apache/commons/csv/CSVLexer.java
Wed Mar 21 19:04:32 2012
@@ -19,13 +19,10 @@ package org.apache.commons.csv;
import java.io.IOException;
-import static org.apache.commons.csv.CSVLexer.Token.Type.*;
+import static org.apache.commons.csv.Token.Type.*;
class CSVLexer {
- /** length of the initial token (content-)buffer */
- private static final int INITIAL_TOKEN_LENGTH = 50;
-
private final StringBuilder wsBuf = new StringBuilder();
private final CSVFormat format;
@@ -33,44 +30,6 @@ class CSVLexer {
/** The input stream */
private final ExtendedBufferedReader in;
- /**
- * Token is an internal token representation.
- * <p/>
- * It is used as contract between the lexer and the parser.
- */
- static class Token {
-
- enum Type {
- /** Token has no valid content, i.e. is in its initialized state.
*/
- INVALID,
-
- /** Token with content, at beginning or in the middle of a line. */
- TOKEN,
-
- /** Token (which can have content) when end of file is reached. */
- EOF,
-
- /** Token with content when end of a line is reached. */
- EORECORD
- }
-
- /** Token type */
- Type type = INVALID;
-
- /** The content buffer. */
- StringBuilder content = new StringBuilder(INITIAL_TOKEN_LENGTH);
-
- /** Token ready flag: indicates a valid token with content (ready for
the parser). */
- boolean isReady;
-
- Token reset() {
- content.setLength(0);
- type = INVALID;
- isReady = false;
- return this;
- }
- }
-
CSVLexer(CSVFormat format, ExtendedBufferedReader in) {
this.format = format;
this.in = in;
Modified:
commons/proper/csv/trunk/src/main/java/org/apache/commons/csv/CSVParser.java
URL:
http://svn.apache.org/viewvc/commons/proper/csv/trunk/src/main/java/org/apache/commons/csv/CSVParser.java?rev=1303505&r1=1303504&r2=1303505&view=diff
==============================================================================
---
commons/proper/csv/trunk/src/main/java/org/apache/commons/csv/CSVParser.java
(original)
+++
commons/proper/csv/trunk/src/main/java/org/apache/commons/csv/CSVParser.java
Wed Mar 21 19:04:32 2012
@@ -27,9 +27,8 @@ import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
-import org.apache.commons.csv.CSVLexer.Token;
-import static org.apache.commons.csv.CSVLexer.Token.Type.*;
+import static org.apache.commons.csv.Token.Type.*;
/**
* Parses CSV files according to the specified configuration.
Modified:
commons/proper/csv/trunk/src/test/java/org/apache/commons/csv/CSVLexerTest.java
URL:
http://svn.apache.org/viewvc/commons/proper/csv/trunk/src/test/java/org/apache/commons/csv/CSVLexerTest.java?rev=1303505&r1=1303504&r2=1303505&view=diff
==============================================================================
---
commons/proper/csv/trunk/src/test/java/org/apache/commons/csv/CSVLexerTest.java
(original)
+++
commons/proper/csv/trunk/src/test/java/org/apache/commons/csv/CSVLexerTest.java
Wed Mar 21 19:04:32 2012
@@ -20,10 +20,9 @@ package org.apache.commons.csv;
import java.io.IOException;
import java.io.StringReader;
-import org.apache.commons.csv.CSVLexer.Token;
import org.junit.Test;
-import static org.apache.commons.csv.CSVLexer.Token.Type.*;
+import static org.apache.commons.csv.Token.Type.*;
import static org.junit.Assert.*;
public class CSVLexerTest {
