This is an automated email from the ASF dual-hosted git repository.
tilman pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/main by this push:
new 08ebe87ab TIKA-4278: refactor delimiters as a set to include semicolon
pipe and colon, code cleanup
08ebe87ab is described below
commit 08ebe87ab0330362d5d01d8ad809fc5fb6e2d99e
Author: Tilman Hausherr <[email protected]>
AuthorDate: Mon Jul 15 05:44:33 2024 +0200
TIKA-4278: refactor delimiters as a set to include semicolon pipe and
colon, code cleanup
---
.../tika-parser-text-module/pom.xml | 5 +++
.../org/apache/tika/parser/csv/CSVSniffer.java | 8 ++---
.../apache/tika/parser/csv/TextAndCSVParser.java | 20 ++++-------
.../org/apache/tika/parser/csv/CSVSnifferTest.java | 42 +++++++++++++++-------
4 files changed, 45 insertions(+), 30 deletions(-)
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/pom.xml
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/pom.xml
index f0df94415..6621b0486 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/pom.xml
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/pom.xml
@@ -42,6 +42,11 @@
<groupId>org.apache.commons</groupId>
<artifactId>commons-csv</artifactId>
</dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/csv/CSVSniffer.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/csv/CSVSniffer.java
index 52c7f79ce..53c9e4cb5 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/csv/CSVSniffer.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/csv/CSVSniffer.java
@@ -27,6 +27,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import org.apache.commons.io.input.ProxyReader;
@@ -42,15 +43,15 @@ class CSVSniffer {
private static final int PUSH_BACK = 2;
private static final int SPACE = ' ';
- private final char[] delimiters;
+ private final Set<Character> delimiters;
private final int markLimit;
private final double minConfidence;
- CSVSniffer(char[] delimiters) {
+ CSVSniffer(Set<Character> delimiters) {
this(DEFAULT_MARK_LIMIT, delimiters, DEFAULT_MIN_CONFIDENCE);
}
- CSVSniffer(int markLimit, char[] delimiters, double minConfidence) {
+ CSVSniffer(int markLimit, Set<Character> delimiters, double minConfidence)
{
this.markLimit = markLimit;
this.delimiters = delimiters;
this.minConfidence = minConfidence;
@@ -310,7 +311,6 @@ class CSVSniffer {
throw new EOFException();
}
unread(reader, c);
- return;
}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/csv/TextAndCSVParser.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/csv/TextAndCSVParser.java
index 462167d14..b98bc4a45 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/csv/TextAndCSVParser.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/csv/TextAndCSVParser.java
@@ -95,8 +95,6 @@ public class TextAndCSVParser extends
AbstractEncodingDetectorParser {
private static final String TABLE = "table";
private static final int DEFAULT_MARK_LIMIT = 20000;
- private static final char[] DEFAULT_DELIMITERS = new char[]{',', '\t'};
-
private static final Map<Character, String> CHAR_TO_STRING_DELIMITER_MAP =
new HashMap<>();
private static final Map<String, Character> STRING_TO_CHAR_DELIMITER_MAP =
new HashMap<>();
private static final Set<MediaType> SUPPORTED_TYPES = Collections
@@ -116,8 +114,6 @@ public class TextAndCSVParser extends
AbstractEncodingDetectorParser {
}
}
- private char[] delimiters = DEFAULT_DELIMITERS;
-
/**
* This is the mark limit in characters (not bytes) to
* read from the stream when classifying the stream as
@@ -135,7 +131,6 @@ public class TextAndCSVParser extends
AbstractEncodingDetectorParser {
private double minConfidence = 0.50;
public TextAndCSVParser() {
- super();
}
public TextAndCSVParser(EncodingDetector encodingDetector) {
@@ -159,10 +154,7 @@ public class TextAndCSVParser extends
AbstractEncodingDetectorParser {
if (mediaType == null) {
return false;
}
- if (mediaType.getBaseType().equals(TSV) ||
mediaType.getBaseType().equals(CSV)) {
- return true;
- }
- return false;
+ return mediaType.getBaseType().equals(TSV) ||
mediaType.getBaseType().equals(CSV);
}
@Override
@@ -175,8 +167,8 @@ public class TextAndCSVParser extends
AbstractEncodingDetectorParser {
ParseContext context) throws IOException, SAXException,
TikaException {
CSVParams params = getOverride(metadata);
- Reader reader = null;
- Charset charset = null;
+ Reader reader;
+ Charset charset;
if (!params.isComplete()) {
reader = detect(params, stream, metadata, context);
if (params.getCharset() != null) {
@@ -294,7 +286,7 @@ public class TextAndCSVParser extends
AbstractEncodingDetectorParser {
getEncodingDetector(context));
}
}
- Reader reader = null;
+ Reader reader;
if (params.getCharset() == null) {
reader = new AutoDetectReader(CloseShieldInputStream.wrap(stream),
metadata,
getEncodingDetector(context));
@@ -310,7 +302,7 @@ public class TextAndCSVParser extends
AbstractEncodingDetectorParser {
if (params.getDelimiter() == null &&
(params.getMediaType() == null ||
isCSVOrTSV(params.getMediaType()))) {
- CSVSniffer sniffer = new CSVSniffer(markLimit, delimiters,
minConfidence);
+ CSVSniffer sniffer = new CSVSniffer(markLimit,
CHAR_TO_STRING_DELIMITER_MAP.keySet(), minConfidence);
CSVResult result = sniffer.getBest(reader, metadata);
params.setMediaType(result.getMediaType());
params.setDelimiter(result.getDelimiter());
@@ -376,7 +368,7 @@ public class TextAndCSVParser extends
AbstractEncodingDetectorParser {
// deprecated, see TIKA-431
metadata.set(Metadata.CONTENT_ENCODING,
params.getCharset().name());
}
- if (!mediaType.equals(MediaType.TEXT_PLAIN) && params.getDelimiter()
!= null) {
+ if (!MediaType.TEXT_PLAIN.equals(mediaType) && params.getDelimiter()
!= null) {
if
(CHAR_TO_STRING_DELIMITER_MAP.containsKey(params.getDelimiter())) {
attrs.put(DELIMITER,
CHAR_TO_STRING_DELIMITER_MAP.get(params.getDelimiter()));
} else {
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/test/java/org/apache/tika/parser/csv/CSVSnifferTest.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/test/java/org/apache/tika/parser/csv/CSVSnifferTest.java
index 338593901..9e9e2c024 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/test/java/org/apache/tika/parser/csv/CSVSnifferTest.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/test/java/org/apache/tika/parser/csv/CSVSnifferTest.java
@@ -27,7 +27,9 @@ import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import java.util.Set;
+import com.google.common.collect.ImmutableSet;
import org.junit.jupiter.api.Test;
import org.apache.tika.TikaTest;
@@ -35,30 +37,38 @@ import org.apache.tika.mime.MediaType;
public class CSVSnifferTest extends TikaTest {
- private static char[] DELIMITERS = new char[]{',', '\t'};
+ private static final Set<Character> DELIMITERS = ImmutableSet.of(',', ';',
'\t', '|');
- private static byte[] CSV_BASIC =
+ private static final byte[] CSV_BASIC =
("the,quick,brown\tfox\n" + "jumped \tover,the\tlazy,\tdog\n" +
"and
then,ran,down\tthe\tstreet").getBytes(StandardCharsets.UTF_8);
- private static byte[] TSV_BASIC =
+ private static final byte[] CSV_BASIC2 =
+ ("the;quick;brown\tfox\n" + "jumped \tover;the\tlazy;\tdog\n" +
+ "and
then;ran;down\tthe\tstreet").getBytes(StandardCharsets.UTF_8);
+
+ private static final byte[] CSV_BASIC3 =
+ ("the|quick|brown\tfox\n" + "jumped \tover|the\tlazy|\tdog\n" +
+ "and
then|ran|down\tthe\tstreet").getBytes(StandardCharsets.UTF_8);
+
+ private static final byte[] TSV_BASIC =
("the\tquick\tbrown,fox\n" + "jumped ,over\tthe,lazy\t,dog\n" +
"and
then\tran\tdown,the,street").getBytes(StandardCharsets.UTF_8);
- private static byte[] CSV_MID_CELL_QUOTE_EXCEPTION =
+ private static final byte[] CSV_MID_CELL_QUOTE_EXCEPTION =
("the,quick,brown\"fox\n" + "jumped over,the lazy,dog\n" +
"and then,ran,down the
street").getBytes(StandardCharsets.UTF_8);
- private static byte[] ALLOW_SPACES_BEFORE_QUOTE =
+ private static final byte[] ALLOW_SPACES_BEFORE_QUOTE =
("the,quick, \"brown\"\"fox\"\n" + "jumped over,the
lazy,dog\n" +
"and then,ran,down the
street").getBytes(StandardCharsets.UTF_8);
- private static byte[] ALLOW_SPACES_AFTER_QUOTE =
+ private static final byte[] ALLOW_SPACES_AFTER_QUOTE =
("the,\"quick\" ,brown fox\n" + "jumped over,the lazy,dog\n" +
"and then,ran,down the
street").getBytes(StandardCharsets.UTF_8);
- private static List<CSVResult> sniff(char[] delimiters, byte[] bytes,
Charset charset)
+ private static List<CSVResult> sniff(Set<Character> delimiters, byte[]
bytes, Charset charset)
throws IOException {
CSVSniffer sniffer = new CSVSniffer(delimiters);
try (BufferedReader reader = new BufferedReader(
@@ -70,11 +80,19 @@ public class CSVSnifferTest extends TikaTest {
@Test
public void testCSVBasic() throws Exception {
List<CSVResult> results = sniff(DELIMITERS, CSV_BASIC,
StandardCharsets.UTF_8);
- assertEquals(2, results.size());
+ assertEquals(4, results.size());
assertEquals(Character.valueOf(','), results.get(0).getDelimiter());
+ results = sniff(DELIMITERS, CSV_BASIC2, StandardCharsets.UTF_8);
+ assertEquals(4, results.size());
+ assertEquals(Character.valueOf(';'), results.get(0).getDelimiter());
+
+ results = sniff(DELIMITERS, CSV_BASIC3, StandardCharsets.UTF_8);
+ assertEquals(4, results.size());
+ assertEquals(Character.valueOf('|'), results.get(0).getDelimiter());
+
results = sniff(DELIMITERS, TSV_BASIC, StandardCharsets.UTF_8);
- assertEquals(2, results.size());
+ assertEquals(4, results.size());
assertEquals(Character.valueOf('\t'), results.get(0).getDelimiter());
}
@@ -83,18 +101,18 @@ public class CSVSnifferTest extends TikaTest {
List<CSVResult> results =
sniff(DELIMITERS, CSV_MID_CELL_QUOTE_EXCEPTION,
StandardCharsets.UTF_8);
- assertEquals(2, results.size());
+ assertEquals(4, results.size());
}
@Test
public void testAllowWhiteSpacesAroundAQuote() throws Exception {
List<CSVResult> results =
sniff(DELIMITERS, ALLOW_SPACES_BEFORE_QUOTE,
StandardCharsets.UTF_8);
- assertEquals(2, results.size());
+ assertEquals(4, results.size());
assertEquals(Character.valueOf(','), results.get(0).getDelimiter());
results = sniff(DELIMITERS, ALLOW_SPACES_AFTER_QUOTE,
StandardCharsets.UTF_8);
- assertEquals(2, results.size());
+ assertEquals(4, results.size());
assertEquals(Character.valueOf(','), results.get(0).getDelimiter());
}
