Revision: 19724
http://sourceforge.net/p/gate/code/19724
Author: ian_roberts
Date: 2016-11-07 18:42:28 +0000 (Mon, 07 Nov 2016)
Log Message:
-----------
Support for GZIP compressed dictionaries and orth files.
Modified Paths:
--------------
gate/trunk/plugins/Twitter/src/gate/twitter/Normaliser.java
Modified: gate/trunk/plugins/Twitter/src/gate/twitter/Normaliser.java
===================================================================
--- gate/trunk/plugins/Twitter/src/gate/twitter/Normaliser.java 2016-11-07
11:25:06 UTC (rev 19723)
+++ gate/trunk/plugins/Twitter/src/gate/twitter/Normaliser.java 2016-11-07
18:42:28 UTC (rev 19724)
@@ -20,6 +20,7 @@
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
+import java.util.zip.GZIPInputStream;
import gate.*;
import gate.creole.*;
@@ -50,7 +51,7 @@
orthmappings = new HashMap<String, String>();
// initialize spell checker
- try(InputStream in = dictURL.openStream();
+ try(InputStream in = openPossiblyGzip(dictURL);
InputStreamReader inReader = new InputStreamReader(in, dictEncoding);
BufferedReader dictReader = new BufferedReader(inReader)) {
checker.initialize(dictReader);
@@ -59,7 +60,7 @@
}
// read the wordlist
- try(InputStream in = dictURL.openStream();
+ try(InputStream in = openPossiblyGzip(dictURL);
InputStreamReader inReader = new InputStreamReader(in, dictEncoding);
BufferedReader dictReader = new BufferedReader(inReader)) {
String entry;
@@ -89,7 +90,7 @@
* read recursively.
*/
protected void readOrthMappings(URL url) throws IOException {
- try(InputStream in = url.openStream();
+ try(InputStream in = openPossiblyGzip(url);
InputStreamReader reader = new InputStreamReader(in, orthEncoding);
CSVReader csvReader = new CSVReader(reader)) {
String[] line = csvReader.readNext();
@@ -112,6 +113,15 @@
}
}
+ protected InputStream openPossiblyGzip(URL url) throws IOException {
+ InputStream in = url.openStream();
+ // if the URL ends ".gz" assume it's compressed
+ if(url.getPath().endsWith(".gz")) {
+ in = new GZIPInputStream(in);
+ }
+ return in;
+ }
+
@Override
public void execute() throws ExecutionException {
This was sent by the SourceForge.net collaborative development platform, the
world's largest Open Source development site.
------------------------------------------------------------------------------
Developer Access Program for Intel Xeon Phi Processors
Access to Intel Xeon Phi processor-based developer platforms.
With one year of Intel Parallel Studio XE.
Training and support from Colfax.
Order your platform today. http://sdm.link/xeonphi
_______________________________________________
GATE-cvs mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/gate-cvs
