Revision: 19723
http://sourceforge.net/p/gate/code/19723
Author: ian_roberts
Date: 2016-11-07 11:25:06 +0000 (Mon, 07 Nov 2016)
Log Message:
-----------
Allow different encodings for the orth and dict files.
Modified Paths:
--------------
gate/trunk/plugins/Twitter/src/gate/twitter/Normaliser.java
Modified: gate/trunk/plugins/Twitter/src/gate/twitter/Normaliser.java
===================================================================
--- gate/trunk/plugins/Twitter/src/gate/twitter/Normaliser.java 2016-11-07
02:22:30 UTC (rev 19722)
+++ gate/trunk/plugins/Twitter/src/gate/twitter/Normaliser.java 2016-11-07
11:25:06 UTC (rev 19723)
@@ -51,7 +51,7 @@
// initialize spell checker
try(InputStream in = dictURL.openStream();
- InputStreamReader inReader = new InputStreamReader(in, encoding);
+ InputStreamReader inReader = new InputStreamReader(in, dictEncoding);
BufferedReader dictReader = new BufferedReader(inReader)) {
checker.initialize(dictReader);
} catch(Exception e) {
@@ -60,7 +60,7 @@
// read the wordlist
try(InputStream in = dictURL.openStream();
- InputStreamReader inReader = new InputStreamReader(in, encoding);
+ InputStreamReader inReader = new InputStreamReader(in, dictEncoding);
BufferedReader dictReader = new BufferedReader(inReader)) {
String entry;
while((entry = dictReader.readLine()) != null) {
@@ -90,7 +90,7 @@
*/
protected void readOrthMappings(URL url) throws IOException {
try(InputStream in = url.openStream();
- InputStreamReader reader = new InputStreamReader(in, encoding);
+ InputStreamReader reader = new InputStreamReader(in, orthEncoding);
CSVReader csvReader = new CSVReader(reader)) {
String[] line = csvReader.readNext();
if(line == null) return;
@@ -361,6 +361,15 @@
return this.dictURL;
}
+ @CreoleParameter(comment = "Character encoding used to read the dictionary
file", defaultValue = "UTF-8")
+ public void setDictEncoding(String encoding) {
+ this.dictEncoding = encoding;
+ }
+
+ public String getDictEncoding() {
+ return this.dictEncoding;
+ }
+
@CreoleParameter(comment = "Path to common normalisation terms list (for
orthographic mappings, e.g. 'b4' to 'before'). "
+ "This can either be a single two-column CSV file where the first
column is the term to be mapped and the second "
+ "column is the target, or a single column file listing relative paths
to other lists (which is useful if you want "
@@ -373,13 +382,13 @@
return this.orthURL;
}
- @CreoleParameter(comment = "Character encoding used to read the orth and
dictionary files", defaultValue = "UTF-8")
- public void setEncoding(String encoding) {
- this.encoding = encoding;
+ @CreoleParameter(comment = "Character encoding used to read the orth files",
defaultValue = "UTF-8")
+ public void setOrthEncoding(String encoding) {
+ this.orthEncoding = encoding;
}
- public String getEncoding() {
- return this.encoding;
+ public String getOrthEncoding() {
+ return this.orthEncoding;
}
@RunTime
@@ -415,8 +424,10 @@
private URL orthURL;
- private String encoding;
+ private String dictEncoding;
+ private String orthEncoding;
+
private double maxDistance;
}
This was sent by the SourceForge.net collaborative development platform, the
world's largest Open Source development site.
------------------------------------------------------------------------------
Developer Access Program for Intel Xeon Phi Processors
Access to Intel Xeon Phi processor-based developer platforms.
With one year of Intel Parallel Studio XE.
Training and support from Colfax.
Order your platform today. http://sdm.link/xeonphi
_______________________________________________
GATE-cvs mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/gate-cvs
