Revision: 19721
http://sourceforge.net/p/gate/code/19721
Author: ian_roberts
Date: 2016-11-07 00:07:49 +0000 (Mon, 07 Nov 2016)
Log Message:
-----------
Use my fork of jaspell (https://github.com/GateNLP/jaspell) so we can now
specify the character encoding used to load the dictionary file, and it doesn't
necessarily have to be a file: URL.
Modified Paths:
--------------
gate/trunk/plugins/Twitter/creole.xml
gate/trunk/plugins/Twitter/src/gate/twitter/Normaliser.java
Added Paths:
-----------
gate/trunk/plugins/Twitter/lib/jaspell-0.3.1.jar
Removed Paths:
-------------
gate/trunk/plugins/Twitter/lib/jaspell.jar
Modified: gate/trunk/plugins/Twitter/creole.xml
===================================================================
--- gate/trunk/plugins/Twitter/creole.xml 2016-11-06 20:36:04 UTC (rev
19720)
+++ gate/trunk/plugins/Twitter/creole.xml 2016-11-07 00:07:49 UTC (rev
19721)
@@ -1,7 +1,7 @@
<?xml version="1.0"?>
<CREOLE-DIRECTORY>
<JAR SCAN="true">twitter.jar</JAR>
- <JAR>lib/jaspell.jar</JAR>
+ <JAR>lib/jaspell-0.3.1.jar</JAR>
<JAR>lib/opencsv-3.8.jar</JAR>
<REQUIRES>../Format_Twitter</REQUIRES>
</CREOLE-DIRECTORY>
Added: gate/trunk/plugins/Twitter/lib/jaspell-0.3.1.jar
===================================================================
(Binary files differ)
Index: gate/trunk/plugins/Twitter/lib/jaspell-0.3.1.jar
===================================================================
--- gate/trunk/plugins/Twitter/lib/jaspell-0.3.1.jar 2016-11-06 20:36:04 UTC
(rev 19720)
+++ gate/trunk/plugins/Twitter/lib/jaspell-0.3.1.jar 2016-11-07 00:07:49 UTC
(rev 19721)
Property changes on: gate/trunk/plugins/Twitter/lib/jaspell-0.3.1.jar
___________________________________________________________________
Added: svn:mime-type
## -0,0 +1 ##
+application/zip
\ No newline at end of property
Deleted: gate/trunk/plugins/Twitter/lib/jaspell.jar
===================================================================
(Binary files differ)
Modified: gate/trunk/plugins/Twitter/src/gate/twitter/Normaliser.java
===================================================================
--- gate/trunk/plugins/Twitter/src/gate/twitter/Normaliser.java 2016-11-06
20:36:04 UTC (rev 19720)
+++ gate/trunk/plugins/Twitter/src/gate/twitter/Normaliser.java 2016-11-07
00:07:49 UTC (rev 19721)
@@ -45,33 +45,22 @@
if(this.dictURL == null)
throw new ResourceInstantiationException("dict file not set");
- String dictFilePath = null;
checker = new SpellChecker();
wordlist = new HashSet<String>();
orthmappings = new HashMap<String, String>();
- // initialize spell checker - jaspell *requires* a File, it can't read from
- // an arbitrary URL, so write a temp file if necessary
- try {
- dictFilePath = Files.fileFromURL(dictURL).getAbsolutePath();
- } catch(IllegalArgumentException e) {
- // dictURL is not a file: URL
- try {
- File tmpDictFile = Files.writeTempFile(dictURL.openStream());
- dictFilePath = tmpDictFile.getAbsolutePath();
- } catch(Exception e2) {
- throw new ResourceInstantiationException("Could not write dictionary
to temp file", e2);
- }
- }
- try {
- checker.initialize(dictFilePath);
+ // initialize spell checker
+ try(InputStream in = dictURL.openStream();
+ InputStreamReader inReader = new InputStreamReader(in, encoding);
+ BufferedReader dictReader = new BufferedReader(inReader)) {
+ checker.initialize(dictReader);
} catch(Exception e) {
throw new ResourceInstantiationException("Error initializing
spellchecker", e);
}
// read the wordlist
try(InputStream in = dictURL.openStream();
- InputStreamReader inReader = new InputStreamReader(in); //, encoding);
// jaspell always uses default encoding, so we should match it
+ InputStreamReader inReader = new InputStreamReader(in, encoding);
BufferedReader dictReader = new BufferedReader(inReader)) {
String entry;
while((entry = dictReader.readLine()) != null) {
@@ -384,7 +373,7 @@
return this.orthURL;
}
- @CreoleParameter(comment = "Character encoding used to read the orth files",
defaultValue = "UTF-8")
+ @CreoleParameter(comment = "Character encoding used to read the orth and
dictionary files", defaultValue = "UTF-8")
public void setEncoding(String encoding) {
this.encoding = encoding;
}
This was sent by the SourceForge.net collaborative development platform, the
world's largest Open Source development site.
------------------------------------------------------------------------------
Developer Access Program for Intel Xeon Phi Processors
Access to Intel Xeon Phi processor-based developer platforms.
With one year of Intel Parallel Studio XE.
Training and support from Colfax.
Order your platform today. http://sdm.link/xeonphi
_______________________________________________
GATE-cvs mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/gate-cvs
