Author: ehatcher
Date: Mon Apr 4 05:25:16 2005
New Revision: 160034
URL: http://svn.apache.org/viewcvs?view=rev&rev=160034
Log:
enhanced test contributed by Sven. Encoding tweaks
Modified:
lucene/java/trunk/contrib/analyzers/src/java/org/apache/lucene/analysis/ISOLatin1AccentFilter.java
lucene/java/trunk/contrib/analyzers/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java
Modified:
lucene/java/trunk/contrib/analyzers/src/java/org/apache/lucene/analysis/ISOLatin1AccentFilter.java
URL:
http://svn.apache.org/viewcvs/lucene/java/trunk/contrib/analyzers/src/java/org/apache/lucene/analysis/ISOLatin1AccentFilter.java?view=diff&r1=160033&r2=160034
==============================================================================
---
lucene/java/trunk/contrib/analyzers/src/java/org/apache/lucene/analysis/ISOLatin1AccentFilter.java
(original)
+++
lucene/java/trunk/contrib/analyzers/src/java/org/apache/lucene/analysis/ISOLatin1AccentFilter.java
Mon Apr 4 05:25:16 2005
@@ -1,171 +1,171 @@
-package org.apache.lucene.analysis;
-
-/**
- * Copyright 2005 The Apache Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * A filter that replace accented characters in the ISO Latin 1 character set
by
- * their unaccented equivalent. The case will not be altered.
- * <p>
- * For instance, 'Ã' will be replaced by 'a'.
- * <p>
- */
-public class ISOLatin1AccentFilter extends TokenFilter {
- public ISOLatin1AccentFilter(TokenStream input) {
- super(input);
- }
-
- /**
- * To replace accented characters by unaccented equivalents.
- */
- public final Token next() throws java.io.IOException {
- final Token t = input.next();
- if (t == null)
- return null;
- // Return a token with filtered characters.
- return new Token(RemoveAccents(t.termText()), t.startOffset(),
t.endOffset(), t.type());
- }
-
- /**
- * To replace accented characters in a String by unaccented equivalents.
- */
- public final static String RemoveAccents(String input) {
- final StringBuffer output = new StringBuffer();
- for (int i = 0; i < input.length(); i++) {
- switch (input.charAt(i)) {
- case '\u00C0' : // Ã
- case '\u00C1' : // Ã
- case '\u00C2' : // Ã
- case '\u00C3' : // Ã
- case '\u00C4' : // Ã
- case '\u00C5' : // Ã
- output.append("A");
- break;
- case '\u00C6' : // Ã
- output.append("AE");
- break;
- case '\u00C7' : // Ã
- output.append("C");
- break;
- case '\u00C8' : // Ã
- case '\u00C9' : // Ã
- case '\u00CA' : // Ã
- case '\u00CB' : // Ã
- output.append("E");
- break;
- case '\u00CC' : // Ã
- case '\u00CD' : // Ã
- case '\u00CE' : // Ã
- case '\u00CF' : // Ã
- output.append("I");
- break;
- case '\u00D0' : // Ã
- output.append("D");
- break;
- case '\u00D1' : // Ã
- output.append("N");
- break;
- case '\u00D2' : // Ã
- case '\u00D3' : // Ã
- case '\u00D4' : // Ã
- case '\u00D5' : // Ã
- case '\u00D6' : // Ã
- case '\u00D8' : // Ã
- output.append("O");
- break;
- case '\u0152' : // Â
- output.append("OE");
- break;
- case '\u00DE' : // Ã
- output.append("TH");
- break;
- case '\u00D9' : // Ã
- case '\u00DA' : // Ã
- case '\u00DB' : // Ã
- case '\u00DC' : // Ã
- output.append("U");
- break;
- case '\u00DD' : // Ã
- case '\u0178' : // Â
- output.append("Y");
- break;
- case '\u00E0' : // Ã
- case '\u00E1' : // Ã
- case '\u00E2' : // Ã
- case '\u00E3' : // Ã
- case '\u00E4' : // Ã
- case '\u00E5' : // Ã
- output.append("a");
- break;
- case '\u00E6' : // Ã
- output.append("ae");
- break;
- case '\u00E7' : // Ã
- output.append("c");
- break;
- case '\u00E8' : // Ã
- case '\u00E9' : // Ã
- case '\u00EA' : // Ã
- case '\u00EB' : // Ã
- output.append("e");
- break;
- case '\u00EC' : // Ã
- case '\u00ED' : // Ã
- case '\u00EE' : // Ã
- case '\u00EF' : // Ã
- output.append("i");
- break;
- case '\u00F0' : // Ã
- output.append("d");
- break;
- case '\u00F1' : // Ã
- output.append("n");
- break;
- case '\u00F2' : // Ã
- case '\u00F3' : // Ã
- case '\u00F4' : // Ã
- case '\u00F5' : // Ã
- case '\u00F6' : // Ã
- case '\u00F8' : // Ã
- output.append("o");
- break;
- case '\u0153' : // Â
- output.append("oe");
- break;
- case '\u00DF' : // Ã
- output.append("ss");
- break;
- case '\u00FE' : // Ã
- output.append("th");
- break;
- case '\u00F9' : // Ã
- case '\u00FA' : // Ã
- case '\u00FB' : // Ã
- case '\u00FC' : // Â
- output.append("u");
- break;
- case '\u00FD' : // Ã
- case '\u00FF' : // Ã
- output.append("y");
- break;
- default :
- output.append(input.charAt(i));
- break;
- }
- }
- return output.toString();
- }
+package org.apache.lucene.analysis;
+
+/**
+ * Copyright 2005 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * A filter that replace accented characters in the ISO Latin 1 character set
by
+ * their unaccented equivalent. The case will not be altered.
+ * <p>
+ * For instance, 'ÃÂ' will be replaced by 'a'.
+ * <p>
+ */
+public class ISOLatin1AccentFilter extends TokenFilter {
+ public ISOLatin1AccentFilter(TokenStream input) {
+ super(input);
+ }
+
+ /**
+ * To replace accented characters by unaccented equivalents.
+ */
+ public final Token next() throws java.io.IOException {
+ final Token t = input.next();
+ if (t == null)
+ return null;
+ // Return a token with filtered characters.
+ return new Token(RemoveAccents(t.termText()), t.startOffset(),
t.endOffset(), t.type());
+ }
+
+ /**
+ * To replace accented characters in a String by unaccented equivalents.
+ */
+ public final static String RemoveAccents(String input) {
+ final StringBuffer output = new StringBuffer();
+ for (int i = 0; i < input.length(); i++) {
+ switch (input.charAt(i)) {
+ case '\u00C0' : // ÃÂ
+ case '\u00C1' : // ÃÂ
+ case '\u00C2' : // ÃÂ
+ case '\u00C3' : // ÃÂ
+ case '\u00C4' : // ÃÂ
+ case '\u00C5' : // ÃÂ
+ output.append("A");
+ break;
+ case '\u00C6' : // ÃÂ
+ output.append("AE");
+ break;
+ case '\u00C7' : // ÃÂ
+ output.append("C");
+ break;
+ case '\u00C8' : // ÃÂ
+ case '\u00C9' : // ÃÂ
+ case '\u00CA' : // ÃÂ
+ case '\u00CB' : // ÃÂ
+ output.append("E");
+ break;
+ case '\u00CC' : // ÃÂ
+ case '\u00CD' : // ÃÂ
+ case '\u00CE' : // ÃÂ
+ case '\u00CF' : // ÃÂ
+ output.append("I");
+ break;
+ case '\u00D0' : // ÃÂ
+ output.append("D");
+ break;
+ case '\u00D1' : // ÃÂ
+ output.append("N");
+ break;
+ case '\u00D2' : // ÃÂ
+ case '\u00D3' : // ÃÂ
+ case '\u00D4' : // ÃÂ
+ case '\u00D5' : // ÃÂ
+ case '\u00D6' : // ÃÂ
+ case '\u00D8' : // ÃÂ
+ output.append("O");
+ break;
+ case '\u0152' : // ÃÂ
+ output.append("OE");
+ break;
+ case '\u00DE' : // ÃÂ
+ output.append("TH");
+ break;
+ case '\u00D9' : // ÃÂ
+ case '\u00DA' : // ÃÂ
+ case '\u00DB' : // ÃÂ
+ case '\u00DC' : // ÃÂ
+ output.append("U");
+ break;
+ case '\u00DD' : // ÃÂ
+ case '\u0178' : // ÃÂ
+ output.append("Y");
+ break;
+ case '\u00E0' : // ÃÂ
+ case '\u00E1' : // ÃÂ
+ case '\u00E2' : // ÃÂ
+ case '\u00E3' : // ÃÂ
+ case '\u00E4' : // ÃÂ
+ case '\u00E5' : // ÃÂ
+ output.append("a");
+ break;
+ case '\u00E6' : // ÃÂ
+ output.append("ae");
+ break;
+ case '\u00E7' : // ÃÂ
+ output.append("c");
+ break;
+ case '\u00E8' : // ÃÂ
+ case '\u00E9' : // ÃÂ
+ case '\u00EA' : // ÃÂ
+ case '\u00EB' : // ÃÂ
+ output.append("e");
+ break;
+ case '\u00EC' : // ÃÂ
+ case '\u00ED' : // ÃÂ
+ case '\u00EE' : // ÃÂ
+ case '\u00EF' : // ÃÂ
+ output.append("i");
+ break;
+ case '\u00F0' : // ÃÂ
+ output.append("d");
+ break;
+ case '\u00F1' : // ÃÂ
+ output.append("n");
+ break;
+ case '\u00F2' : // ÃÂ
+ case '\u00F3' : // ÃÂ
+ case '\u00F4' : // ÃÂ
+ case '\u00F5' : // ÃÂ
+ case '\u00F6' : // ÃÂ
+ case '\u00F8' : // ÃÂ
+ output.append("o");
+ break;
+ case '\u0153' : // ÃÂ
+ output.append("oe");
+ break;
+ case '\u00DF' : // ÃÂ
+ output.append("ss");
+ break;
+ case '\u00FE' : // ÃÂ
+ output.append("th");
+ break;
+ case '\u00F9' : // ÃÂ
+ case '\u00FA' : // ÃÂ
+ case '\u00FB' : // ÃÂ
+ case '\u00FC' : // ÃÂ
+ output.append("u");
+ break;
+ case '\u00FD' : // ÃÂ
+ case '\u00FF' : // ÃÂ
+ output.append("y");
+ break;
+ default :
+ output.append(input.charAt(i));
+ break;
+ }
+ }
+ return output.toString();
+ }
}
Modified:
lucene/java/trunk/contrib/analyzers/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java
URL:
http://svn.apache.org/viewcvs/lucene/java/trunk/contrib/analyzers/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java?view=diff&r1=160033&r2=160034
==============================================================================
---
lucene/java/trunk/contrib/analyzers/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java
(original)
+++
lucene/java/trunk/contrib/analyzers/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java
Mon Apr 4 05:25:16 2005
@@ -6,11 +6,79 @@
public class TestISOLatin1AccentFilter extends TestCase {
public void testU() throws Exception {
- TokenStream stream = new WhitespaceTokenizer(new StringReader("\u00FC"));
+ TokenStream stream = new WhitespaceTokenizer(new StringReader("Des mot
clÃÂs ÃÂ LA CHAÃÂNE ÃÂ ÃÂ ÃÂ ÃÂ ÃÂ ÃÂ ÃÂ ÃÂ ÃÂ ÃÂ
ÃÂ ÃÂ ÃÂ ÃÂ ÃÂ ÃÂ ÃÂ ÃÂ ÃÂ ÃÂ ÃÂ ÃÂ ÃÂ ÃÂ ÃÂ ÃÂ
ÃÂ ÃÂ ÃÂ ÃÂ ÃÂ ÃÂ ÃÂ ÃÂ ÃÂ ÃÂ ÃÂ ÃÂ ÃÂ ÃÂ ÃÂ ÃÂ
ÃÂ ÃÂ ÃÂ ÃÂ ÃÂ ÃÂ ÃÂ ÃÂ ÃÂ ÃÂ ÃÂ ÃÂ ÃÂ ÃÂ ÃÂ ÃÂ
ÃÂ ÃÂ ÃÂ ÃÂ ÃÂ ÃÂ ÃÂ"));
ISOLatin1AccentFilter filter = new ISOLatin1AccentFilter(stream);
- Token token = filter.next();
- assertEquals("u", token.termText);
-
+ assertEquals("Des", filter.next().termText());
+ assertEquals("mot", filter.next().termText());
+ assertEquals("cles", filter.next().termText());
+ assertEquals("A", filter.next().termText());
+ assertEquals("LA", filter.next().termText());
+ assertEquals("CHAINE", filter.next().termText());
+ assertEquals("A", filter.next().termText());
+ assertEquals("A", filter.next().termText());
+ assertEquals("A", filter.next().termText());
+ assertEquals("A", filter.next().termText());
+ assertEquals("A", filter.next().termText());
+ assertEquals("A", filter.next().termText());
+ assertEquals("AE", filter.next().termText());
+ assertEquals("C", filter.next().termText());
+ assertEquals("E", filter.next().termText());
+ assertEquals("E", filter.next().termText());
+ assertEquals("E", filter.next().termText());
+ assertEquals("E", filter.next().termText());
+ assertEquals("I", filter.next().termText());
+ assertEquals("I", filter.next().termText());
+ assertEquals("I", filter.next().termText());
+ assertEquals("I", filter.next().termText());
+ assertEquals("D", filter.next().termText());
+ assertEquals("N", filter.next().termText());
+ assertEquals("O", filter.next().termText());
+ assertEquals("O", filter.next().termText());
+ assertEquals("O", filter.next().termText());
+ assertEquals("O", filter.next().termText());
+ assertEquals("O", filter.next().termText());
+ assertEquals("O", filter.next().termText());
+ assertEquals("OE", filter.next().termText());
+ assertEquals("TH", filter.next().termText());
+ assertEquals("U", filter.next().termText());
+ assertEquals("U", filter.next().termText());
+ assertEquals("U", filter.next().termText());
+ assertEquals("U", filter.next().termText());
+ assertEquals("Y", filter.next().termText());
+ assertEquals("Y", filter.next().termText());
+ assertEquals("a", filter.next().termText());
+ assertEquals("a", filter.next().termText());
+ assertEquals("a", filter.next().termText());
+ assertEquals("a", filter.next().termText());
+ assertEquals("a", filter.next().termText());
+ assertEquals("a", filter.next().termText());
+ assertEquals("ae", filter.next().termText());
+ assertEquals("c", filter.next().termText());
+ assertEquals("e", filter.next().termText());
+ assertEquals("e", filter.next().termText());
+ assertEquals("e", filter.next().termText());
+ assertEquals("e", filter.next().termText());
+ assertEquals("i", filter.next().termText());
+ assertEquals("i", filter.next().termText());
+ assertEquals("i", filter.next().termText());
+ assertEquals("i", filter.next().termText());
+ assertEquals("d", filter.next().termText());
+ assertEquals("n", filter.next().termText());
+ assertEquals("o", filter.next().termText());
+ assertEquals("o", filter.next().termText());
+ assertEquals("o", filter.next().termText());
+ assertEquals("o", filter.next().termText());
+ assertEquals("o", filter.next().termText());
+ assertEquals("o", filter.next().termText());
+ assertEquals("oe", filter.next().termText());
+ assertEquals("ss", filter.next().termText());
+ assertEquals("th", filter.next().termText());
+ assertEquals("u", filter.next().termText());
+ assertEquals("u", filter.next().termText());
+ assertEquals("u", filter.next().termText());
+ assertEquals("u", filter.next().termText());
+ assertEquals("y", filter.next().termText());
+ assertEquals("y", filter.next().termText());
assertNull(filter.next());
}
-}
+}
\ No newline at end of file
