Have you looked at the standard WordDelimiterFilter from Solr by chance? In keeping with the spirit of Lucene.Net, I did a simple line-by-line conversion from Java just a couple of days ago and it might fulfill your needs. I'm going to try to attach it to this email - hopefully it doesn't mess up the list engine.
Cheers, Ben /** * Splits words into subwords and performs optional transformations on subword groups. * Words are split into subwords with the following rules: * - split on intra-word delimiters (by default, all non alpha-numeric characters). * - "Wi-Fi" -> "Wi", "Fi" * - split on case transitions * - "PowerShot" -> "Power", "Shot" * - split on letter-number transitions * - "SD500" -> "SD", "500" * - leading and trailing intra-word delimiters on each subword are ignored * - "//hello---there, 'dude'" -> "hello", "there", "dude" * - trailing "'s" are removed for each subword * - "O'Neil's" -> "O", "Neil" * - Note: this step isn't performed in a separate filter because of possible subword combinations. * * The <b>combinations</b> parameter affects how subwords are combined: * - combinations="0" causes no subword combinations. * - "PowerShot" -> 0:"Power", 1:"Shot" (0 and 1 are the token positions) * - combinations="1" means that in addition to the subwords, maximum runs of non-numeric subwords are catenated and produced at the same position of the last subword in the run. * - "PowerShot" -> 0:"Power", 1:"Shot" 1:"PowerShot" * - "A's+B's&C's" -> 0:"A", 1:"B", 2:"C", 2:"ABC" * - "Super-Duper-XL500-42-AutoCoder!" -> 0:"Super", 1:"Duper", 2:"XL", 2:"SuperDuperXL", 3:"500" 4:"42", 5:"Auto", 6:"Coder", 6:"AutoCoder" * * One use for WordDelimiterFilter is to help match words with different subword delimiters. * For example, if the source text contained "wi-fi" one may want "wifi" "WiFi" "wi-fi" "wi+fi" * queries to all match. * One way of doing so is to specify combinations="1" in the analyzer * used for indexing, and combinations="0" (the default) in the analyzer * used for querying. Given that the current StandardTokenizer * immediately removes many intra-word delimiters, it is recommended that * this filter be used after a tokenizer that does not do this * (such as WhitespaceTokenizer). * * @author yonik * @version $Id: WordDelimiterFilter.java 472574 2006-11-08 18:25:52Z yonik $ */ On Tue, Apr 29, 2008 at 5:58 AM, Sudhanya Chatterjee < [EMAIL PROTECTED]> wrote: > Hi, > > The StandardAnalyzer fulfills all requirements in our application, apart > from one. > > Input text that goes for indexing is - > > > > My name is Sudhanya.Chatterjee is my second name. Cost of book is 50.6 > > > > StandardTokenizer successfully tokenizes on punctuations. > > But if a dot is not followed by a space it considers it as one token. > > In above case "Sudhanya" "Chatterjee" should be considered as two tokens > but > 50.6 as one token. > > So if a dot is preceded or followed by a number it should be kept intact. > > > > This is one extra rule I want apart from the exsisting ones of a > StandardAnalyzer. > > > > How to add the above requirement into the existing code of > StandardAnalyzer? > > Taken into consideration that the rest of the rules are still required. > > > > Thanks, > > Sudhanya > > -- 13:37 - Someone stole the precinct toilet. The cops have nothing to go on. 14:37 - Officers dispatched to a daycare where a three-year-old was resisting a rest. 21:11 - Hole found in nudist camp wall. Officers are looking into it.
/** * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /* Conversion from Java by Ben Martz, 25 Apr 2008 */ using System; using System.Collections.Generic; using System.Text; using Lucene.Net.Analysis; namespace Portal.LuceneInterface { /** * Splits words into subwords and performs optional transformations on subword groups. * Words are split into subwords with the following rules: * - split on intra-word delimiters (by default, all non alpha-numeric characters). * - "Wi-Fi" -> "Wi", "Fi" * - split on case transitions * - "PowerShot" -> "Power", "Shot" * - split on letter-number transitions * - "SD500" -> "SD", "500" * - leading and trailing intra-word delimiters on each subword are ignored * - "//hello---there, 'dude'" -> "hello", "there", "dude" * - trailing "'s" are removed for each subword * - "O'Neil's" -> "O", "Neil" * - Note: this step isn't performed in a separate filter because of possible subword combinations. * * The <b>combinations</b> parameter affects how subwords are combined: * - combinations="0" causes no subword combinations. * - "PowerShot" -> 0:"Power", 1:"Shot" (0 and 1 are the token positions) * - combinations="1" means that in addition to the subwords, maximum runs of non-numeric subwords are catenated and produced at the same position of the last subword in the run. * - "PowerShot" -> 0:"Power", 1:"Shot" 1:"PowerShot" * - "A's+B's&C's" -> 0:"A", 1:"B", 2:"C", 2:"ABC" * - "Super-Duper-XL500-42-AutoCoder!" -> 0:"Super", 1:"Duper", 2:"XL", 2:"SuperDuperXL", 3:"500" 4:"42", 5:"Auto", 6:"Coder", 6:"AutoCoder" * * One use for WordDelimiterFilter is to help match words with different subword delimiters. * For example, if the source text contained "wi-fi" one may want "wifi" "WiFi" "wi-fi" "wi+fi" * queries to all match. * One way of doing so is to specify combinations="1" in the analyzer * used for indexing, and combinations="0" (the default) in the analyzer * used for querying. Given that the current StandardTokenizer * immediately removes many intra-word delimiters, it is recommended that * this filter be used after a tokenizer that does not do this * (such as WhitespaceTokenizer). * * @author yonik * @version $Id: WordDelimiterFilter.java 472574 2006-11-08 18:25:52Z yonik $ */ class WordDelimiterFilter : TokenFilter { private byte[] charTypeTable; public static byte LOWER = 0x01; public static byte UPPER = 0x02; public static byte DIGIT = 0x04; public static byte SUBWORD_DELIM = 0x08; // combinations: for testing, not for setting bits public static byte ALPHA = 0x03; public static byte ALPHANUM = 0x07; // TODO: should there be a WORD_DELIM category for // chars that only separate words (no catenation of subwords // will be done if separated by these chars?) // "," would be an obvious candidate... static byte[] defaultWordDelimTable; static WordDelimiterFilter() { byte[] tab = new byte[256]; for (int i = 0; i < 256; i++) { byte code = 0; if (Char.IsLower((char)i)) code |= LOWER; else if (Char.IsUpper((char)i)) code |= UPPER; else if (Char.IsDigit((char)i)) code |= DIGIT; if (code == 0) code = SUBWORD_DELIM; tab[i] = code; } defaultWordDelimTable = tab; } /** * If 1, causes parts of words to be generated: * <p/> * "PowerShot" => "Power" "Shot" */ int generateWordParts; /** * If 1, causes number subwords to be generated: * <p/> * "500-42" => "500" "42" */ int generateNumberParts; /** * If 1, causes maximum runs of word parts to be catenated: * <p/> * "wi-fi" => "wifi" */ int catenateWords; /** * If 1, causes maximum runs of number parts to be catenated: * <p/> * "500-42" => "50042" */ int catenateNumbers; /** * If 1, causes all subword parts to be catenated: * <p/> * "wi-fi-4000" => "wifi4000" */ int catenateAll; /** * * @param in Token stream to be filtered. * @param charTypeTable * @param generateWordParts If 1, causes parts of words to be generated: "PowerShot" => "Power" "Shot" * @param generateNumberParts If 1, causes number subwords to be generated: "500-42" => "500" "42" * @param catenateWords 1, causes maximum runs of word parts to be catenated: "wi-fi" => "wifi" * @param catenateNumbers If 1, causes maximum runs of number parts to be catenated: "500-42" => "50042" * @param catenateAll If 1, causes all subword parts to be catenated: "wi-fi-4000" => "wifi4000" */ public WordDelimiterFilter(TokenStream inStream, byte[] charTypeTable, int generateWordParts, int generateNumberParts, int catenateWords, int catenateNumbers, int catenateAll) : base(inStream) { this.generateWordParts = generateWordParts; this.generateNumberParts = generateNumberParts; this.catenateWords = catenateWords; this.catenateNumbers = catenateNumbers; this.catenateAll = catenateAll; this.charTypeTable = charTypeTable; } /** * @param in Token stream to be filtered. * @param generateWordParts If 1, causes parts of words to be generated: "PowerShot" => "Power" "Shot" * @param generateNumberParts If 1, causes number subwords to be generated: "500-42" => "500" "42" * @param catenateWords 1, causes maximum runs of word parts to be catenated: "wi-fi" => "wifi" * @param catenateNumbers If 1, causes maximum runs of number parts to be catenated: "500-42" => "50042" * @param catenateAll If 1, causes all subword parts to be catenated: "wi-fi-4000" => "wifi4000" */ public WordDelimiterFilter(TokenStream inStream, int generateWordParts, int generateNumberParts, int catenateWords, int catenateNumbers, int catenateAll) : base(inStream) { this.generateWordParts = generateWordParts; this.generateNumberParts = generateNumberParts; this.catenateWords = catenateWords; this.catenateNumbers = catenateNumbers; this.catenateAll = catenateAll; this.charTypeTable = defaultWordDelimTable; } int CharType(char ch) { if (ch < charTypeTable.Length) { return charTypeTable[ch]; } else if (Char.IsLower(ch)) { return LOWER; } else if (Char.IsLetter(ch)) { return UPPER; } else { return SUBWORD_DELIM; } } // use the type of the first char as the type // of the token. private int TokType(Token t) { return CharType(t.TermText()[0]); } // There isn't really an efficient queue class, so we will // just use an array for now. private List<Token> queue = new List<Token>(4); private int queuePos = 0; // temporary working queue private List<Token> tlist = new List<Token>(4); private Token NewTok(Token orig, int start, int end) { // end appears to be erroneously named return new Token(orig.TermText().Substring(start, end - start), orig.StartOffset() + start, orig.StartOffset() + end, orig.Type()); } public override Token Next() { // check the queue first if (queuePos < queue.Count) { return queue[queuePos++]; } // reset the queue if it had been previously used if (queuePos != 0) { queuePos = 0; queue.Clear(); } // optimize for the common case: assume there will be // no subwords (just a simple word) // // Would it actually be faster to check for the common form // of isLetter() isLower()*, and then backtrack if it doesn't match? int origPosIncrement; while (true) { Token t = input.Next(); if (t == null) return null; String s = t.TermText(); int start = 0; int end = s.Length; if (end == 0) continue; origPosIncrement = t.GetPositionIncrement(); // Avoid calling charType more than once for each char (basically // avoid any backtracking). // makes code slightly more difficult, but faster. char ch = s[start]; int type = CharType(ch); int numWords = 0; while (start < end) { // first eat delimiters at the start of this subword while ((type & SUBWORD_DELIM) != 0 && ++start < end) { ch = s[start]; type = CharType(ch); } int pos = start; // save the type of the first char of the subword // as a way to tell what type of subword token this is (number, word, etc) int firstType = type; int lastType = type; // type of the previously read char while (pos < end) { if (type != lastType) { // check and remove "'s" from the end of a token. // the pattern to check for is // ALPHA "'" ("s"|"S") (SUBWORD_DELIM | END) if ((lastType & ALPHA) != 0) { if (ch == '\'' && pos + 1 < end && (s[pos + 1] == 's' || s[pos + 1] == 'S')) { int subWordEnd = pos; if (pos + 2 >= end) { // end of string detected after "'s" pos += 2; } else { // make sure that a delimiter follows "'s" char ch2 = s[pos + 2]; int type2 = CharType(ch2); if ((type2 & SUBWORD_DELIM) != 0) { // if delimiter, move position pointer // to it (skipping over "'s" ch = ch2; type = type2; pos += 2; } } queue.Add(NewTok(t, start, subWordEnd)); if ((firstType & ALPHA) != 0) numWords++; break; } } // For case changes, only split on a transition from // lower to upper case, not vice-versa. // That will correctly handle the // case of a word starting with a capital (won't split). // It will also handle pluralization of // an uppercase word such as FOOs (won't split). if ((lastType & UPPER) != 0 && (type & LOWER) != 0) { // UPPER->LOWER: Don't split } else { // NOTE: this code currently assumes that only one flag // is set for each character now, so we don't have // to explicitly check for all the classes of transitions // listed below. // LOWER->UPPER // ALPHA->NUMERIC // NUMERIC->ALPHA // *->DELIMITER queue.Add(NewTok(t, start, pos)); if ((firstType & ALPHA) != 0) numWords++; break; } } if (++pos >= end) { if (start == 0) { // the subword is the whole original token, so // return it unchanged. return t; } Token newtok = NewTok(t, start, pos); // optimization... if this is the only token, // return it immediately. if (queue.Count == 0) { newtok.SetPositionIncrement(origPosIncrement); return newtok; } queue.Add(newtok); if ((firstType & ALPHA) != 0) numWords++; break; } lastType = type; ch = s[pos]; type = CharType(ch); } // start of the next subword is the current position start = pos; } // System.out.println("##########TOKEN=" + s + " ######### WORD DELIMITER QUEUE=" + str(queue)); int numtok = queue.Count; // We reached the end of the current token. // If the queue is empty, we should continue by reading // the next token if (numtok == 0) { continue; } // if number of tokens is 1, always return the single tok if (numtok == 1) { break; } int numNumbers = numtok - numWords; // check conditions under which the current token // queue may be used as-is (no catenations needed) if (catenateAll == 0 // no "everything" to catenate && (catenateWords == 0 || numWords <= 1) // no words to catenate && (catenateNumbers == 0 || numNumbers <= 1) // no numbers to catenate && (generateWordParts != 0 || numWords == 0) // word generation is on && (generateNumberParts != 0 || numNumbers == 0)) // number generation is on { break; } // swap queue and the temporary working list, then clear the // queue in preparation for adding all combinations back to it. List<Token> tmp = tlist; tlist = queue; queue = tmp; queue.Clear(); if (numWords == 0) { // all numbers AddCombos(tlist, 0, numtok, generateNumberParts != 0, catenateNumbers != 0 || catenateAll != 0, 1); if (queue.Count > 0) break; else continue; } else if (numNumbers == 0) { // all words AddCombos(tlist, 0, numtok, generateWordParts != 0, catenateWords != 0 || catenateAll != 0, 1); if (queue.Count > 0) break; else continue; } else if (generateNumberParts == 0 && generateWordParts == 0 && catenateNumbers == 0 && catenateWords == 0) { // catenate all *only* // OPT:could be optimized to add to current queue... AddCombos(tlist, 0, numtok, false, catenateAll != 0, 1); if (queue.Count > 0) break; else continue; } // // Find all adjacent tokens of the same type. // Token tok = tlist[0]; bool isWord = (TokType(tok) & ALPHA) != 0; bool wasWord = isWord; for (int i = 0; i < numtok; ) { int j; for (j = i + 1; j < numtok; j++) { wasWord = isWord; tok = tlist[j]; isWord = (TokType(tok) & ALPHA) != 0; if (isWord != wasWord) break; } if (wasWord) { AddCombos(tlist, i, j, generateWordParts != 0, catenateWords != 0, 1); } else { AddCombos(tlist, i, j, generateNumberParts != 0, catenateNumbers != 0, 1); } i = j; } // take care catenating all subwords if (catenateAll != 0) { AddCombos(tlist, 0, numtok, false, true, 0); } // NOTE: in certain cases, queue may be empty (for instance, if catenate // and generate are both set to false). Only exit the loop if the queue // is not empty. if (queue.Count > 0) break; } // System.out.println("##########AFTER COMBINATIONS:"+ str(queue)); queuePos = 1; Token tok2 = queue[0]; tok2.SetPositionIncrement(origPosIncrement); return tok2; } // index "a","b","c" as pos0="a", pos1="b", pos2="c", pos2="abc" private void AddCombos(List<Token> lst, int start, int end, bool generateSubwords, bool catenateSubwords, int posOffset) { if (end - start == 1) { // always generate a word alone, even if generateSubwords=0 because // the catenation of all the subwords *is* the subword. queue.Add(lst[start]); return; } StringBuilder sb = null; if (catenateSubwords) sb = new StringBuilder(); Token firstTok = null; Token tok = null; for (int i = start; i < end; i++) { tok = lst[i]; if (catenateSubwords) { if (i == start) firstTok = tok; sb.Append(tok.TermText()); } if (generateSubwords) { queue.Add(tok); } } if (catenateSubwords) { Token concatTok = new Token(sb.ToString(), firstTok.StartOffset(), tok.EndOffset(), firstTok.Type()); // if we indexed some other tokens, then overlap concatTok with the last. // Otherwise, use the value passed in as the position offset. concatTok.SetPositionIncrement(generateSubwords == true ? 0 : posOffset); queue.Add(concatTok); } } // questions: // negative numbers? -42 indexed as just 42? // dollar sign? $42 // percent sign? 33% // downsides: if source text is "powershot" then a query of "PowerShot" won't match! } }