Author: norman Date: Sat Jun 11 12:28:05 2011 New Revision: 1134591 URL: http://svn.apache.org/viewvc?rev=1134591&view=rev Log: Make sure we normalize the Date header to GMT so searching via SENTON, SENTSINCE, SENTBEFORE works like expected. See MAILBOX-10
Modified: james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/search/lucene/LuceneMessageSearchIndex.java Modified: james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/search/lucene/LuceneMessageSearchIndex.java URL: http://svn.apache.org/viewvc/james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/search/lucene/LuceneMessageSearchIndex.java?rev=1134591&r1=1134590&r2=1134591&view=diff ============================================================================== --- james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/search/lucene/LuceneMessageSearchIndex.java (original) +++ james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/search/lucene/LuceneMessageSearchIndex.java Sat Jun 11 12:28:05 2011 @@ -22,7 +22,10 @@ import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; +import java.io.StringReader; import java.nio.charset.Charset; +import java.text.DateFormat; +import java.text.ParseException; import java.util.ArrayList; import java.util.Calendar; import java.util.Date; @@ -31,20 +34,19 @@ import java.util.Iterator; import java.util.List; import java.util.Locale; import java.util.Set; +import java.util.TimeZone; import javax.mail.Flags; import javax.mail.Flags.Flag; -import org.apache.commons.lang.time.DateUtils; import org.apache.james.mailbox.MailboxException; import org.apache.james.mailbox.MailboxSession; import org.apache.james.mailbox.MessageRange; import org.apache.james.mailbox.SearchQuery; -import org.apache.james.mailbox.SearchQuery.CustomFlagCriterion; -import org.apache.james.mailbox.UnsupportedSearchException; import org.apache.james.mailbox.SearchQuery.AllCriterion; import org.apache.james.mailbox.SearchQuery.ContainsOperator; import org.apache.james.mailbox.SearchQuery.Criterion; +import org.apache.james.mailbox.SearchQuery.CustomFlagCriterion; import org.apache.james.mailbox.SearchQuery.DateOperator; import org.apache.james.mailbox.SearchQuery.DateResolution; import org.apache.james.mailbox.SearchQuery.FlagCriterion; @@ -53,6 +55,7 @@ import org.apache.james.mailbox.SearchQu import org.apache.james.mailbox.SearchQuery.NumericOperator; import org.apache.james.mailbox.SearchQuery.NumericRange; import org.apache.james.mailbox.SearchQuery.UidCriterion; +import org.apache.james.mailbox.UnsupportedSearchException; import org.apache.james.mailbox.store.mail.MessageMapperFactory; import org.apache.james.mailbox.store.mail.model.Mailbox; import org.apache.james.mailbox.store.mail.model.Message; @@ -66,16 +69,19 @@ import org.apache.james.mime4j.field.add import org.apache.james.mime4j.field.address.AddressList; import org.apache.james.mime4j.field.address.Group; import org.apache.james.mime4j.field.address.MailboxList; +import org.apache.james.mime4j.field.datetime.DateTime; +import org.apache.james.mime4j.field.datetime.parser.DateTimeParser; import org.apache.james.mime4j.message.Header; import org.apache.james.mime4j.message.SimpleContentHandler; import org.apache.james.mime4j.parser.MimeEntityConfig; import org.apache.james.mime4j.parser.MimeStreamParser; import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.document.DateTools; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; -import org.apache.lucene.document.NumericField; import org.apache.lucene.document.Field.Index; import org.apache.lucene.document.Field.Store; +import org.apache.lucene.document.NumericField; import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; @@ -92,6 +98,7 @@ import org.apache.lucene.search.ScoreDoc import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TermRangeQuery; import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.WildcardQuery; import org.apache.lucene.store.Directory; @@ -106,7 +113,18 @@ import org.apache.lucene.util.Version; * @param <Id> */ public class LuceneMessageSearchIndex<Id> extends ListeningMessageSearchIndex<Id>{ - + private final static Date MAX_DATE; + private final static Date MIN_DATE; + + static { + Calendar cal = Calendar.getInstance(); + cal.set(9999, 11, 31); + MAX_DATE = cal.getTime(); + + cal.set(0000, 0, 1); + MIN_DATE = cal.getTime(); + } + /** * Default max query results */ @@ -437,13 +455,13 @@ public class LuceneMessageSearchIndex<Id // create an unqiue key for the document which can be used later on updates to find the document doc.add(new Field(ID_FIELD, membership.getMailboxId().toString().toUpperCase(Locale.ENGLISH) +"-" + Long.toString(membership.getUid()), Store.YES, Index.NOT_ANALYZED)); - doc.add(new NumericField(INTERNAL_DATE_FIELD_YEAR_RESOLUTION,Store.NO, true).setLongValue(DateUtils.truncate(membership.getInternalDate(),Calendar.YEAR).getTime())); - doc.add(new NumericField(INTERNAL_DATE_FIELD_MONTH_RESOLUTION,Store.NO, true).setLongValue(DateUtils.truncate(membership.getInternalDate(),Calendar.MONTH).getTime())); - doc.add(new NumericField(INTERNAL_DATE_FIELD_DAY_RESOLUTION,Store.NO, true).setLongValue(DateUtils.truncate(membership.getInternalDate(),Calendar.DAY_OF_MONTH).getTime())); - doc.add(new NumericField(INTERNAL_DATE_FIELD_HOUR_RESOLUTION,Store.NO, true).setLongValue(DateUtils.truncate(membership.getInternalDate(),Calendar.HOUR_OF_DAY).getTime())); - doc.add(new NumericField(INTERNAL_DATE_FIELD_MINUTE_RESOLUTION,Store.NO, true).setLongValue(DateUtils.truncate(membership.getInternalDate(),Calendar.MINUTE).getTime())); - doc.add(new NumericField(INTERNAL_DATE_FIELD_SECOND_RESOLUTION,Store.YES, true).setLongValue(DateUtils.truncate(membership.getInternalDate(),Calendar.SECOND).getTime())); - doc.add(new NumericField(INTERNAL_DATE_FIELD_MILLISECOND_RESOLUTION,Store.NO, true).setLongValue(DateUtils.truncate(membership.getInternalDate(),Calendar.MILLISECOND).getTime())); + doc.add(new Field(INTERNAL_DATE_FIELD_YEAR_RESOLUTION, DateTools.dateToString(membership.getInternalDate(), DateTools.Resolution.YEAR), Store.NO, Index.NOT_ANALYZED)); + doc.add(new Field(INTERNAL_DATE_FIELD_MONTH_RESOLUTION, DateTools.dateToString(membership.getInternalDate(), DateTools.Resolution.MONTH), Store.NO, Index.NOT_ANALYZED)); + doc.add(new Field(INTERNAL_DATE_FIELD_DAY_RESOLUTION, DateTools.dateToString(membership.getInternalDate(), DateTools.Resolution.DAY), Store.NO, Index.NOT_ANALYZED)); + doc.add(new Field(INTERNAL_DATE_FIELD_HOUR_RESOLUTION, DateTools.dateToString(membership.getInternalDate(), DateTools.Resolution.HOUR), Store.NO, Index.NOT_ANALYZED)); + doc.add(new Field(INTERNAL_DATE_FIELD_MINUTE_RESOLUTION, DateTools.dateToString(membership.getInternalDate(), DateTools.Resolution.MINUTE), Store.NO, Index.NOT_ANALYZED)); + doc.add(new Field(INTERNAL_DATE_FIELD_SECOND_RESOLUTION, DateTools.dateToString(membership.getInternalDate(), DateTools.Resolution.SECOND), Store.NO, Index.NOT_ANALYZED)); + doc.add(new Field(INTERNAL_DATE_FIELD_MILLISECOND_RESOLUTION, DateTools.dateToString(membership.getInternalDate(), DateTools.Resolution.MILLISECOND), Store.NO, Index.NOT_ANALYZED)); doc.add(new NumericField(SIZE_FIELD,Store.YES, true).setLongValue(membership.getFullContentOctets())); @@ -470,7 +488,19 @@ public class LuceneMessageSearchIndex<Id doc.add(new Field(PREFIX_HEADER_FIELD + headerName, headerValue, Store.NO, Index.ANALYZED)); if (f instanceof DateTimeField) { - sentDate = ((DateTimeField) f).getDate(); + // We need to make sure we convert it to GMT + final StringReader reader = new StringReader(f.getBody()); + try { + DateTime dateTime = new DateTimeParser(reader).parseAll(); + Calendar cal = getGMT(); + cal.set(dateTime.getYear(), dateTime.getMonth() - 1, dateTime.getDay(), dateTime.getHour(), dateTime.getMinute(), dateTime.getSecond()); + sentDate = cal.getTime(); + + } catch (org.apache.james.mime4j.field.datetime.parser.ParseException e) { + // This should never happen anyway fallback to the already parsed field + sentDate = ((DateTimeField) f).getDate(); + } + } if (f instanceof AddressListField) { @@ -550,16 +580,17 @@ public class LuceneMessageSearchIndex<Id if (sentDate == null) { sentDate = membership.getInternalDate(); } else { - doc.add(new NumericField(SENT_DATE_FIELD_YEAR_RESOLUTION, Store.NO, true).setLongValue(DateUtils.truncate(sentDate, Calendar.YEAR).getTime())); - doc.add(new NumericField(SENT_DATE_FIELD_MONTH_RESOLUTION, Store.NO, true).setLongValue(DateUtils.truncate(sentDate, Calendar.MONTH).getTime())); - doc.add(new NumericField(SENT_DATE_FIELD_DAY_RESOLUTION, Store.NO, true).setLongValue(DateUtils.truncate(sentDate, Calendar.DAY_OF_MONTH).getTime())); - doc.add(new NumericField(SENT_DATE_FIELD_HOUR_RESOLUTION, Store.NO, true).setLongValue(DateUtils.truncate(sentDate, Calendar.HOUR_OF_DAY).getTime())); - doc.add(new NumericField(SENT_DATE_FIELD_MINUTE_RESOLUTION, Store.NO, true).setLongValue(DateUtils.truncate(sentDate, Calendar.MINUTE).getTime())); - doc.add(new NumericField(SENT_DATE_FIELD_SECOND_RESOLUTION, Store.YES, true).setLongValue(DateUtils.truncate(sentDate, Calendar.SECOND).getTime())); - doc.add(new NumericField(SENT_DATE_FIELD_MILLISECOND_RESOLUTION, Store.NO, true).setLongValue(DateUtils.truncate(sentDate, Calendar.MILLISECOND).getTime())); + + doc.add(new Field(SENT_DATE_FIELD_YEAR_RESOLUTION, DateTools.dateToString(sentDate, DateTools.Resolution.YEAR), Store.NO, Index.NOT_ANALYZED)); + doc.add(new Field(SENT_DATE_FIELD_MONTH_RESOLUTION, DateTools.dateToString(sentDate, DateTools.Resolution.MONTH), Store.NO, Index.NOT_ANALYZED)); + doc.add(new Field(SENT_DATE_FIELD_DAY_RESOLUTION, DateTools.dateToString(sentDate, DateTools.Resolution.DAY), Store.NO, Index.NOT_ANALYZED)); + doc.add(new Field(SENT_DATE_FIELD_HOUR_RESOLUTION, DateTools.dateToString(sentDate, DateTools.Resolution.HOUR), Store.NO, Index.NOT_ANALYZED)); + doc.add(new Field(SENT_DATE_FIELD_MINUTE_RESOLUTION, DateTools.dateToString(sentDate, DateTools.Resolution.MINUTE), Store.NO, Index.NOT_ANALYZED)); + doc.add(new Field(SENT_DATE_FIELD_SECOND_RESOLUTION, DateTools.dateToString(sentDate, DateTools.Resolution.SECOND), Store.NO, Index.NOT_ANALYZED)); + doc.add(new Field(SENT_DATE_FIELD_MILLISECOND_RESOLUTION, DateTools.dateToString(sentDate, DateTools.Resolution.MILLISECOND), Store.NO, Index.NOT_ANALYZED)); } - doc.add(new NumericField(SENT_DATE_SORT_FIELD_MILLISECOND_RESOLUTION,Store.YES, true).setLongValue(DateUtils.truncate(sentDate,Calendar.MILLISECOND).getTime())); + doc.add(new Field(SENT_DATE_SORT_FIELD_MILLISECOND_RESOLUTION,DateTools.dateToString(sentDate, DateTools.Resolution.MILLISECOND), Store.NO, Index.NOT_ANALYZED)); doc.add(new Field(FIRST_FROM_MAILBOX_NAME_FIELD, firstFromMailbox, Store.YES, Index.NOT_ANALYZED)); doc.add(new Field(FIRST_TO_MAILBOX_NAME_FIELD, firstToMailbox, Store.YES, Index.NOT_ANALYZED)); @@ -648,6 +679,12 @@ public class LuceneMessageSearchIndex<Id return field; } + + private static Calendar getGMT() { + return Calendar.getInstance(TimeZone.getTimeZone("GMT"), Locale.ENGLISH); + } + + private String toInteralDateField(DateResolution res) { String field; switch (res) { @@ -755,19 +792,39 @@ public class LuceneMessageSearchIndex<Id private Query createQuery(String field, DateOperator dop) throws UnsupportedSearchException { Date date = dop.getDate(); DateResolution res = dop.getDateResultion(); - long value = DateUtils.truncate(date, SearchQuery.toCalendarType(res)).getTime(); + DateTools.Resolution dRes = toResolution(res); + String value = DateTools.dateToString(date, dRes); switch(dop.getType()) { case ON: - return NumericRangeQuery.newLongRange(field ,value, value, true, true); + return new TermQuery(new Term(field ,value)); case BEFORE: - return NumericRangeQuery.newLongRange(field ,0L, value, true, false); + return new TermRangeQuery(field, DateTools.dateToString(MIN_DATE, dRes), value, true, false); case AFTER: - return NumericRangeQuery.newLongRange(field ,value, Long.MAX_VALUE, false, true); + return new TermRangeQuery(field, value, DateTools.dateToString(MAX_DATE, dRes), false, true); default: throw new UnsupportedSearchException(); } } + private DateTools.Resolution toResolution(DateResolution res) { + switch (res) { + case Year: + return DateTools.Resolution.YEAR; + case Month: + return DateTools.Resolution.MONTH; + case Day: + return DateTools.Resolution.DAY; + case Hour: + return DateTools.Resolution.HOUR; + case Minute: + return DateTools.Resolution.MINUTE; + case Second: + return DateTools.Resolution.SECOND; + default: + return DateTools.Resolution.MILLISECOND; + } + } + /** * Return a {@link Query} which is build based on the given {@link SearchQuery.UidCriterion} * --------------------------------------------------------------------- To unsubscribe, e-mail: server-dev-unsubscr...@james.apache.org For additional commands, e-mail: server-dev-h...@james.apache.org