Repository: lucenenet Updated Branches: refs/heads/master 6c653073c -> 9c2d16143
use reference instead of value type to match Lucene's logic Project: http://git-wip-us.apache.org/repos/asf/lucenenet/repo Commit: http://git-wip-us.apache.org/repos/asf/lucenenet/commit/cd4e2e19 Tree: http://git-wip-us.apache.org/repos/asf/lucenenet/tree/cd4e2e19 Diff: http://git-wip-us.apache.org/repos/asf/lucenenet/diff/cd4e2e19 Branch: refs/heads/master Commit: cd4e2e19f1895313e28f01c1bb1673a99c472a5f Parents: 6c65307 Author: Laimonas Simutis <[email protected]> Authored: Thu Jan 15 06:03:22 2015 -0500 Committer: Laimonas Simutis <[email protected]> Committed: Thu Jan 15 06:03:22 2015 -0500 ---------------------------------------------------------------------- src/Lucene.Net.Core/Codecs/DocValuesConsumer.cs | 15 +++--- .../Codecs/Lucene42/Lucene42NormsConsumer.cs | 26 +++++----- .../Lucene45/Lucene45DocValuesConsumer.cs | 50 ++++++++++---------- .../Codecs/Perfield/PerFieldDocValuesFormat.cs | 6 +-- .../Index/NumericDocValuesWriter.cs | 18 +++++-- src/Lucene.Net.Core/Index/ReadersAndUpdates.cs | 9 ++-- .../Index/SortedDocValuesWriter.cs | 2 +- .../Index/SortedSetDocValuesWriter.cs | 4 +- .../Codecs/MissingOrdRemapper.cs | 36 +++++++------- .../asserting/AssertingDocValuesFormat.cs | 30 ++++++------ .../Codecs/lucene3x/PreFlexRWNormsConsumer.cs | 8 ++-- .../Codecs/lucene40/Lucene40DocValuesWriter.cs | 32 ++++++------- .../lucene42/Lucene42DocValuesConsumer.cs | 36 +++++++------- 13 files changed, 141 insertions(+), 131 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lucenenet/blob/cd4e2e19/src/Lucene.Net.Core/Codecs/DocValuesConsumer.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Core/Codecs/DocValuesConsumer.cs b/src/Lucene.Net.Core/Codecs/DocValuesConsumer.cs index 65614ea..ecec5e1 100644 --- a/src/Lucene.Net.Core/Codecs/DocValuesConsumer.cs +++ b/src/Lucene.Net.Core/Codecs/DocValuesConsumer.cs @@ -1,3 +1,4 @@ +using System.IO; using System.Linq; using Lucene.Net.Support; using System; @@ -76,7 +77,7 @@ namespace Lucene.Net.Codecs /// <param name="values"> Iterable of numeric values (one for each document). {@code null} indicates /// a missing value. </param> /// <exception cref="IOException"> if an I/O error occurred. </exception> - public abstract void AddNumericField(FieldInfo field, IEnumerable<long> values); + public abstract void AddNumericField(FieldInfo field, IEnumerable<long?> values); /// <summary> /// Writes binary docvalues for a field. </summary> @@ -93,7 +94,7 @@ namespace Lucene.Net.Codecs /// <param name="docToOrd"> Iterable of ordinals (one for each document). {@code -1} indicates /// a missing value. </param> /// <exception cref="IOException"> if an I/O error occurred. </exception> - public abstract void AddSortedField(FieldInfo field, IEnumerable<BytesRef> values, IEnumerable<long> docToOrd); + public abstract void AddSortedField(FieldInfo field, IEnumerable<BytesRef> values, IEnumerable<long?> docToOrd); /// <summary> /// Writes pre-sorted set docvalues for a field </summary> @@ -103,7 +104,7 @@ namespace Lucene.Net.Codecs /// count indicates a missing value. </param> /// <param name="ords"> Iterable of ordinal occurrences (docToOrdCount*maxDoc total). </param> /// <exception cref="IOException"> if an I/O error occurred. </exception> - public abstract void AddSortedSetField(FieldInfo field, IEnumerable<BytesRef> values, IEnumerable<long> docToOrdCount, IEnumerable<long> ords); + public abstract void AddSortedSetField(FieldInfo field, IEnumerable<BytesRef> values, IEnumerable<long?> docToOrdCount, IEnumerable<long?> ords); /// <summary> /// Merges the numeric docvalues from <code>toMerge</code>. @@ -117,7 +118,7 @@ namespace Lucene.Net.Codecs AddNumericField(fieldInfo, GetMergeNumericFieldEnumerable(fieldInfo, mergeState, toMerge)); } - private IEnumerable<long> GetMergeNumericFieldEnumerable(FieldInfo fieldinfo, MergeState mergeState, IList<NumericDocValues> toMerge) + private IEnumerable<long?> GetMergeNumericFieldEnumerable(FieldInfo fieldinfo, MergeState mergeState, IList<NumericDocValues> toMerge) { int readerUpto = -1; int docIDUpto = 0; @@ -487,7 +488,7 @@ namespace Lucene.Net.Codecs } } - private IEnumerable<long> GetMergeSortedFieldDocToOrdEnumerable(AtomicReader[] readers, SortedDocValues[] dvs, OrdinalMap map) + private IEnumerable<long?> GetMergeSortedFieldDocToOrdEnumerable(AtomicReader[] readers, SortedDocValues[] dvs, OrdinalMap map) { int readerUpTo = -1; int docIDUpTo = 0; @@ -748,7 +749,7 @@ namespace Lucene.Net.Codecs } } - private IEnumerable<long> GetMergeSortedSetDocToOrdCountEnumerable(AtomicReader[] readers, SortedSetDocValues[] dvs) + private IEnumerable<long?> GetMergeSortedSetDocToOrdCountEnumerable(AtomicReader[] readers, SortedSetDocValues[] dvs) { int readerUpto = -1; int docIDUpto = 0; @@ -792,7 +793,7 @@ namespace Lucene.Net.Codecs } } - private IEnumerable<long> GetMergeSortedSetOrdsEnumerable(AtomicReader[] readers, SortedSetDocValues[] dvs, OrdinalMap map) + private IEnumerable<long?> GetMergeSortedSetOrdsEnumerable(AtomicReader[] readers, SortedSetDocValues[] dvs, OrdinalMap map) { int readerUpto = -1; int docIDUpto = 0; http://git-wip-us.apache.org/repos/asf/lucenenet/blob/cd4e2e19/src/Lucene.Net.Core/Codecs/Lucene42/Lucene42NormsConsumer.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Core/Codecs/Lucene42/Lucene42NormsConsumer.cs b/src/Lucene.Net.Core/Codecs/Lucene42/Lucene42NormsConsumer.cs index cccf94f..f947584 100644 --- a/src/Lucene.Net.Core/Codecs/Lucene42/Lucene42NormsConsumer.cs +++ b/src/Lucene.Net.Core/Codecs/Lucene42/Lucene42NormsConsumer.cs @@ -76,7 +76,7 @@ namespace Lucene.Net.Codecs.Lucene42 } } - public override void AddNumericField(FieldInfo field, IEnumerable<long> values) + public override void AddNumericField(FieldInfo field, IEnumerable<long?> values) { Meta.WriteVInt(field.Number); Meta.WriteByte((byte)NUMBER); @@ -91,10 +91,10 @@ namespace Lucene.Net.Codecs.Lucene42 uniqueValues = new HashSet<long>(); long count = 0; - foreach (long nv in values) + foreach (long? nv in values) { Debug.Assert(nv != null); - long v = nv; + long v = nv.Value; if (gcd != 1) { @@ -138,9 +138,9 @@ namespace Lucene.Net.Codecs.Lucene42 if (formatAndBits.bitsPerValue == 8 && minValue >= sbyte.MinValue && maxValue <= sbyte.MaxValue) { Meta.WriteByte((byte)UNCOMPRESSED); // uncompressed - foreach (long nv in values) + foreach (long? nv in values) { - Data.WriteByte((byte)(sbyte)nv); + Data.WriteByte(nv == null ? (byte)0 : (byte)(sbyte)nv.Value); } } else @@ -161,9 +161,9 @@ namespace Lucene.Net.Codecs.Lucene42 Data.WriteVInt(formatAndBits.bitsPerValue); PackedInts.Writer writer = PackedInts.GetWriterNoHeader(Data, formatAndBits.format, MaxDoc, formatAndBits.bitsPerValue, PackedInts.DEFAULT_BUFFER_SIZE); - foreach (long nv in values) + foreach (long? nv in values) { - writer.Add(encode[nv == null ? 0 : nv]); + writer.Add(encode[nv == null ? 0 : nv.Value]); } writer.Finish(); } @@ -177,9 +177,9 @@ namespace Lucene.Net.Codecs.Lucene42 Data.WriteVInt(BLOCK_SIZE); var writer = new BlockPackedWriter(Data, BLOCK_SIZE); - foreach (long nv in values) + foreach (long? nv in values) { - long value = nv; + long value = nv == null ? 0 : nv.Value; writer.Add((value - minValue) / gcd); } writer.Finish(); @@ -192,9 +192,9 @@ namespace Lucene.Net.Codecs.Lucene42 Data.WriteVInt(BLOCK_SIZE); var writer = new BlockPackedWriter(Data, BLOCK_SIZE); - foreach (long nv in values) + foreach (long? nv in values) { - writer.Add(nv); + writer.Add(nv == null ? 0 : nv.Value); } writer.Finish(); } @@ -238,12 +238,12 @@ namespace Lucene.Net.Codecs.Lucene42 throw new System.NotSupportedException(); } - public override void AddSortedField(FieldInfo field, IEnumerable<BytesRef> values, IEnumerable<long> docToOrd) + public override void AddSortedField(FieldInfo field, IEnumerable<BytesRef> values, IEnumerable<long?> docToOrd) { throw new System.NotSupportedException(); } - public override void AddSortedSetField(FieldInfo field, IEnumerable<BytesRef> values, IEnumerable<long> docToOrdCount, IEnumerable<long> ords) + public override void AddSortedSetField(FieldInfo field, IEnumerable<BytesRef> values, IEnumerable<long?> docToOrdCount, IEnumerable<long?> ords) { throw new System.NotSupportedException(); } http://git-wip-us.apache.org/repos/asf/lucenenet/blob/cd4e2e19/src/Lucene.Net.Core/Codecs/Lucene45/Lucene45DocValuesConsumer.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Core/Codecs/Lucene45/Lucene45DocValuesConsumer.cs b/src/Lucene.Net.Core/Codecs/Lucene45/Lucene45DocValuesConsumer.cs index 510297a..6f2fe9a 100644 --- a/src/Lucene.Net.Core/Codecs/Lucene45/Lucene45DocValuesConsumer.cs +++ b/src/Lucene.Net.Core/Codecs/Lucene45/Lucene45DocValuesConsumer.cs @@ -109,12 +109,12 @@ namespace Lucene.Net.Codecs.Lucene45 } } - public override void AddNumericField(FieldInfo field, IEnumerable<long> values) + public override void AddNumericField(FieldInfo field, IEnumerable<long?> values) { AddNumericField(field, values, true); } - internal virtual void AddNumericField(FieldInfo field, IEnumerable<long> values, bool optimizeStorage) + internal virtual void AddNumericField(FieldInfo field, IEnumerable<long?> values, bool optimizeStorage) { long count = 0; long minValue = long.MaxValue; @@ -123,11 +123,13 @@ namespace Lucene.Net.Codecs.Lucene45 bool missing = false; // TODO: more efficient? HashSet<long> uniqueValues = null; + // LUCENE TODO: is this necessary because of multiple iterations? + if (optimizeStorage) { uniqueValues = new HashSet<long>(); - foreach (long nv in values) + foreach (long? nv in values) { long v; if (nv == null) @@ -137,7 +139,7 @@ namespace Lucene.Net.Codecs.Lucene45 } else { - v = nv; + v = nv.Value; } if (gcd != 1) @@ -174,7 +176,7 @@ namespace Lucene.Net.Codecs.Lucene45 } else { - foreach (long nv in values) + foreach (var nv in values) { ++count; } @@ -201,8 +203,7 @@ namespace Lucene.Net.Codecs.Lucene45 if (missing) { Meta.WriteLong(Data.FilePointer); - //LUCENE TO-DO - //WriteMissingBitset(values); + WriteMissingBitset(values); } else { @@ -219,9 +220,9 @@ namespace Lucene.Net.Codecs.Lucene45 Meta.WriteLong(minValue); Meta.WriteLong(gcd); BlockPackedWriter quotientWriter = new BlockPackedWriter(Data, BLOCK_SIZE); - foreach (long nv in values) + foreach (long? nv in values) { - long value = nv == null ? 0 : nv; + long value = nv == null ? 0 : nv.Value; quotientWriter.Add((value - minValue) / gcd); } quotientWriter.Finish(); @@ -229,9 +230,9 @@ namespace Lucene.Net.Codecs.Lucene45 case DELTA_COMPRESSED: BlockPackedWriter writer = new BlockPackedWriter(Data, BLOCK_SIZE); - foreach (long nv in values) + foreach (long? nv in values) { - writer.Add(nv == null ? 0 : nv); + writer.Add(nv == null ? 0 : nv.Value); } writer.Finish(); break; @@ -247,9 +248,9 @@ namespace Lucene.Net.Codecs.Lucene45 } int bitsRequired = PackedInts.BitsRequired(uniqueValues.Count - 1); PackedInts.Writer ordsWriter = PackedInts.GetWriterNoHeader(Data, PackedInts.Format.PACKED, (int)count, bitsRequired, PackedInts.DEFAULT_BUFFER_SIZE); - foreach (long nv in values) + foreach (long? nv in values) { - ordsWriter.Add(encode[nv == null ? 0 : nv]); + ordsWriter.Add(encode[nv == null ? 0 : nv.Value]); } ordsWriter.Finish(); break; @@ -261,7 +262,7 @@ namespace Lucene.Net.Codecs.Lucene45 // TODO: in some cases representing missing with minValue-1 wouldn't take up additional space and so on, // but this is very simple, and algorithms only check this for values of 0 anyway (doesnt slow down normal decode) - internal virtual void WriteMissingBitset(IEnumerable<object> values) + internal virtual void WriteMissingBitset(IEnumerable values) { sbyte bits = 0; int count = 0; @@ -319,8 +320,7 @@ namespace Lucene.Net.Codecs.Lucene45 if (missing) { Meta.WriteLong(Data.FilePointer); - //LUCENE TO-DO - //WriteMissingBitset(values); + WriteMissingBitset(values); } else { @@ -419,7 +419,7 @@ namespace Lucene.Net.Codecs.Lucene45 } } - public override void AddSortedField(FieldInfo field, IEnumerable<BytesRef> values, IEnumerable<long> docToOrd) + public override void AddSortedField(FieldInfo field, IEnumerable<BytesRef> values, IEnumerable<long?> docToOrd) { Meta.WriteVInt(field.Number); Meta.WriteByte((byte)Lucene45DocValuesFormat.SORTED); @@ -427,12 +427,12 @@ namespace Lucene.Net.Codecs.Lucene45 AddNumericField(field, docToOrd, false); } - private static bool IsSingleValued(IEnumerable<long> docToOrdCount) + private static bool IsSingleValued(IEnumerable<long?> docToOrdCount) { return docToOrdCount.All(ordCount => ordCount <= 1); } - public override void AddSortedSetField(FieldInfo field, IEnumerable<BytesRef> values, IEnumerable<long> docToOrdCount, IEnumerable<long> ords) + public override void AddSortedSetField(FieldInfo field, IEnumerable<BytesRef> values, IEnumerable<long?> docToOrdCount, IEnumerable<long?> ords) { Meta.WriteVInt(field.Number); Meta.WriteByte((byte)Lucene45DocValuesFormat.SORTED_SET); @@ -466,24 +466,24 @@ namespace Lucene.Net.Codecs.Lucene45 var writer = new MonotonicBlockPackedWriter(Data, BLOCK_SIZE); long addr = 0; - foreach (int v in docToOrdCount) + foreach (long? v in docToOrdCount) { - addr += (long)v; + addr += v.Value; writer.Add(addr); } writer.Finish(); } - private IEnumerable<long> GetSortedSetEnumerable(IEnumerable<long> docToOrdCount, IEnumerable<long> ords) + private IEnumerable<long?> GetSortedSetEnumerable(IEnumerable<long?> docToOrdCount, IEnumerable<long?> ords) { - IEnumerator<long> docToOrdCountIter = docToOrdCount.GetEnumerator(); - IEnumerator<long> ordsIter = ords.GetEnumerator(); + IEnumerator<long?> docToOrdCountIter = docToOrdCount.GetEnumerator(); + IEnumerator<long?> ordsIter = ords.GetEnumerator(); const long MISSING_ORD = -1; while (docToOrdCountIter.MoveNext()) { - long current = docToOrdCountIter.Current; + long current = docToOrdCountIter.Current.Value; if (current == 0) { yield return MISSING_ORD; http://git-wip-us.apache.org/repos/asf/lucenenet/blob/cd4e2e19/src/Lucene.Net.Core/Codecs/Perfield/PerFieldDocValuesFormat.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Core/Codecs/Perfield/PerFieldDocValuesFormat.cs b/src/Lucene.Net.Core/Codecs/Perfield/PerFieldDocValuesFormat.cs index 2ae8cc9..e64abf8 100644 --- a/src/Lucene.Net.Core/Codecs/Perfield/PerFieldDocValuesFormat.cs +++ b/src/Lucene.Net.Core/Codecs/Perfield/PerFieldDocValuesFormat.cs @@ -107,7 +107,7 @@ namespace Lucene.Net.Codecs.Perfield SegmentWriteState = state; } - public override void AddNumericField(FieldInfo field, IEnumerable<long> values) + public override void AddNumericField(FieldInfo field, IEnumerable<long?> values) { GetInstance(field).AddNumericField(field, values); } @@ -117,12 +117,12 @@ namespace Lucene.Net.Codecs.Perfield GetInstance(field).AddBinaryField(field, values); } - public override void AddSortedField(FieldInfo field, IEnumerable<BytesRef> values, IEnumerable<long> docToOrd) + public override void AddSortedField(FieldInfo field, IEnumerable<BytesRef> values, IEnumerable<long?> docToOrd) { GetInstance(field).AddSortedField(field, values, docToOrd); } - public override void AddSortedSetField(FieldInfo field, IEnumerable<BytesRef> values, IEnumerable<long> docToOrdCount, IEnumerable<long> ords) + public override void AddSortedSetField(FieldInfo field, IEnumerable<BytesRef> values, IEnumerable<long?> docToOrdCount, IEnumerable<long?> ords) { GetInstance(field).AddSortedSetField(field, values, docToOrdCount, ords); } http://git-wip-us.apache.org/repos/asf/lucenenet/blob/cd4e2e19/src/Lucene.Net.Core/Index/NumericDocValuesWriter.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Core/Index/NumericDocValuesWriter.cs b/src/Lucene.Net.Core/Index/NumericDocValuesWriter.cs index 6194d4c..73bbca0 100644 --- a/src/Lucene.Net.Core/Index/NumericDocValuesWriter.cs +++ b/src/Lucene.Net.Core/Index/NumericDocValuesWriter.cs @@ -1,3 +1,4 @@ +using System; using System.Collections.Generic; namespace Lucene.Net.Index @@ -99,24 +100,31 @@ namespace Lucene.Net.Index dvConsumer.AddNumericField(FieldInfo, GetNumericIterator(maxDoc)); } - private IEnumerable<long> GetNumericIterator(int maxDoc) + private IEnumerable<long?> GetNumericIterator(int maxDoc) { // .NET Port: using yield return instead of custom iterator type. Much less code. - AbstractAppendingLongBuffer.Iterator iter = Pending.GetIterator(); int size = (int)Pending.Size(); int upto = 0; while (upto < maxDoc) { - long value; + long? value; if (upto < size) { - value = iter.Next(); + var v = iter.Next(); + if (DocsWithField == null || DocsWithField.Get(upto)) + { + value = v; + } + else + { + value = null; + } } else { - value = 0; + value = DocsWithField != null ? (long?) null : MISSING; } upto++; // TODO: make reusable Number http://git-wip-us.apache.org/repos/asf/lucenenet/blob/cd4e2e19/src/Lucene.Net.Core/Index/ReadersAndUpdates.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Core/Index/ReadersAndUpdates.cs b/src/Lucene.Net.Core/Index/ReadersAndUpdates.cs index a779eab..1becab9 100644 --- a/src/Lucene.Net.Core/Index/ReadersAndUpdates.cs +++ b/src/Lucene.Net.Core/Index/ReadersAndUpdates.cs @@ -2,6 +2,7 @@ using System; using System.Collections.Generic; using System.Diagnostics; using System.Globalization; +using System.Linq; using System.Text; using Lucene.Net.Documents; @@ -669,7 +670,7 @@ namespace Lucene.Net.Index } } - private IEnumerable<long> GetLongEnumerable(SegmentReader reader, string field, NumericDocValuesFieldUpdates fieldUpdates) + private IEnumerable<long?> GetLongEnumerable(SegmentReader reader, string field, NumericDocValuesFieldUpdates fieldUpdates) { int maxDoc = reader.MaxDoc; Bits DocsWithField = reader.GetDocsWithField(field); @@ -681,9 +682,9 @@ namespace Lucene.Net.Index { if (curDoc == updateDoc) //document has an updated value { - long? value = (long?)(iter.Value()); // either null or updated + long? value = (long?)iter.Value(); // either null or updated updateDoc = iter.NextDoc(); //prepare for next round - yield return value ?? default(long); + yield return value; } else { // no update for this document @@ -694,7 +695,7 @@ namespace Lucene.Net.Index } else { - yield return default(long); + yield return null; } } } http://git-wip-us.apache.org/repos/asf/lucenenet/blob/cd4e2e19/src/Lucene.Net.Core/Index/SortedDocValuesWriter.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Core/Index/SortedDocValuesWriter.cs b/src/Lucene.Net.Core/Index/SortedDocValuesWriter.cs index 3300fdc..eefdb6d 100644 --- a/src/Lucene.Net.Core/Index/SortedDocValuesWriter.cs +++ b/src/Lucene.Net.Core/Index/SortedDocValuesWriter.cs @@ -141,7 +141,7 @@ namespace Lucene.Net.Index } } - private IEnumerable<long> GetOrdsEnumberable(int maxDoc, int[] ordMap) + private IEnumerable<long?> GetOrdsEnumberable(int maxDoc, int[] ordMap) { AppendingDeltaPackedLongBuffer.Iterator iter = Pending.GetIterator(); http://git-wip-us.apache.org/repos/asf/lucenenet/blob/cd4e2e19/src/Lucene.Net.Core/Index/SortedSetDocValuesWriter.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Core/Index/SortedSetDocValuesWriter.cs b/src/Lucene.Net.Core/Index/SortedSetDocValuesWriter.cs index a3fe6a7..6969b3a 100644 --- a/src/Lucene.Net.Core/Index/SortedSetDocValuesWriter.cs +++ b/src/Lucene.Net.Core/Index/SortedSetDocValuesWriter.cs @@ -196,7 +196,7 @@ namespace Lucene.Net.Index } } - private IEnumerable<long> GetOrdsEnumberable(int maxDoc) + private IEnumerable<long?> GetOrdsEnumberable(int maxDoc) { AppendingDeltaPackedLongBuffer.Iterator iter = PendingCounts.GetIterator(); @@ -208,7 +208,7 @@ namespace Lucene.Net.Index } } - private IEnumerable<long> GetOrdCountEnumberable(int maxCountPerDoc, int[] ordMap) + private IEnumerable<long?> GetOrdCountEnumberable(int maxCountPerDoc, int[] ordMap) { int currentUpTo = 0, currentLength = 0; AppendingPackedLongBuffer.Iterator iter = Pending.GetIterator(); http://git-wip-us.apache.org/repos/asf/lucenenet/blob/cd4e2e19/src/Lucene.Net.TestFramework/Codecs/MissingOrdRemapper.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.TestFramework/Codecs/MissingOrdRemapper.cs b/src/Lucene.Net.TestFramework/Codecs/MissingOrdRemapper.cs index 2d5a5c8..c610022 100644 --- a/src/Lucene.Net.TestFramework/Codecs/MissingOrdRemapper.cs +++ b/src/Lucene.Net.TestFramework/Codecs/MissingOrdRemapper.cs @@ -112,21 +112,21 @@ namespace Lucene.Net.Codecs /// <summary> /// remaps ord -1 to ord 0 on this iterable. </summary> - public static IEnumerable<long> MapMissingToOrd0(IEnumerable<long> iterable) + public static IEnumerable<long?> MapMissingToOrd0(IEnumerable<long?> iterable) { return new IterableAnonymousInnerClassHelper2(iterable); } - private class IterableAnonymousInnerClassHelper2 : IEnumerable<long> + private class IterableAnonymousInnerClassHelper2 : IEnumerable<long?> { - private IEnumerable<long> Iterable; + private IEnumerable<long?> Iterable; - public IterableAnonymousInnerClassHelper2(IEnumerable<long> iterable) + public IterableAnonymousInnerClassHelper2(IEnumerable<long?> iterable) { this.Iterable = iterable; } - public IEnumerator<long> GetEnumerator() + public IEnumerator<long?> GetEnumerator() { return new IteratorAnonymousInnerClassHelper2(this); } @@ -136,7 +136,7 @@ namespace Lucene.Net.Codecs return GetEnumerator(); } - private class IteratorAnonymousInnerClassHelper2 : IEnumerator<long> + private class IteratorAnonymousInnerClassHelper2 : IEnumerator<long?> { private readonly IterableAnonymousInnerClassHelper2 OuterInstance; @@ -146,7 +146,7 @@ namespace Lucene.Net.Codecs @in = outerInstance.Iterable.GetEnumerator(); } - private IEnumerator<long> @in; + private IEnumerator<long?> @in; private long current; public bool MoveNext() @@ -156,14 +156,14 @@ namespace Lucene.Net.Codecs return false; } - long n = @in.Current; + long n = @in.Current.Value; current = n == -1 ? 0 : n; return true; } - public long Current + public long? Current { get { return current; } } @@ -186,21 +186,21 @@ namespace Lucene.Net.Codecs /// <summary> /// remaps every ord+1 on this iterable </summary> - public static IEnumerable<long> MapAllOrds(IEnumerable<long> iterable) + public static IEnumerable<long?> MapAllOrds(IEnumerable<long?> iterable) { return new IterableAnonymousInnerClassHelper3(iterable); } - private class IterableAnonymousInnerClassHelper3 : IEnumerable<long> + private class IterableAnonymousInnerClassHelper3 : IEnumerable<long?> { - private IEnumerable<long> Iterable; + private IEnumerable<long?> Iterable; - public IterableAnonymousInnerClassHelper3(IEnumerable<long> iterable) + public IterableAnonymousInnerClassHelper3(IEnumerable<long?> iterable) { this.Iterable = iterable; } - public IEnumerator<long> GetEnumerator() + public IEnumerator<long?> GetEnumerator() { return new IteratorAnonymousInnerClassHelper3(this); } @@ -210,7 +210,7 @@ namespace Lucene.Net.Codecs return GetEnumerator(); } - private class IteratorAnonymousInnerClassHelper3 : IEnumerator<long> + private class IteratorAnonymousInnerClassHelper3 : IEnumerator<long?> { private readonly IterableAnonymousInnerClassHelper3 OuterInstance; @@ -220,7 +220,7 @@ namespace Lucene.Net.Codecs @in = outerInstance.Iterable.GetEnumerator(); } - private IEnumerator<long> @in; + private IEnumerator<long?> @in; private long current; public bool MoveNext() @@ -230,13 +230,13 @@ namespace Lucene.Net.Codecs return false; } - long n = @in.Current; + long n = @in.Current.Value; current = n + 1; return true; } - public long Current + public long? Current { get { return current; } } http://git-wip-us.apache.org/repos/asf/lucenenet/blob/cd4e2e19/src/Lucene.Net.TestFramework/Codecs/asserting/AssertingDocValuesFormat.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.TestFramework/Codecs/asserting/AssertingDocValuesFormat.cs b/src/Lucene.Net.TestFramework/Codecs/asserting/AssertingDocValuesFormat.cs index fd0e528..829aadc 100644 --- a/src/Lucene.Net.TestFramework/Codecs/asserting/AssertingDocValuesFormat.cs +++ b/src/Lucene.Net.TestFramework/Codecs/asserting/AssertingDocValuesFormat.cs @@ -74,10 +74,10 @@ namespace Lucene.Net.Codecs.asserting this.MaxDoc = maxDoc; } - public override void AddNumericField(FieldInfo field, IEnumerable<long> values) + public override void AddNumericField(FieldInfo field, IEnumerable<long?> values) { int count = 0; - foreach (long v in values) + foreach (var v in values) { count++; } @@ -99,7 +99,7 @@ namespace Lucene.Net.Codecs.asserting @in.AddBinaryField(field, values); } - public override void AddSortedField(FieldInfo field, IEnumerable<BytesRef> values, IEnumerable<long> docToOrd) + public override void AddSortedField(FieldInfo field, IEnumerable<BytesRef> values, IEnumerable<long?> docToOrd) { int valueCount = 0; BytesRef lastValue = null; @@ -119,10 +119,10 @@ namespace Lucene.Net.Codecs.asserting FixedBitSet seenOrds = new FixedBitSet(valueCount); int count = 0; - foreach (long v in docToOrd) + foreach (long? v in docToOrd) { Debug.Assert(v != null); - int ord = (int)v; + int ord = (int)v.Value; Debug.Assert(ord >= -1 && ord < valueCount); if (ord >= 0) { @@ -138,7 +138,7 @@ namespace Lucene.Net.Codecs.asserting @in.AddSortedField(field, values, docToOrd); } - public override void AddSortedSetField(FieldInfo field, IEnumerable<BytesRef> values, IEnumerable<long> docToOrdCount, IEnumerable<long> ords) + public override void AddSortedSetField(FieldInfo field, IEnumerable<BytesRef> values, IEnumerable<long?> docToOrdCount, IEnumerable<long?> ords) { long valueCount = 0; BytesRef lastValue = null; @@ -157,11 +157,11 @@ namespace Lucene.Net.Codecs.asserting int docCount = 0; long ordCount = 0; LongBitSet seenOrds = new LongBitSet(valueCount); - IEnumerator<long> ordIterator = ords.GetEnumerator(); - foreach (long v in docToOrdCount) + IEnumerator<long?> ordIterator = ords.GetEnumerator(); + foreach (long? v in docToOrdCount) { Debug.Assert(v != null); - int count = (int)v; + int count = (int)v.Value; Debug.Assert(count >= 0); docCount++; ordCount += count; @@ -170,9 +170,9 @@ namespace Lucene.Net.Codecs.asserting for (int i = 0; i < count; i++) { ordIterator.MoveNext(); - long o = ordIterator.Current; + long? o = ordIterator.Current; Debug.Assert(o != null); - long ord = (long)o; + long ord = o.Value; Debug.Assert(ord >= 0 && ord < valueCount); Debug.Assert(ord > lastOrd, "ord=" + ord + ",lastOrd=" + lastOrd); seenOrds.Set(ord); @@ -207,10 +207,10 @@ namespace Lucene.Net.Codecs.asserting this.MaxDoc = maxDoc; } - public override void AddNumericField(FieldInfo field, IEnumerable<long> values) + public override void AddNumericField(FieldInfo field, IEnumerable<long?> values) { int count = 0; - foreach (long v in values) + foreach (long? v in values) { Debug.Assert(v != null); count++; @@ -231,12 +231,12 @@ namespace Lucene.Net.Codecs.asserting throw new InvalidOperationException(); } - public override void AddSortedField(FieldInfo field, IEnumerable<BytesRef> values, IEnumerable<long> docToOrd) + public override void AddSortedField(FieldInfo field, IEnumerable<BytesRef> values, IEnumerable<long?> docToOrd) { throw new InvalidOperationException(); } - public override void AddSortedSetField(FieldInfo field, IEnumerable<BytesRef> values, IEnumerable<long> docToOrdCount, IEnumerable<long> ords) + public override void AddSortedSetField(FieldInfo field, IEnumerable<BytesRef> values, IEnumerable<long?> docToOrdCount, IEnumerable<long?> ords) { throw new InvalidOperationException(); } http://git-wip-us.apache.org/repos/asf/lucenenet/blob/cd4e2e19/src/Lucene.Net.TestFramework/Codecs/lucene3x/PreFlexRWNormsConsumer.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.TestFramework/Codecs/lucene3x/PreFlexRWNormsConsumer.cs b/src/Lucene.Net.TestFramework/Codecs/lucene3x/PreFlexRWNormsConsumer.cs index 3783440..cf9153a 100644 --- a/src/Lucene.Net.TestFramework/Codecs/lucene3x/PreFlexRWNormsConsumer.cs +++ b/src/Lucene.Net.TestFramework/Codecs/lucene3x/PreFlexRWNormsConsumer.cs @@ -78,10 +78,10 @@ namespace Lucene.Net.Codecs.Lucene3x } } - public override void AddNumericField(FieldInfo field, IEnumerable<long> values) + public override void AddNumericField(FieldInfo field, IEnumerable<long?> values) { Debug.Assert(field.Number > LastFieldNumber, "writing norms fields out of order" + LastFieldNumber + " -> " + field.Number); - foreach (long n in values) + foreach (long? n in values) { if ((long)n < sbyte.MinValue || (long)n > sbyte.MaxValue) { @@ -103,12 +103,12 @@ namespace Lucene.Net.Codecs.Lucene3x throw new InvalidOperationException(); } - public override void AddSortedField(FieldInfo field, IEnumerable<BytesRef> values, IEnumerable<long> docToOrd) + public override void AddSortedField(FieldInfo field, IEnumerable<BytesRef> values, IEnumerable<long?> docToOrd) { throw new InvalidOperationException(); } - public override void AddSortedSetField(FieldInfo field, IEnumerable<BytesRef> values, IEnumerable<long> docToOrdCount, IEnumerable<long> ords) + public override void AddSortedSetField(FieldInfo field, IEnumerable<BytesRef> values, IEnumerable<long?> docToOrdCount, IEnumerable<long?> ords) { throw new InvalidOperationException(); } http://git-wip-us.apache.org/repos/asf/lucenenet/blob/cd4e2e19/src/Lucene.Net.TestFramework/Codecs/lucene40/Lucene40DocValuesWriter.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.TestFramework/Codecs/lucene40/Lucene40DocValuesWriter.cs b/src/Lucene.Net.TestFramework/Codecs/lucene40/Lucene40DocValuesWriter.cs index 739cceb..e8cfda1 100644 --- a/src/Lucene.Net.TestFramework/Codecs/lucene40/Lucene40DocValuesWriter.cs +++ b/src/Lucene.Net.TestFramework/Codecs/lucene40/Lucene40DocValuesWriter.cs @@ -49,12 +49,12 @@ namespace Lucene.Net.Codecs.Lucene40 this.Dir = new CompoundFileDirectory(state.Directory, filename, state.Context, true); } - public override void AddNumericField(FieldInfo field, IEnumerable<long> values) + public override void AddNumericField(FieldInfo field, IEnumerable<long?> values) { // examine the values to determine best type to use long minValue = long.MaxValue; long maxValue = long.MinValue; - foreach (long n in values) + foreach (long? n in values) { long v = n == null ? 0 : (long)n; minValue = Math.Min(minValue, v); @@ -100,40 +100,40 @@ namespace Lucene.Net.Codecs.Lucene40 } } - private void AddBytesField(FieldInfo field, IndexOutput output, IEnumerable<long> values) + private void AddBytesField(FieldInfo field, IndexOutput output, IEnumerable<long?> values) { field.PutAttribute(LegacyKey, LegacyDocValuesType.FIXED_INTS_8.Name); CodecUtil.WriteHeader(output, Lucene40DocValuesFormat.INTS_CODEC_NAME, Lucene40DocValuesFormat.INTS_VERSION_CURRENT); output.WriteInt(1); // size - foreach (long n in values) + foreach (long? n in values) { output.WriteByte(n == null ? (byte)0 : (byte)n); } } - private void AddShortsField(FieldInfo field, IndexOutput output, IEnumerable<long> values) + private void AddShortsField(FieldInfo field, IndexOutput output, IEnumerable<long?> values) { field.PutAttribute(LegacyKey, LegacyDocValuesType.FIXED_INTS_16.Name); CodecUtil.WriteHeader(output, Lucene40DocValuesFormat.INTS_CODEC_NAME, Lucene40DocValuesFormat.INTS_VERSION_CURRENT); output.WriteInt(2); // size - foreach (long n in values) + foreach (long? n in values) { output.WriteShort(n == null ? (short)0 : (short)n); } } - private void AddIntsField(FieldInfo field, IndexOutput output, IEnumerable<long> values) + private void AddIntsField(FieldInfo field, IndexOutput output, IEnumerable<long?> values) { field.PutAttribute(LegacyKey, LegacyDocValuesType.FIXED_INTS_32.Name); CodecUtil.WriteHeader(output, Lucene40DocValuesFormat.INTS_CODEC_NAME, Lucene40DocValuesFormat.INTS_VERSION_CURRENT); output.WriteInt(4); // size - foreach (long n in values) + foreach (long? n in values) { output.WriteInt(n == null ? 0 : (int)n); } } - private void AddVarIntsField(FieldInfo field, IndexOutput output, IEnumerable<long> values, long minValue, long maxValue) + private void AddVarIntsField(FieldInfo field, IndexOutput output, IEnumerable<long?> values, long minValue, long maxValue) { field.PutAttribute(LegacyKey, LegacyDocValuesType.VAR_INTS.Name); @@ -145,9 +145,9 @@ namespace Lucene.Net.Codecs.Lucene40 { // writes longs output.WriteByte((byte)Lucene40DocValuesFormat.VAR_INTS_FIXED_64); - foreach (long n in values) + foreach (long? n in values) { - output.WriteLong(n == null ? 0 : n); + output.WriteLong(n == null ? 0 : n.Value); } } else @@ -157,7 +157,7 @@ namespace Lucene.Net.Codecs.Lucene40 output.WriteLong(minValue); output.WriteLong(0 - minValue); // default value (representation of 0) PackedInts.Writer writer = PackedInts.GetWriter(output, State.SegmentInfo.DocCount, PackedInts.BitsRequired(delta), PackedInts.DEFAULT); - foreach (long n in values) + foreach (long? n in values) { long v = n == null ? 0 : (long)n; writer.Add(v - minValue); @@ -454,7 +454,7 @@ namespace Lucene.Net.Codecs.Lucene40 } } - public override void AddSortedField(FieldInfo field, IEnumerable<BytesRef> values, IEnumerable<long> docToOrd) + public override void AddSortedField(FieldInfo field, IEnumerable<BytesRef> values, IEnumerable<long?> docToOrd) { // examine the values to determine best type to use int minLength = int.MaxValue; @@ -526,7 +526,7 @@ namespace Lucene.Net.Codecs.Lucene40 } } - private void AddFixedSortedBytesField(FieldInfo field, IndexOutput data, IndexOutput index, IEnumerable<BytesRef> values, IEnumerable<long> docToOrd, int length) + private void AddFixedSortedBytesField(FieldInfo field, IndexOutput data, IndexOutput index, IEnumerable<BytesRef> values, IEnumerable<long?> docToOrd, int length) { field.PutAttribute(LegacyKey, LegacyDocValuesType.BYTES_FIXED_SORTED.Name); @@ -557,7 +557,7 @@ namespace Lucene.Net.Codecs.Lucene40 w.Finish(); } - private void AddVarSortedBytesField(FieldInfo field, IndexOutput data, IndexOutput index, IEnumerable<BytesRef> values, IEnumerable<long> docToOrd) + private void AddVarSortedBytesField(FieldInfo field, IndexOutput data, IndexOutput index, IEnumerable<BytesRef> values, IEnumerable<long?> docToOrd) { field.PutAttribute(LegacyKey, LegacyDocValuesType.BYTES_VAR_SORTED.Name); @@ -607,7 +607,7 @@ namespace Lucene.Net.Codecs.Lucene40 ords.Finish(); } - public override void AddSortedSetField(FieldInfo field, IEnumerable<BytesRef> values, IEnumerable<long> docToOrdCount, IEnumerable<long> ords) + public override void AddSortedSetField(FieldInfo field, IEnumerable<BytesRef> values, IEnumerable<long?> docToOrdCount, IEnumerable<long?> ords) { throw new System.NotSupportedException("Lucene 4.0 does not support SortedSet docvalues"); } http://git-wip-us.apache.org/repos/asf/lucenenet/blob/cd4e2e19/src/Lucene.Net.TestFramework/Codecs/lucene42/Lucene42DocValuesConsumer.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.TestFramework/Codecs/lucene42/Lucene42DocValuesConsumer.cs b/src/Lucene.Net.TestFramework/Codecs/lucene42/Lucene42DocValuesConsumer.cs index f59e0bf..32a0609 100644 --- a/src/Lucene.Net.TestFramework/Codecs/lucene42/Lucene42DocValuesConsumer.cs +++ b/src/Lucene.Net.TestFramework/Codecs/lucene42/Lucene42DocValuesConsumer.cs @@ -79,12 +79,12 @@ namespace Lucene.Net.Codecs.Lucene42 } } - public override void AddNumericField(FieldInfo field, IEnumerable<long> values) + public override void AddNumericField(FieldInfo field, IEnumerable<long?> values) { AddNumericField(field, values, true); } - internal virtual void AddNumericField(FieldInfo field, IEnumerable<long> values, bool optimizeStorage) + internal virtual void AddNumericField(FieldInfo field, IEnumerable<long?> values, bool optimizeStorage) { Meta.WriteVInt(field.Number); Meta.WriteByte((byte)Lucene42DocValuesProducer.NUMBER); @@ -99,7 +99,7 @@ namespace Lucene.Net.Codecs.Lucene42 uniqueValues = new HashSet<long>(); long count = 0; - foreach (long nv in values) + foreach (long? nv in values) { // TODO: support this as MemoryDVFormat (and be smart about missing maybe) long v = nv == null ? 0 : (long)nv; @@ -146,7 +146,7 @@ namespace Lucene.Net.Codecs.Lucene42 if (formatAndBits.bitsPerValue == 8 && minValue >= sbyte.MinValue && maxValue <= sbyte.MaxValue) { Meta.WriteByte((byte)Lucene42DocValuesProducer.UNCOMPRESSED); // uncompressed - foreach (long nv in values) + foreach (long? nv in values) { Data.WriteByte(nv == null ? (byte)0 : (byte)nv); } @@ -168,7 +168,7 @@ namespace Lucene.Net.Codecs.Lucene42 Data.WriteVInt(formatAndBits.bitsPerValue); PackedInts.Writer writer = PackedInts.GetWriterNoHeader(Data, formatAndBits.format, MaxDoc, formatAndBits.bitsPerValue, PackedInts.DEFAULT_BUFFER_SIZE); - foreach (long nv in values) + foreach (long? nv in values) { writer.Add(encode[nv == null ? 0 : (long)nv]); } @@ -184,7 +184,7 @@ namespace Lucene.Net.Codecs.Lucene42 Data.WriteVInt(Lucene42DocValuesProducer.BLOCK_SIZE); BlockPackedWriter writer = new BlockPackedWriter(Data, Lucene42DocValuesProducer.BLOCK_SIZE); - foreach (long nv in values) + foreach (long? nv in values) { long value = nv == null ? 0 : (long)nv; writer.Add((value - minValue) / gcd); @@ -199,7 +199,7 @@ namespace Lucene.Net.Codecs.Lucene42 Data.WriteVInt(Lucene42DocValuesProducer.BLOCK_SIZE); BlockPackedWriter writer = new BlockPackedWriter(Data, Lucene42DocValuesProducer.BLOCK_SIZE); - foreach (long nv in values) + foreach (long? nv in values) { writer.Add(nv == null ? 0 : (long)nv); } @@ -304,16 +304,16 @@ namespace Lucene.Net.Codecs.Lucene42 Meta.WriteVLong(ord); } - public override void AddSortedField(FieldInfo field, IEnumerable<BytesRef> values, IEnumerable<long> docToOrd) + public override void AddSortedField(FieldInfo field, IEnumerable<BytesRef> values, IEnumerable<long?> docToOrd) { // three cases for simulating the old writer: // 1. no missing // 2. missing (and empty string in use): remap ord=-1 -> ord=0 // 3. missing (and empty string not in use): remap all ords +1, insert empty string into values bool anyMissing = false; - foreach (long n in docToOrd) + foreach (long? n in docToOrd) { - if ((long)n == -1) + if (n.Value == -1) { anyMissing = true; break; @@ -349,7 +349,7 @@ namespace Lucene.Net.Codecs.Lucene42 } // note: this might not be the most efficient... but its fairly simple - public override void AddSortedSetField(FieldInfo field, IEnumerable<BytesRef> values, IEnumerable<long> docToOrdCount, IEnumerable<long> ords) + public override void AddSortedSetField(FieldInfo field, IEnumerable<BytesRef> values, IEnumerable<long?> docToOrdCount, IEnumerable<long?> ords) { // write the ordinals as a binary field AddBinaryField(field, new IterableAnonymousInnerClassHelper(this, values, docToOrdCount, ords)); @@ -362,11 +362,11 @@ namespace Lucene.Net.Codecs.Lucene42 { private readonly Lucene42DocValuesConsumer OuterInstance; - private IEnumerable<long> DocToOrdCount; - private IEnumerable<long> Ords; + private IEnumerable<long?> DocToOrdCount; + private IEnumerable<long?> Ords; private IEnumerable<BytesRef> Values; - public IterableAnonymousInnerClassHelper(Lucene42DocValuesConsumer outerInstance, IEnumerable<BytesRef> values, IEnumerable<long> docToOrdCount, IEnumerable<long> ords) + public IterableAnonymousInnerClassHelper(Lucene42DocValuesConsumer outerInstance, IEnumerable<BytesRef> values, IEnumerable<long?> docToOrdCount, IEnumerable<long?> ords) { this.OuterInstance = outerInstance; this.Values = values; @@ -393,10 +393,10 @@ namespace Lucene.Net.Codecs.Lucene42 internal BytesRef @ref = new BytesRef(); internal readonly IEnumerator<BytesRef> Values; - internal readonly IEnumerator<long> Counts; - internal readonly IEnumerator<long> Ords; + internal readonly IEnumerator<long?> Counts; + internal readonly IEnumerator<long?> Ords; - internal SortedSetIterator(IEnumerator<BytesRef> values, IEnumerator<long> counts, IEnumerator<long> ords) + internal SortedSetIterator(IEnumerator<BytesRef> values, IEnumerator<long?> counts, IEnumerator<long?> ords) { this.Values = values; this.Counts = counts; @@ -453,7 +453,7 @@ namespace Lucene.Net.Codecs.Lucene42 for (int i = 0; i < count; i++) { Ords.MoveNext(); - long ord = Ords.Current; + long ord = Ords.Current.Value; @out.WriteVLong(ord - lastOrd); lastOrd = ord; }
