I think this commit is missing the TypePromoter class file?
On Oct 10, 2011, at 11:28 AM, [email protected] wrote: > Author: simonw > Date: Mon Oct 10 15:28:07 2011 > New Revision: 1181020 > > URL: http://svn.apache.org/viewvc?rev=1181020&view=rev > Log: > LUCENE-3186: Promote docvalues types during merge if ValueType or size > differes across segments > > Modified: > lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldInfo.java > > lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/MultiPerDocValues.java > lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentMerger.java > > lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/DocValuesConsumer.java > > lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/PerDocConsumer.java > > lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/FixedDerefBytesImpl.java > > lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/FixedSortedBytesImpl.java > > lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/FixedStraightBytesImpl.java > lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/Floats.java > > lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/IndexDocValues.java > lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/Ints.java > > lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/MultiIndexDocValues.java > > lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/VarStraightBytesImpl.java > lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/Writer.java > > Modified: > lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldInfo.java > URL: > http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldInfo.java?rev=1181020&r1=1181019&r2=1181020&view=diff > ============================================================================== > --- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldInfo.java > (original) > +++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldInfo.java > Mon Oct 10 15:28:07 2011 > @@ -86,7 +86,7 @@ public final class FieldInfo { > public int getCodecId() { > return codecId; > } > - > + > @Override > public Object clone() { > FieldInfo clone = new FieldInfo(name, isIndexed, number, storeTermVector, > storePositionWithTermVector, > @@ -132,6 +132,12 @@ public final class FieldInfo { > } > } > > + public void resetDocValues(ValueType v) { > + if (docValues != null) { > + docValues = v; > + } > + } > + > public boolean hasDocValues() { > return docValues != null; > } > > Modified: > lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/MultiPerDocValues.java > URL: > http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/MultiPerDocValues.java?rev=1181020&r1=1181019&r2=1181020&view=diff > ============================================================================== > --- > lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/MultiPerDocValues.java > (original) > +++ > lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/MultiPerDocValues.java > Mon Oct 10 15:28:07 2011 > @@ -128,7 +128,7 @@ public class MultiPerDocValues extends P > if (docsUpto != start) { > type = values.type(); > docValuesIndex.add(new MultiIndexDocValues.DocValuesIndex( > - new MultiIndexDocValues.DummyDocValues(start, type), > docsUpto, start > + new MultiIndexDocValues.EmptyDocValues(start, type), > docsUpto, start > - docsUpto)); > } > docValuesIndex.add(new MultiIndexDocValues.DocValuesIndex(values, > start, > @@ -137,7 +137,7 @@ public class MultiPerDocValues extends P > > } else if (i + 1 == subs.length && !docValuesIndex.isEmpty()) { > docValuesIndex.add(new MultiIndexDocValues.DocValuesIndex( > - new MultiIndexDocValues.DummyDocValues(start, type), docsUpto, > start > + new MultiIndexDocValues.EmptyDocValues(start, type), docsUpto, > start > - docsUpto)); > } > } > > Modified: > lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentMerger.java > URL: > http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentMerger.java?rev=1181020&r1=1181019&r2=1181020&view=diff > ============================================================================== > --- > lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentMerger.java > (original) > +++ > lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentMerger.java > Mon Oct 10 15:28:07 2011 > @@ -33,7 +33,6 @@ import org.apache.lucene.index.codecs.Fi > import org.apache.lucene.index.codecs.FieldsWriter; > import org.apache.lucene.index.codecs.MergeState; > import org.apache.lucene.index.codecs.PerDocConsumer; > -import org.apache.lucene.index.codecs.PerDocValues; > import org.apache.lucene.store.CompoundFileDirectory; > import org.apache.lucene.store.Directory; > import org.apache.lucene.store.IOContext; > @@ -141,6 +140,8 @@ final class SegmentMerger { > if (fieldInfos.hasVectors()) { > mergeVectors(); > } > + // write FIS once merge is done. IDV might change types or drops fields > + fieldInfos.write(directory, segment + "." + > IndexFileNames.FIELD_INFOS_EXTENSION); > return mergedDocs; > } > > @@ -254,7 +255,6 @@ final class SegmentMerger { > } > } > final SegmentCodecs codecInfo = fieldInfos.buildSegmentCodecs(false); > - fieldInfos.write(directory, segment + "." + > IndexFileNames.FIELD_INFOS_EXTENSION); > > int docCount = 0; > > @@ -584,28 +584,11 @@ final class SegmentMerger { > } > > private void mergePerDoc() throws IOException { > - final List<PerDocValues> perDocProducers = new > ArrayList<PerDocValues>(); > - final List<ReaderUtil.Slice> perDocSlices = new > ArrayList<ReaderUtil.Slice>(); > - int docBase = 0; > - for (MergeState.IndexReaderAndLiveDocs r : readers) { > - final int maxDoc = r.reader.maxDoc(); > - final PerDocValues producer = r.reader.perDocValues(); > - if (producer != null) { > - perDocSlices.add(new ReaderUtil.Slice(docBase, maxDoc, > perDocProducers > - .size())); > - perDocProducers.add(producer); > - } > - docBase += maxDoc; > - } > - if (!perDocSlices.isEmpty()) { > final PerDocConsumer docsConsumer = codec > .docsConsumer(new PerDocWriteState(segmentWriteState)); > boolean success = false; > try { > - final MultiPerDocValues multiPerDocValues = new MultiPerDocValues( > - perDocProducers.toArray(PerDocValues.EMPTY_ARRAY), > - perDocSlices.toArray(ReaderUtil.Slice.EMPTY_ARRAY)); > - docsConsumer.merge(mergeState, multiPerDocValues); > + docsConsumer.merge(mergeState); > success = true; > } finally { > if (success) { > @@ -614,11 +597,8 @@ final class SegmentMerger { > IOUtils.closeWhileHandlingException(docsConsumer); > } > } > - } > - /* don't close the perDocProducers here since they are private segment > producers > - * and will be closed once the SegmentReader goes out of scope */ > } > - > + > private MergeState mergeState; > > public boolean getAnyNonBulkMerges() { > > Modified: > lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/DocValuesConsumer.java > URL: > http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/DocValuesConsumer.java?rev=1181020&r1=1181019&r2=1181020&view=diff > ============================================================================== > --- > lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/DocValuesConsumer.java > (original) > +++ > lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/DocValuesConsumer.java > Mon Oct 10 15:28:07 2011 > @@ -92,48 +92,38 @@ public abstract class DocValuesConsumer > * > * @param mergeState > * the state to merge > - * @param values > - * the docValues to merge in > + * @param docValues docValues array containing one instance per reader ( > + * {@link MergeState#readers}) or <code>null</code> if the reader > has > + * no {@link IndexDocValues} instance. > * @throws IOException > * if an {@link IOException} occurs > */ > - public void merge(org.apache.lucene.index.codecs.MergeState mergeState, > - IndexDocValues values) throws IOException { > + public void merge(MergeState mergeState, IndexDocValues[] docValues) > throws IOException { > assert mergeState != null; > - // TODO we need some kind of compatibility notation for values such > - // that two slightly different segments can be merged eg. fixed vs. > - // variable byte len or float32 vs. float64 > - boolean merged = false; > - /* > - * We ignore the given DocValues here and merge from the subReaders > directly > - * to support bulk copies on the DocValues Writer level. if this gets > merged > - * with MultiDocValues the writer can not optimize for bulk-copyable data > - */ > + boolean hasMerged = false; > for(int readerIDX=0;readerIDX<mergeState.readers.size();readerIDX++) { > final org.apache.lucene.index.codecs.MergeState.IndexReaderAndLiveDocs > reader = mergeState.readers.get(readerIDX); > - final IndexDocValues r = > reader.reader.docValues(mergeState.fieldInfo.name); > - if (r != null) { > - merged = true; > - merge(new Writer.MergeState(r, mergeState.docBase[readerIDX], > reader.reader.maxDoc(), > + if (docValues[readerIDX] != null) { > + hasMerged = true; > + merge(new Writer.SingleSubMergeState(docValues[readerIDX], > mergeState.docBase[readerIDX], reader.reader.maxDoc(), > reader.liveDocs)); > } > } > - if (merged) { > + // only finish if no exception is thrown! > + if (hasMerged) { > finish(mergeState.mergedDocCount); > } > } > > /** > - * Merges the given {@link MergeState} into this {@link DocValuesConsumer}. > - * {@link MergeState#docBase} must always be increasing. Merging segments > out > - * of order is not supported. > + * Merges the given {@link SingleSubMergeState} into this {@link > DocValuesConsumer}. > * > * @param mergeState > - * the {@link MergeState} to merge > + * the {@link SingleSubMergeState} to merge > * @throws IOException > * if an {@link IOException} occurs > */ > - protected abstract void merge(MergeState mergeState) throws IOException; > + protected abstract void merge(SingleSubMergeState mergeState) throws > IOException; > > /** > * Specialized auxiliary MergeState is necessary since we don't want to > @@ -141,7 +131,7 @@ public abstract class DocValuesConsumer > * created for each merged low level {@link IndexReader} we are merging to > * support low level bulk copies. > */ > - public static class MergeState { > + public static class SingleSubMergeState { > /** > * the source reader for this MergeState - merged values should be read > from > * this instance > @@ -154,7 +144,7 @@ public abstract class DocValuesConsumer > /** the not deleted bits for this MergeState */ > public final Bits liveDocs; > > - public MergeState(IndexDocValues reader, int docBase, int docCount, Bits > liveDocs) { > + public SingleSubMergeState(IndexDocValues reader, int docBase, int > docCount, Bits liveDocs) { > assert reader != null; > this.reader = reader; > this.docBase = docBase; > > Modified: > lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/PerDocConsumer.java > URL: > http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/PerDocConsumer.java?rev=1181020&r1=1181019&r2=1181020&view=diff > ============================================================================== > --- > lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/PerDocConsumer.java > (original) > +++ > lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/PerDocConsumer.java > Mon Oct 10 15:28:07 2011 > @@ -19,7 +19,10 @@ import java.io.Closeable; > import java.io.IOException; > > import org.apache.lucene.index.FieldInfo; > +import org.apache.lucene.index.FieldInfos; > import org.apache.lucene.index.values.IndexDocValues; > +import org.apache.lucene.index.values.TypePromoter; > +import org.apache.lucene.index.values.ValueType; > > /** > * Abstract API that consumes per document values. Concrete implementations of > @@ -40,28 +43,75 @@ public abstract class PerDocConsumer imp > * Consumes and merges the given {@link PerDocValues} producer > * into this consumers format. > */ > - public void merge(MergeState mergeState, PerDocValues producer) > + public void merge(MergeState mergeState) > throws IOException { > - Iterable<String> fields = producer.fields(); > - for (String field : fields) { > - mergeState.fieldInfo = mergeState.fieldInfos.fieldInfo(field); > - assert mergeState.fieldInfo != null : "FieldInfo for field is null: " > - + field; > - if (mergeState.fieldInfo.hasDocValues()) { > - final IndexDocValues docValues = producer.docValues(field); > - if (docValues == null) { > - /* > - * It is actually possible that a fieldInfo has a values type but > no > - * values are actually available. this can happen if there are > already > - * segments without values around. > - */ > + final FieldInfos fieldInfos = mergeState.fieldInfos; > + final IndexDocValues[] docValues = new > IndexDocValues[mergeState.readers.size()]; > + final PerDocValues[] perDocValues = new > PerDocValues[mergeState.readers.size()]; > + // pull all PerDocValues > + for (int i = 0; i < perDocValues.length; i++) { > + perDocValues[i] = mergeState.readers.get(i).reader.perDocValues(); > + } > + for (FieldInfo fieldInfo : fieldInfos) { > + mergeState.fieldInfo = fieldInfo; > + TypePromoter currentPromoter = TypePromoter.getIdentityPromoter(); > + if (fieldInfo.hasDocValues()) { > + for (int i = 0; i < perDocValues.length; i++) { > + if (perDocValues[i] != null) { // get all IDV to merge > + docValues[i] = perDocValues[i].docValues(fieldInfo.name); > + if (docValues[i] != null) { > + currentPromoter = promoteValueType(fieldInfo, docValues[i], > currentPromoter); > + if (currentPromoter == null) { > + break; > + } > + } > + } > + } > + > + if (currentPromoter == null) { > + fieldInfo.resetDocValues(null); > continue; > } > + assert currentPromoter != TypePromoter.getIdentityPromoter(); > + if (fieldInfo.getDocValues() != currentPromoter.type()) { > + // reset the type if we got promoted > + fieldInfo.resetDocValues(currentPromoter.type()); > + } > + > final DocValuesConsumer docValuesConsumer = > addValuesField(mergeState.fieldInfo); > assert docValuesConsumer != null; > docValuesConsumer.merge(mergeState, docValues); > } > } > + /* NOTE: don't close the perDocProducers here since they are private > segment producers > + * and will be closed once the SegmentReader goes out of scope */ > + } > > + protected TypePromoter promoteValueType(final FieldInfo fieldInfo, final > IndexDocValues docValues, > + TypePromoter currentPromoter) { > + assert currentPromoter != null; > + final TypePromoter incomingPromoter = > TypePromoter.create(docValues.type(), docValues.getValueSize()); > + assert incomingPromoter != null; > + final TypePromoter newPromoter = > currentPromoter.promote(incomingPromoter); > + return newPromoter == null ? handleIncompatibleValueType(fieldInfo, > incomingPromoter, currentPromoter) : newPromoter; > + } > + > + /** > + * Resolves a conflicts of incompatible {@link TypePromoter}s. The default > + * implementation promotes incompatible types to > + * {@link ValueType#BYTES_VAR_STRAIGHT} and preserves all values. If this > + * method returns <code>null</code> all docvalues for the given > + * {@link FieldInfo} are dropped and all values are lost. > + * > + * @param incomingPromoter > + * the incompatible incoming promoter > + * @param currentPromoter > + * the current promoter > + * @return a promoted {@link TypePromoter} or <code>null</code> iff this > index > + * docvalues should be dropped for this field. > + */ > + protected TypePromoter handleIncompatibleValueType(FieldInfo fieldInfo, > TypePromoter incomingPromoter, TypePromoter currentPromoter) { > + return TypePromoter.create(ValueType.BYTES_VAR_STRAIGHT, > TypePromoter.VAR_TYPE_VALUE_SIZE); > } > + > } > > Modified: > lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/FixedDerefBytesImpl.java > URL: > http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/FixedDerefBytesImpl.java?rev=1181020&r1=1181019&r2=1181020&view=diff > ============================================================================== > --- > lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/FixedDerefBytesImpl.java > (original) > +++ > lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/FixedDerefBytesImpl.java > Mon Oct 10 15:28:07 2011 > @@ -86,6 +86,12 @@ class FixedDerefBytesImpl { > throws IOException { > return new DirectFixedDerefSource(cloneData(), cloneIndex(), size, > type()); > } > + > + @Override > + public int getValueSize() { > + return size; > + } > + > } > > static final class FixedDerefSource extends BytesSourceBase { > > Modified: > lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/FixedSortedBytesImpl.java > URL: > http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/FixedSortedBytesImpl.java?rev=1181020&r1=1181019&r2=1181020&view=diff > ============================================================================== > --- > lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/FixedSortedBytesImpl.java > (original) > +++ > lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/FixedSortedBytesImpl.java > Mon Oct 10 15:28:07 2011 > @@ -104,6 +104,11 @@ class FixedSortedBytesImpl { > return new DirectFixedSortedSource(cloneData(), cloneIndex(), size, > valueCount, comparator, type); > } > + > + @Override > + public int getValueSize() { > + return size; > + } > } > > static final class FixedSortedSource extends BytesSortedSourceBase { > > Modified: > lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/FixedStraightBytesImpl.java > URL: > http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/FixedStraightBytesImpl.java?rev=1181020&r1=1181019&r2=1181020&view=diff > ============================================================================== > --- > lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/FixedStraightBytesImpl.java > (original) > +++ > lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/FixedStraightBytesImpl.java > Mon Oct 10 15:28:07 2011 > @@ -59,6 +59,7 @@ class FixedStraightBytesImpl { > int version, Counter bytesUsed, IOContext context) throws IOException > { > super(dir, id, codecName, version, bytesUsed, context); > pool = new ByteBlockPool(new DirectTrackingAllocator(bytesUsed)); > + pool.nextBuffer(); > } > > @Override > @@ -70,7 +71,6 @@ class FixedStraightBytesImpl { > throw new IllegalArgumentException("bytes arrays > " + > Short.MAX_VALUE + " are not supported"); > } > size = bytes.length; > - pool.nextBuffer(); > } else if (bytes.length != size) { > throw new IllegalArgumentException("expected bytes size=" + size > + " but got " + bytes.length); > @@ -120,7 +120,7 @@ class FixedStraightBytesImpl { > } > > static class Writer extends FixedBytesWriterBase { > - private boolean merge; > + private boolean hasMerged; > private IndexOutput datOut; > > public Writer(Directory dir, String id, Counter bytesUsed, IOContext > context) throws IOException { > @@ -133,12 +133,15 @@ class FixedStraightBytesImpl { > > > @Override > - protected void merge(MergeState state) throws IOException { > - merge = true; > + protected void merge(SingleSubMergeState state) throws IOException { > datOut = getOrCreateDataOut(); > boolean success = false; > try { > - if (state.liveDocs == null && state.reader instanceof > FixedStraightReader ) { > + if (!hasMerged && size != -1) { > + datOut.writeInt(size); > + } > + > + if (state.liveDocs == null && tryBulkMerge(state.reader)) { > FixedStraightReader reader = (FixedStraightReader) state.reader; > final int maxDocs = reader.maxDoc; > if (maxDocs == 0) { > @@ -172,24 +175,33 @@ class FixedStraightBytesImpl { > if (!success) { > IOUtils.closeWhileHandlingException(datOut); > } > + hasMerged = true; > } > } > > + protected boolean tryBulkMerge(IndexDocValues docValues) { > + return docValues instanceof FixedStraightReader; > + } > + > @Override > protected void mergeDoc(int docID, int sourceDoc) throws IOException { > assert lastDocID < docID; > - currentMergeSource.getBytes(sourceDoc, bytesRef); > + setMergeBytes(sourceDoc); > if (size == -1) { > size = bytesRef.length; > datOut.writeInt(size); > } > - assert size == bytesRef.length; > + assert size == bytesRef.length : "size: " + size + " ref: " + > bytesRef.length; > if (lastDocID+1 < docID) { > fill(datOut, docID); > } > datOut.writeBytes(bytesRef.bytes, bytesRef.offset, bytesRef.length); > lastDocID = docID; > } > + > + protected void setMergeBytes(int sourceDoc) { > + currentMergeSource.getBytes(sourceDoc, bytesRef); > + } > > > > @@ -203,7 +215,7 @@ class FixedStraightBytesImpl { > public void finish(int docCount) throws IOException { > boolean success = false; > try { > - if (!merge) { > + if (!hasMerged) { > // indexing path - no disk IO until here > assert datOut == null; > datOut = getOrCreateDataOut(); > @@ -267,6 +279,11 @@ class FixedStraightBytesImpl { > public Source getDirectSource() throws IOException { > return new DirectFixedStraightSource(cloneData(), size, type()); > } > + > + @Override > + public int getValueSize() { > + return size; > + } > } > > // specialized version for single bytes > > Modified: > lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/Floats.java > URL: > http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/Floats.java?rev=1181020&r1=1181019&r2=1181020&view=diff > ============================================================================== > --- > lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/Floats.java > (original) > +++ > lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/Floats.java > Mon Oct 10 15:28:07 2011 > @@ -85,6 +85,18 @@ public class Floats { > public void add(int docID, PerDocFieldValues docValues) throws > IOException { > add(docID, docValues.getFloat()); > } > + > + @Override > + protected boolean tryBulkMerge(IndexDocValues docValues) { > + // only bulk merge if value type is the same otherwise size differs > + return super.tryBulkMerge(docValues) && docValues.type() == > template.type(); > + } > + > + @Override > + protected void setMergeBytes(int sourceDoc) { > + final double value = currentMergeSource.getFloat(sourceDoc); > + template.toBytes(value, bytesRef); > + } > } > > final static class FloatsReader extends > FixedStraightBytesImpl.FixedStraightReader { > > Modified: > lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/IndexDocValues.java > URL: > http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/IndexDocValues.java?rev=1181020&r1=1181019&r2=1181020&view=diff > ============================================================================== > --- > lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/IndexDocValues.java > (original) > +++ > lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/IndexDocValues.java > Mon Oct 10 15:28:07 2011 > @@ -107,6 +107,17 @@ public abstract class IndexDocValues imp > } > > /** > + * Returns the size per value in bytes or <code>-1</code> iff size per > value > + * is variable. > + * > + * @return the size per value in bytes or <code>-1</code> iff size per > value > + * is variable. > + */ > + public int getValueSize() { > + return -1; > + } > + > + /** > * Sets the {@link SourceCache} used by this {@link IndexDocValues} > instance. This > * method should be called before {@link #load()} is called. All {@link > Source} instances in the currently used cache will be closed > * before the new cache is installed. > > Modified: > lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/Ints.java > URL: > http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/Ints.java?rev=1181020&r1=1181019&r2=1181020&view=diff > ============================================================================== > --- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/Ints.java > (original) > +++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/Ints.java > Mon Oct 10 15:28:07 2011 > @@ -93,9 +93,9 @@ public final class Ints { > protected IntsWriter(Directory dir, String id, String codecName, > int version, Counter bytesUsed, IOContext context, ValueType > valueType) throws IOException { > super(dir, id, codecName, version, bytesUsed, context); > - final int expectedSize = typeToSize(valueType); > - this.bytesRef = new BytesRef(expectedSize); > - bytesRef.length = expectedSize; > + size = typeToSize(valueType); > + this.bytesRef = new BytesRef(size); > + bytesRef.length = size; > template = IndexDocValuesArray.TEMPLATES.get(valueType); > } > > @@ -109,6 +109,18 @@ public final class Ints { > public void add(int docID, PerDocFieldValues docValues) throws > IOException { > add(docID, docValues.getInt()); > } > + > + @Override > + protected void setMergeBytes(int sourceDoc) { > + final long value = currentMergeSource.getInt(sourceDoc); > + template.toBytes(value, bytesRef); > + } > + > + @Override > + protected boolean tryBulkMerge(IndexDocValues docValues) { > + // only bulk merge if value type is the same otherwise size differs > + return super.tryBulkMerge(docValues) && docValues.type() == > template.type(); > + } > } > > final static class IntsReader extends > FixedStraightBytesImpl.FixedStraightReader { > > Modified: > lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/MultiIndexDocValues.java > URL: > http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/MultiIndexDocValues.java?rev=1181020&r1=1181019&r2=1181020&view=diff > ============================================================================== > --- > lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/MultiIndexDocValues.java > (original) > +++ > lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/MultiIndexDocValues.java > Mon Oct 10 15:28:07 2011 > @@ -46,6 +46,8 @@ public class MultiIndexDocValues extends > > private DocValuesIndex[] docValuesIdx; > private int[] starts; > + private ValueType type; > + private int valueSize; > > public MultiIndexDocValues() { > starts = new int[0]; > @@ -62,38 +64,51 @@ public class MultiIndexDocValues extends > } > > public IndexDocValues reset(DocValuesIndex[] docValuesIdx) { > - int[] start = new int[docValuesIdx.length]; > + final int[] start = new int[docValuesIdx.length]; > + TypePromoter promoter = TypePromoter.getIdentityPromoter(); > for (int i = 0; i < docValuesIdx.length; i++) { > start[i] = docValuesIdx[i].start; > + if (!(docValuesIdx[i].docValues instanceof EmptyDocValues)) { > + // only promote if not a dummy > + final TypePromoter incomingPromoter = TypePromoter.create( > + docValuesIdx[i].docValues.type(), > + docValuesIdx[i].docValues.getValueSize()); > + promoter = promoter.promote(incomingPromoter); > + if (promoter == null) { > + throw new IllegalStateException("Can not promote " + > incomingPromoter); > + } > + } > } > + this.type = promoter.type(); > + this.valueSize = promoter.getValueSize(); > this.starts = start; > this.docValuesIdx = docValuesIdx; > return this; > } > > - public static class DummyDocValues extends IndexDocValues { > + public static class EmptyDocValues extends IndexDocValues { > final int maxDoc; > - final Source emptySoruce; > + final Source emptySource; > > - public DummyDocValues(int maxDoc, ValueType type) { > + public EmptyDocValues(int maxDoc, ValueType type) { > this.maxDoc = maxDoc; > - this.emptySoruce = new EmptySource(type); > + this.emptySource = new EmptySource(type); > } > > @Override > public Source load() throws IOException { > - return emptySoruce; > + return emptySource; > } > > @Override > public ValueType type() { > - return emptySoruce.type(); > + return emptySource.type(); > } > > > @Override > public Source getDirectSource() throws IOException { > - return emptySoruce; > + return emptySource; > } > } > > @@ -180,7 +195,12 @@ public class MultiIndexDocValues extends > > @Override > public ValueType type() { > - return this.docValuesIdx[0].docValues.type(); > + return type; > + } > + > + @Override > + public int getValueSize() { > + return valueSize; > } > > @Override > > Modified: > lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/VarStraightBytesImpl.java > URL: > http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/VarStraightBytesImpl.java?rev=1181020&r1=1181019&r2=1181020&view=diff > ============================================================================== > --- > lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/VarStraightBytesImpl.java > (original) > +++ > lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/VarStraightBytesImpl.java > Mon Oct 10 15:28:07 2011 > @@ -93,7 +93,7 @@ class VarStraightBytesImpl { > } > > @Override > - protected void merge(MergeState state) throws IOException { > + protected void merge(SingleSubMergeState state) throws IOException { > merge = true; > datOut = getOrCreateDataOut(); > boolean success = false; > > Modified: > lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/Writer.java > URL: > http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/Writer.java?rev=1181020&r1=1181019&r2=1181020&view=diff > ============================================================================== > --- > lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/Writer.java > (original) > +++ > lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/Writer.java > Mon Oct 10 15:28:07 2011 > @@ -138,7 +138,7 @@ public abstract class Writer extends Doc > public abstract void finish(int docCount) throws IOException; > > @Override > - protected void merge(MergeState state) throws IOException { > + protected void merge(SingleSubMergeState state) throws IOException { > // This enables bulk copies in subclasses per MergeState, subclasses can > // simply override this and decide if they want to merge > // segments using this generic implementation or if a bulk merge is > possible > > - Mark Miller lucidimagination.com 2011.lucene-eurocon.org | Oct 17-20 | Barcelona --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
