uschindler commented on a change in pull request #2052: URL: https://github.com/apache/lucene-solr/pull/2052#discussion_r546454653
########## File path: lucene/misc/src/java/org/apache/lucene/misc/store/DirectIODirectory.java ########## @@ -0,0 +1,392 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.misc.store; + +import java.io.EOFException; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.OpenOption; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Arrays; + +import org.apache.lucene.store.*; +import org.apache.lucene.store.IOContext.Context; + +/** + * A {@link Directory} implementation for all Unixes and Windows that uses + * DIRECT I/O to bypass OS level IO caching during + * merging. For all other cases (searching, writing) we delegate + * to the provided Directory instance. + * + * <p>See <a + * href="{@docRoot}/overview-summary.html#DirectIODirectory">Overview</a> + * for more details. + * + * <p><b>WARNING</b>: this code is very new and quite easily + * could contain horrible bugs. + * + * <p>This directory passes Solr and Lucene tests on Linux, OS X, + * and Windows; other systems should work but have not been + * tested! Use at your own risk. + * + * <p>@throws UnsupportedOperationException if the operating system, file system or JDK + * does not support Direct I/O or a sufficient equivalent. + * + * @lucene.experimental + */ +public class DirectIODirectory extends FilterDirectory { + + /** Default buffer size before writing to disk (256 KB); + * larger means less IO load but more RAM and direct + * buffer storage space consumed during merging. */ + + public final static int DEFAULT_MERGE_BUFFER_SIZE = 262144; + + /** Default min expected merge size before direct IO is + * used (10 MB): */ + public final static long DEFAULT_MIN_BYTES_DIRECT = 10*1024*1024; + + private final int blockSize, mergeBufferSize; + private final long minBytesDirect; + private final Path path; + + volatile boolean isOpen = true; + + /** Reference to {@code com.sun.nio.file.ExtendedOpenOption.DIRECT} by reflective class and enum lookup. + * There are two reasons for using this instead of directly referencing ExtendedOpenOption.DIRECT: + * <ol> + * <li> ExtendedOpenOption.DIRECT is OpenJDK's internal proprietary API. This API causes un-suppressible(?) warning to be emitted + * when compiling with --release flag and value N, where N is smaller than the the version of javac used for compilation.</li> + * <li> It is possible that Lucene is run using JDK that does not support ExtendedOpenOption.DIRECT. In such a + * case, dynamic lookup allows us to bail out with UnsupportedOperationException with meaningful error message.</li> + * </ol> + * <p>This reference is {@code null}, if the JDK does not support direct I/O. + */ + static final OpenOption ExtendedOpenOption_DIRECT; // visible for test + static { + OpenOption option; + try { + final Class<? extends OpenOption> clazz = Class.forName("com.sun.nio.file.ExtendedOpenOption").asSubclass(OpenOption.class); + option = Arrays.stream(clazz.getEnumConstants()) + .filter(e -> e.toString().equalsIgnoreCase("DIRECT")) + .findFirst() + .orElse(null); + } catch (Exception e) { + option = null; + } + ExtendedOpenOption_DIRECT = option; + } + + /** Create a new DirectIODirectory for the named location. + * + * @param path the path of the directory + * @param mergeBufferSize Size of buffer to use for + * merging. + * @param minBytesDirect Merges, or files to be opened for + * reading, smaller than this will + * not use direct IO. See {@link + * #DEFAULT_MIN_BYTES_DIRECT} + * @param delegate fallback Directory for non-merges + * @throws IOException If there is a low-level I/O error + */ + public DirectIODirectory(Path path, int mergeBufferSize, long minBytesDirect, FSDirectory delegate) throws IOException { + super(delegate); + this.blockSize = Math.toIntExact(Files.getFileStore(path).getBlockSize()); + this.mergeBufferSize = mergeBufferSize; + this.minBytesDirect = minBytesDirect; + this.path = path.toRealPath(); + } + + /** Create a new DirectIODirectory for the named location. + * + * @param path the path of the directory + * @param delegate fallback Directory for non-merges + * @throws IOException If there is a low-level I/O error + */ + public DirectIODirectory(Path path, FSDirectory delegate) throws IOException { Review comment: I fixed this in https://github.com/apache/lucene-solr/pull/2052/commits/9a59b56f2f94aed53d2268c3053e0fa37382ce44 ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org --------------------------------------------------------------------- To unsubscribe, e-mail: issues-unsubscr...@lucene.apache.org For additional commands, e-mail: issues-h...@lucene.apache.org
