I had to fix a couple things for addBackcompatIndexes.py to work properly. I pushed directly because it would have been a bit cumbersome to run this script without pushing these changes first, but I'd still appreciate a review if anyone is up for it.
On Tue, Feb 20, 2024 at 10:14 PM <jpou...@apache.org> wrote: > > This is an automated email from the ASF dual-hosted git repository. > > jpountz pushed a commit to branch main > in repository https://gitbox.apache.org/repos/asf/lucene.git > > > The following commit(s) were added to refs/heads/main by this push: > new 13d561af1d6 Fix bw index generation logic. > 13d561af1d6 is described below > > commit 13d561af1d624f35f8a27a05490062ac2472e786 > Author: Adrien Grand <jpou...@gmail.com> > AuthorDate: Tue Feb 20 22:10:01 2024 +0100 > > Fix bw index generation logic. > --- > dev-tools/scripts/addBackcompatIndexes.py | 13 +++--------- > .../BackwardsCompatibilityTestBase.java | 23 > +++++++++++++++++++--- > .../backward_index/TestGenerateBwcIndices.java | 2 ++ > 3 files changed, 25 insertions(+), 13 deletions(-) > > diff --git a/dev-tools/scripts/addBackcompatIndexes.py > b/dev-tools/scripts/addBackcompatIndexes.py > index bbaf0b40630..7faacb8b8e3 100755 > --- a/dev-tools/scripts/addBackcompatIndexes.py > +++ b/dev-tools/scripts/addBackcompatIndexes.py > @@ -45,16 +45,13 @@ def create_and_add_index(source, indextype, > index_version, current_version, temp > 'emptyIndex': 'empty' > }[indextype] > if indextype in ('cfs', 'nocfs'): > - dirname = 'index.%s' % indextype > filename = '%s.%s-%s.zip' % (prefix, index_version, indextype) > else: > - dirname = indextype > filename = '%s.%s.zip' % (prefix, index_version) > > print(' creating %s...' % filename, end='', flush=True) > module = 'backward-codecs' > index_dir = os.path.join('lucene', module, > 'src/test/org/apache/lucene/backward_index') > - test_file = os.path.join(index_dir, filename) > if os.path.exists(os.path.join(index_dir, filename)): > print('uptodate') > return > @@ -76,24 +73,20 @@ def create_and_add_index(source, indextype, > index_version, current_version, temp > '-Dtests.codec=default' > ]) > base_dir = os.getcwd() > - bc_index_dir = os.path.join(temp_dir, dirname) > - bc_index_file = os.path.join(bc_index_dir, filename) > + bc_index_file = os.path.join(temp_dir, filename) > > if os.path.exists(bc_index_file): > print('alreadyexists') > else: > - if os.path.exists(bc_index_dir): > - shutil.rmtree(bc_index_dir) > os.chdir(source) > scriptutil.run('./gradlew %s' % gradle_args) > - os.chdir(bc_index_dir) > - scriptutil.run('zip %s *' % filename) > + if not os.path.exists(bc_index_file): > + raise Exception("Expected file can't be found: %s" %bc_index_file) > print('done') > > print(' adding %s...' % filename, end='', flush=True) > scriptutil.run('cp %s %s' % (bc_index_file, os.path.join(base_dir, > index_dir))) > os.chdir(base_dir) > - scriptutil.run('rm -rf %s' % bc_index_dir) > print('done') > > def update_backcompat_tests(index_version, current_version): > diff --git > a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/BackwardsCompatibilityTestBase.java > > b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/BackwardsCompatibilityTestBase.java > index 8df28d40dbc..b131bb9497b 100644 > --- > a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/BackwardsCompatibilityTestBase.java > +++ > b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/BackwardsCompatibilityTestBase.java > @@ -17,6 +17,7 @@ > package org.apache.lucene.backward_index; > > import com.carrotsearch.randomizedtesting.annotations.Name; > +import java.io.FileOutputStream; > import java.io.IOException; > import java.io.InputStream; > import java.io.LineNumberReader; > @@ -38,11 +39,17 @@ import java.util.function.Predicate; > import java.util.regex.Matcher; > import java.util.regex.Pattern; > import java.util.stream.Collectors; > +import java.util.zip.ZipEntry; > +import java.util.zip.ZipOutputStream; > import org.apache.lucene.codecs.Codec; > import org.apache.lucene.index.DirectoryReader; > import org.apache.lucene.index.LeafReaderContext; > import org.apache.lucene.index.SegmentReader; > import org.apache.lucene.store.Directory; > +import org.apache.lucene.store.FSDirectory; > +import org.apache.lucene.store.IOContext; > +import org.apache.lucene.store.IndexInput; > +import org.apache.lucene.store.OutputStreamDataOutput; > import org.apache.lucene.tests.util.LuceneTestCase; > import org.apache.lucene.tests.util.TestUtil; > import org.apache.lucene.util.BytesRef; > @@ -253,10 +260,20 @@ public abstract class BackwardsCompatibilityTestBase > extends LuceneTestCase { > protected abstract void createIndex(Directory directory) throws > IOException; > > public final void createBWCIndex() throws IOException { > - Path indexDir = getIndexDir().resolve(indexName(Version.LATEST)); > - Files.deleteIfExists(indexDir); > - try (Directory dir = newFSDirectory(indexDir)) { > + Path zipFile = getIndexDir().resolve(indexName(Version.LATEST)); > + Files.deleteIfExists(zipFile); > + Path tmpDir = createTempDir(); > + > + try (Directory dir = FSDirectory.open(tmpDir); > + ZipOutputStream zipOut = new ZipOutputStream(new > FileOutputStream(zipFile.toFile()))) { > createIndex(dir); > + for (String file : dir.listAll()) { > + try (IndexInput in = dir.openInput(file, IOContext.READONCE)) { > + zipOut.putNextEntry(new ZipEntry(file)); > + new OutputStreamDataOutput(zipOut).copyBytes(in, in.length()); > + zipOut.closeEntry(); > + } > + } > } > } > > diff --git > a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestGenerateBwcIndices.java > > b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestGenerateBwcIndices.java > index 0cd9f37d5c3..c7b1ea3fb4a 100644 > --- > a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestGenerateBwcIndices.java > +++ > b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestGenerateBwcIndices.java > @@ -20,8 +20,10 @@ import static > org.apache.lucene.backward_index.BackwardsCompatibilityTestBase.cr > > import java.io.IOException; > import org.apache.lucene.tests.util.LuceneTestCase; > +import org.apache.lucene.tests.util.LuceneTestCase.SuppressFileSystems; > import org.apache.lucene.util.Version; > > +@SuppressFileSystems("ExtrasFS") > public class TestGenerateBwcIndices extends LuceneTestCase { > > // Backcompat index generation, described below, is mostly automated in: > -- Adrien --------------------------------------------------------------------- To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org For additional commands, e-mail: dev-h...@lucene.apache.org