I had to fix a couple things for addBackcompatIndexes.py to work
properly. I pushed directly because it would have been a bit
cumbersome to run this script without pushing these changes first, but
I'd still appreciate a review if anyone is up for it.
On Tue, Feb 20, 2024 at 10:14?PM <jpountz@apache.org> wrote:
>
> This is an automated email from the ASF dual-hosted git repository.
>
> jpountz pushed a commit to branch main
> in repository https://gitbox.apache.org/repos/asf/lucene.git
>
>
> The following commit(s) were added to refs/heads/main by this push:
> new 13d561af1d6 Fix bw index generation logic.
> 13d561af1d6 is described below
>
> commit 13d561af1d624f35f8a27a05490062ac2472e786
> Author: Adrien Grand <jpountz@gmail.com>
> AuthorDate: Tue Feb 20 22:10:01 2024 +0100
>
> Fix bw index generation logic.
> ---
> dev-tools/scripts/addBackcompatIndexes.py | 13 +++---------
> .../BackwardsCompatibilityTestBase.java | 23 +++++++++++++++++++---
> .../backward_index/TestGenerateBwcIndices.java | 2 ++
> 3 files changed, 25 insertions(+), 13 deletions(-)
>
> diff --git a/dev-tools/scripts/addBackcompatIndexes.py b/dev-tools/scripts/addBackcompatIndexes.py
> index bbaf0b40630..7faacb8b8e3 100755
> --- a/dev-tools/scripts/addBackcompatIndexes.py
> +++ b/dev-tools/scripts/addBackcompatIndexes.py
> @@ -45,16 +45,13 @@ def create_and_add_index(source, indextype, index_version, current_version, temp
> 'emptyIndex': 'empty'
> }[indextype]
> if indextype in ('cfs', 'nocfs'):
> - dirname = 'index.%s' % indextype
> filename = '%s.%s-%s.zip' % (prefix, index_version, indextype)
> else:
> - dirname = indextype
> filename = '%s.%s.zip' % (prefix, index_version)
>
> print(' creating %s...' % filename, end='', flush=True)
> module = 'backward-codecs'
> index_dir = os.path.join('lucene', module, 'src/test/org/apache/lucene/backward_index')
> - test_file = os.path.join(index_dir, filename)
> if os.path.exists(os.path.join(index_dir, filename)):
> print('uptodate')
> return
> @@ -76,24 +73,20 @@ def create_and_add_index(source, indextype, index_version, current_version, temp
> '-Dtests.codec=default'
> ])
> base_dir = os.getcwd()
> - bc_index_dir = os.path.join(temp_dir, dirname)
> - bc_index_file = os.path.join(bc_index_dir, filename)
> + bc_index_file = os.path.join(temp_dir, filename)
>
> if os.path.exists(bc_index_file):
> print('alreadyexists')
> else:
> - if os.path.exists(bc_index_dir):
> - shutil.rmtree(bc_index_dir)
> os.chdir(source)
> scriptutil.run('./gradlew %s' % gradle_args)
> - os.chdir(bc_index_dir)
> - scriptutil.run('zip %s *' % filename)
> + if not os.path.exists(bc_index_file):
> + raise Exception("Expected file can't be found: %s" %bc_index_file)
> print('done')
>
> print(' adding %s...' % filename, end='', flush=True)
> scriptutil.run('cp %s %s' % (bc_index_file, os.path.join(base_dir, index_dir)))
> os.chdir(base_dir)
> - scriptutil.run('rm -rf %s' % bc_index_dir)
> print('done')
>
> def update_backcompat_tests(index_version, current_version):
> diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/BackwardsCompatibilityTestBase.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/BackwardsCompatibilityTestBase.java
> index 8df28d40dbc..b131bb9497b 100644
> --- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/BackwardsCompatibilityTestBase.java
> +++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/BackwardsCompatibilityTestBase.java
> @@ -17,6 +17,7 @@
> package org.apache.lucene.backward_index;
>
> import com.carrotsearch.randomizedtesting.annotations.Name;
> +import java.io.FileOutputStream;
> import java.io.IOException;
> import java.io.InputStream;
> import java.io.LineNumberReader;
> @@ -38,11 +39,17 @@ import java.util.function.Predicate;
> import java.util.regex.Matcher;
> import java.util.regex.Pattern;
> import java.util.stream.Collectors;
> +import java.util.zip.ZipEntry;
> +import java.util.zip.ZipOutputStream;
> import org.apache.lucene.codecs.Codec;
> import org.apache.lucene.index.DirectoryReader;
> import org.apache.lucene.index.LeafReaderContext;
> import org.apache.lucene.index.SegmentReader;
> import org.apache.lucene.store.Directory;
> +import org.apache.lucene.store.FSDirectory;
> +import org.apache.lucene.store.IOContext;
> +import org.apache.lucene.store.IndexInput;
> +import org.apache.lucene.store.OutputStreamDataOutput;
> import org.apache.lucene.tests.util.LuceneTestCase;
> import org.apache.lucene.tests.util.TestUtil;
> import org.apache.lucene.util.BytesRef;
> @@ -253,10 +260,20 @@ public abstract class BackwardsCompatibilityTestBase extends LuceneTestCase {
> protected abstract void createIndex(Directory directory) throws IOException;
>
> public final void createBWCIndex() throws IOException {
> - Path indexDir = getIndexDir().resolve(indexName(Version.LATEST));
> - Files.deleteIfExists(indexDir);
> - try (Directory dir = newFSDirectory(indexDir)) {
> + Path zipFile = getIndexDir().resolve(indexName(Version.LATEST));
> + Files.deleteIfExists(zipFile);
> + Path tmpDir = createTempDir();
> +
> + try (Directory dir = FSDirectory.open(tmpDir);
> + ZipOutputStream zipOut = new ZipOutputStream(new FileOutputStream(zipFile.toFile()))) {
> createIndex(dir);
> + for (String file : dir.listAll()) {
> + try (IndexInput in = dir.openInput(file, IOContext.READONCE)) {
> + zipOut.putNextEntry(new ZipEntry(file));
> + new OutputStreamDataOutput(zipOut).copyBytes(in, in.length());
> + zipOut.closeEntry();
> + }
> + }
> }
> }
>
> diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestGenerateBwcIndices.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestGenerateBwcIndices.java
> index 0cd9f37d5c3..c7b1ea3fb4a 100644
> --- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestGenerateBwcIndices.java
> +++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestGenerateBwcIndices.java
> @@ -20,8 +20,10 @@ import static org.apache.lucene.backward_index.BackwardsCompatibilityTestBase.cr
>
> import java.io.IOException;
> import org.apache.lucene.tests.util.LuceneTestCase;
> +import org.apache.lucene.tests.util.LuceneTestCase.SuppressFileSystems;
> import org.apache.lucene.util.Version;
>
> +@SuppressFileSystems("ExtrasFS")
> public class TestGenerateBwcIndices extends LuceneTestCase {
>
> // Backcompat index generation, described below, is mostly automated in:
>
--
Adrien
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org
properly. I pushed directly because it would have been a bit
cumbersome to run this script without pushing these changes first, but
I'd still appreciate a review if anyone is up for it.
On Tue, Feb 20, 2024 at 10:14?PM <jpountz@apache.org> wrote:
>
> This is an automated email from the ASF dual-hosted git repository.
>
> jpountz pushed a commit to branch main
> in repository https://gitbox.apache.org/repos/asf/lucene.git
>
>
> The following commit(s) were added to refs/heads/main by this push:
> new 13d561af1d6 Fix bw index generation logic.
> 13d561af1d6 is described below
>
> commit 13d561af1d624f35f8a27a05490062ac2472e786
> Author: Adrien Grand <jpountz@gmail.com>
> AuthorDate: Tue Feb 20 22:10:01 2024 +0100
>
> Fix bw index generation logic.
> ---
> dev-tools/scripts/addBackcompatIndexes.py | 13 +++---------
> .../BackwardsCompatibilityTestBase.java | 23 +++++++++++++++++++---
> .../backward_index/TestGenerateBwcIndices.java | 2 ++
> 3 files changed, 25 insertions(+), 13 deletions(-)
>
> diff --git a/dev-tools/scripts/addBackcompatIndexes.py b/dev-tools/scripts/addBackcompatIndexes.py
> index bbaf0b40630..7faacb8b8e3 100755
> --- a/dev-tools/scripts/addBackcompatIndexes.py
> +++ b/dev-tools/scripts/addBackcompatIndexes.py
> @@ -45,16 +45,13 @@ def create_and_add_index(source, indextype, index_version, current_version, temp
> 'emptyIndex': 'empty'
> }[indextype]
> if indextype in ('cfs', 'nocfs'):
> - dirname = 'index.%s' % indextype
> filename = '%s.%s-%s.zip' % (prefix, index_version, indextype)
> else:
> - dirname = indextype
> filename = '%s.%s.zip' % (prefix, index_version)
>
> print(' creating %s...' % filename, end='', flush=True)
> module = 'backward-codecs'
> index_dir = os.path.join('lucene', module, 'src/test/org/apache/lucene/backward_index')
> - test_file = os.path.join(index_dir, filename)
> if os.path.exists(os.path.join(index_dir, filename)):
> print('uptodate')
> return
> @@ -76,24 +73,20 @@ def create_and_add_index(source, indextype, index_version, current_version, temp
> '-Dtests.codec=default'
> ])
> base_dir = os.getcwd()
> - bc_index_dir = os.path.join(temp_dir, dirname)
> - bc_index_file = os.path.join(bc_index_dir, filename)
> + bc_index_file = os.path.join(temp_dir, filename)
>
> if os.path.exists(bc_index_file):
> print('alreadyexists')
> else:
> - if os.path.exists(bc_index_dir):
> - shutil.rmtree(bc_index_dir)
> os.chdir(source)
> scriptutil.run('./gradlew %s' % gradle_args)
> - os.chdir(bc_index_dir)
> - scriptutil.run('zip %s *' % filename)
> + if not os.path.exists(bc_index_file):
> + raise Exception("Expected file can't be found: %s" %bc_index_file)
> print('done')
>
> print(' adding %s...' % filename, end='', flush=True)
> scriptutil.run('cp %s %s' % (bc_index_file, os.path.join(base_dir, index_dir)))
> os.chdir(base_dir)
> - scriptutil.run('rm -rf %s' % bc_index_dir)
> print('done')
>
> def update_backcompat_tests(index_version, current_version):
> diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/BackwardsCompatibilityTestBase.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/BackwardsCompatibilityTestBase.java
> index 8df28d40dbc..b131bb9497b 100644
> --- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/BackwardsCompatibilityTestBase.java
> +++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/BackwardsCompatibilityTestBase.java
> @@ -17,6 +17,7 @@
> package org.apache.lucene.backward_index;
>
> import com.carrotsearch.randomizedtesting.annotations.Name;
> +import java.io.FileOutputStream;
> import java.io.IOException;
> import java.io.InputStream;
> import java.io.LineNumberReader;
> @@ -38,11 +39,17 @@ import java.util.function.Predicate;
> import java.util.regex.Matcher;
> import java.util.regex.Pattern;
> import java.util.stream.Collectors;
> +import java.util.zip.ZipEntry;
> +import java.util.zip.ZipOutputStream;
> import org.apache.lucene.codecs.Codec;
> import org.apache.lucene.index.DirectoryReader;
> import org.apache.lucene.index.LeafReaderContext;
> import org.apache.lucene.index.SegmentReader;
> import org.apache.lucene.store.Directory;
> +import org.apache.lucene.store.FSDirectory;
> +import org.apache.lucene.store.IOContext;
> +import org.apache.lucene.store.IndexInput;
> +import org.apache.lucene.store.OutputStreamDataOutput;
> import org.apache.lucene.tests.util.LuceneTestCase;
> import org.apache.lucene.tests.util.TestUtil;
> import org.apache.lucene.util.BytesRef;
> @@ -253,10 +260,20 @@ public abstract class BackwardsCompatibilityTestBase extends LuceneTestCase {
> protected abstract void createIndex(Directory directory) throws IOException;
>
> public final void createBWCIndex() throws IOException {
> - Path indexDir = getIndexDir().resolve(indexName(Version.LATEST));
> - Files.deleteIfExists(indexDir);
> - try (Directory dir = newFSDirectory(indexDir)) {
> + Path zipFile = getIndexDir().resolve(indexName(Version.LATEST));
> + Files.deleteIfExists(zipFile);
> + Path tmpDir = createTempDir();
> +
> + try (Directory dir = FSDirectory.open(tmpDir);
> + ZipOutputStream zipOut = new ZipOutputStream(new FileOutputStream(zipFile.toFile()))) {
> createIndex(dir);
> + for (String file : dir.listAll()) {
> + try (IndexInput in = dir.openInput(file, IOContext.READONCE)) {
> + zipOut.putNextEntry(new ZipEntry(file));
> + new OutputStreamDataOutput(zipOut).copyBytes(in, in.length());
> + zipOut.closeEntry();
> + }
> + }
> }
> }
>
> diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestGenerateBwcIndices.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestGenerateBwcIndices.java
> index 0cd9f37d5c3..c7b1ea3fb4a 100644
> --- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestGenerateBwcIndices.java
> +++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestGenerateBwcIndices.java
> @@ -20,8 +20,10 @@ import static org.apache.lucene.backward_index.BackwardsCompatibilityTestBase.cr
>
> import java.io.IOException;
> import org.apache.lucene.tests.util.LuceneTestCase;
> +import org.apache.lucene.tests.util.LuceneTestCase.SuppressFileSystems;
> import org.apache.lucene.util.Version;
>
> +@SuppressFileSystems("ExtrasFS")
> public class TestGenerateBwcIndices extends LuceneTestCase {
>
> // Backcompat index generation, described below, is mostly automated in:
>
--
Adrien
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org