Mailing List Archive

Author: creamyg
Date: 2008-09-09 20:36:51 -0700 (Tue, 09 Sep 2008)
New Revision: 3857

Modified:
trunk/perl/t/213-segment_merging.t
Log:
Revise segment merging test so that deletions-on-Windows tests work properly.

Modified: trunk/perl/t/213-segment_merging.t
===================================================================
--- trunk/perl/t/213-segment_merging.t 2008-09-10 02:53:54 UTC (rev 3856)
+++ trunk/perl/t/213-segment_merging.t 2008-09-10 03:36:51 UTC (rev 3857)
@@ -2,6 +2,13 @@
use warnings;
use lib 'buildlib';

+package NonMergingInvIndexer;
+use base qw( KinoSearch::InvIndexer );
+
+sub segreaders_to_merge {
+ return KinoSearch::Util::VArray->new( capacity => 0 );
+}
+
# BiggerSchema is like TestSchema, but it has an extra field named "aux".
# Because "aux" sorts before "content", it forces a remapping of field numbers
# when an invindex created under TestSchema is opened/modified under
@@ -18,7 +25,7 @@
sub analyzer { KinoSearch::Analysis::Tokenizer->new }

package main;
-use Test::More tests => 7;
+use Test::More tests => 8;

use KinoSearch::InvIndexer;
use KinoSearch::InvIndex;
@@ -29,71 +36,62 @@
use KinoSearch::Test::TestSchema;
use KinoSearch::Test::TestUtils qw( create_invindex init_test_invindex_loc );

-{ # Wrap script in a block, so we can test object destruction at end.

- my ( $invindex, $invindexer, $searcher, $hits );
-
+my $num_reps;
+{
# Verify that optimization truly cuts down on the number of segments.
my $folder = KinoSearch::Store::RAMFolder->new;
- $invindex = KinoSearch::InvIndex->clobber(
+ my $invindex = KinoSearch::InvIndex->clobber(
folder => $folder,
schema => KinoSearch::Test::TestSchema->new,
);
- for ( my $num_reps = 1;; $num_reps++ ) {
- $invindexer = KinoSearch::InvIndexer->new( invindex => $invindex );
+ for ( $num_reps = 1;; $num_reps++ ) {
+ my $invindexer = KinoSearch::InvIndexer->new( invindex => $invindex );
my $num_cf_files = grep {/\.cf$/} @{ $folder->list };
if ( $num_reps > 2 and $num_cf_files > 1 ) {
$invindexer->finish( optimize => 1 );
$num_cf_files = grep {/\.cf$/} @{ $folder->list };
is( $num_cf_files, 1, 'finish with optimize' );
- undef $invindex;
- undef $invindexer;
- undef $folder;
last;
}
else {
$invindexer->add_doc( { content => $_ } ) for 1 .. 5;
$invindexer->finish;
- undef $invindexer;
}
}
+}

- my $invindex_loc = init_test_invindex_loc();
- KinoSearch::Test::TestSchema->clobber($invindex_loc);
+my $invindex_loc = init_test_invindex_loc();
+KinoSearch::Test::TestSchema->clobber($invindex_loc);

- sub init_invindexer {
- undef $invindex;
- undef $invindexer;
- $invindex = KinoSearch::Test::TestSchema->open($invindex_loc);
- $invindexer = KinoSearch::InvIndexer->new( invindex => $invindex );
- }

- my @correct;
- for my $num_letters ( reverse 1 .. 10 ) {
- init_invindexer();
+my @correct;
+for my $num_letters ( reverse 1 .. 10 ) {
+ my $invindex = KinoSearch::Test::TestSchema->open($invindex_loc);
+ my $invindexer = KinoSearch::InvIndexer->new( invindex => $invindex );

- for my $letter ( 'a' .. 'b' ) {
- my $content = ( "$letter " x $num_letters ) . ( 'z ' x 50 );
- $invindexer->add_doc( { content => $content } );
- push @correct, $content if $letter eq 'b';
- }
- $invindexer->finish;
+ for my $letter ( 'a' .. 'b' ) {
+ my $content = ( "$letter " x $num_letters ) . ( 'z ' x 50 );
+ $invindexer->add_doc( { content => $content } );
+ push @correct, $content if $letter eq 'b';
}
+ $invindexer->finish;
+}

- $searcher = KinoSearch::Searcher->new(
+{
+ my $searcher = KinoSearch::Searcher->new(
invindex => KinoSearch::Test::TestSchema->read($invindex_loc) );
- $hits = $searcher->search( query => 'b' );
+ my $hits = $searcher->search( query => 'b' );
is( $hits->total_hits, 10, "correct total_hits from merged invindex" );
my @got;
push @got, $hits->next->{content} for 1 .. $hits->total_hits;
is_deeply( \@got, \@correct,
"correct top scoring hit from merged invindex" );
- undef $searcher;
- undef $hits;
+}

+{
# Reopen invindex under BiggerSchema and add some content.
- undef $invindexer;
- $invindexer = KinoSearch::InvIndexer->new(
+ my $invindexer = NonMergingInvIndexer->new(
invindex => BiggerSchema->open($invindex_loc) );
$invindexer->add_doc( { aux => 'foo', content => 'bar' } );

@@ -113,21 +111,25 @@
$invindexer->add_invindex($_)
for ( $another_invindex, $yet_another_invindex );
$invindexer->finish;
+ cmp_ok(num_cf_files($invindex_loc), '>', 1,
+ "non-merging InvIndexer should produce multi-seg index" );
+}

- $searcher = KinoSearch::Searcher->new(
+{
+ my $searcher = KinoSearch::Searcher->new(
invindex => BiggerSchema->read($invindex_loc) );
- $hits = $searcher->search( query => 'fish' );
+ my $hits = $searcher->search( query => 'fish' );
is( $hits->total_hits, 1, "correct total_hits after add_invindex" );
is( $hits->next->{content},
'fresh fish', "other invindexes successfully absorbed" );
- undef $searcher;
- undef $hits;
+}

- # Open an IndexReader, to prevent the deletion of files on Win32 and
+{
+ # Open an IndexReader, to prevent the deletion of files on Windows and
# verify the file purging mechanism.
my $reader = KinoSearch::Index::IndexReader->open(
invindex => BiggerSchema->read($invindex_loc) );
- $invindexer = KinoSearch::InvIndexer->new(
+ my $invindexer = KinoSearch::InvIndexer->new(
invindex => BiggerSchema->open($invindex_loc) );
$invindexer->finish( optimize => 1 );
$reader->close;
@@ -135,17 +137,20 @@
$invindexer = KinoSearch::InvIndexer->new(
invindex => BiggerSchema->open($invindex_loc) );
$invindexer->finish( optimize => 1 );
+}

- opendir( my $invindex_dh, $invindex_loc )
- or die "Couldn't opendir '$invindex_loc': $!";
+is( num_cf_files($invindex_loc), 1, "merged segment files successfully deleted" );
+
+is( KinoSearch::Store::FileDes::object_count(),
+ 0, "All FileDes objects have been cleaned up" );
+
+sub num_cf_files {
+ my $dir = shift;
+ opendir( my $invindex_dh, $dir )
+ or die "Couldn't opendir '$dir': $!";
my @cf_files = grep {m/\.cf$/} readdir $invindex_dh;
- closedir $invindex_dh, $invindex_loc
+ closedir $invindex_dh
or die "Couldn't closedir '$invindex_loc': $!";
- is( scalar @cf_files, 1, "merged segment files successfully deleted" );
-
- undef $invindexer;
- undef $invindex;
+ return scalar @cf_files;
}

-is( KinoSearch::Store::FileDes::object_count(),
- 0, "All FileDes objects have been cleaned up" );


_______________________________________________
kinosearch-commits mailing list
kinosearch-commits@rectangular.com
http://www.rectangular.com/mailman/listinfo/kinosearch-commits