Mailing List Archive

It seems that a lot of people run into the same set of problems with
maintaining readers, writers, and searchers in Lucene. Part of the problem
comes from the fact that delete() is on a reader and part of it comes from
the need to keep a reader or search open as long as possible to avoid
open/close overhead. Add to that the need for some applications (like mine)
to maintain multiple indexes and you have a pretty complex problem space.
Bottom line: Use of Lucene (especially in an online, interactive
application) doesn't seem as easy as it could be.

Anyway... I've attempted to address these issues in my application with a
control class. And, I'm submitting what I have to lucene dev for review...
and consideration for possible inclusion in the core lucene framework. I
tried to be fairly efficient and to handle all cases that I know about so
let me know if it fails in either case and what you think overall.

Thanks,
Scott

-------

import java.io.*;
import java.util.*;
import org.apache.lucene.index.*;
import org.apache.lucene.document.*;
import org.apache.lucene.search.*;
import org.apache.lucene.analysis.*;

/** Rules:
* Once created, searchers are valid until closed.
* If doing an update, searchers must not be created between close of the
reader
* that deletes and close of the writer that adds.
* Writers may be reused until Reader or Searcher is needed.
* Readers may be reused until Writer or Searcher is needed.
* Searchers may be reused until the index is changed.
* There may be only one of a Reader and/or Writer at a time.
* If you get it, release it.
*/
public class IndexAccessControl
{
public static final Analyzer LUCENE_ANALYZER = new LuceneAnalyzer();
private static final Map WRITER_PATHS = new HashMap(); // path ->
CheckoutInfo
private static final Map SEARCHER_PATHS = new HashMap(); // path ->
Searcher
private static final Map OLD_SEARCHERS = new HashMap(); // Searcher ->
CheckoutInfo

/** get for adding documents.
* blocks: readers until released
*/
public static IndexWriter getWriter(File path) throws IOException
{
IndexWriter writer = null;
String sync = path.getAbsolutePath().intern();
synchronized (sync) // sync on specific index
{
do
{
CheckoutInfo info = (CheckoutInfo)WRITER_PATHS.get(path);
if (info != null) // may already have a writer, use it
{
if (info.writer != null) // yup, have a writer
{
info.checkoutCount++;
writer = info.writer;
}
else // not a writer, it must be a reader, wait for it
to finish to try again
{
try
{
info.wait(); // wait for info to be released
}
catch (InterruptedException e)
{
// TODO: Will this ever happen?
e.printStackTrace();
return null;
}
}
}
else // no writer, create one
{
boolean missing = !path.exists();
if (missing) path.mkdir();
writer = new IndexWriter(path, LUCENE_ANALYZER,
/*create*/missing);
writer.mergeFactor = 2;
info = new CheckoutInfo(writer);
}
}
while (writer == null);
}
return writer;
}

public static void releaseWriter(File path, IndexWriter writer) throws
IOException
{
String sync = path.getAbsolutePath().intern();
synchronized (sync) // sync on specific index
{
CheckoutInfo info = (CheckoutInfo)WRITER_PATHS.get(path);
if (info != null && writer == info.writer) // writer was checked
out
{
if (info.checkoutCount > 1) // writer has other references
{
info.checkoutCount--;
writer = null; // avoid close()
}
else // last reference to writer
{
WRITER_PATHS.remove(path);
writer = info.writer;
info.notify(); // notify waiters to try again
}
}
}
// close the writer (unless it still has checkouts)
if (writer != null) writer.close();
}

/** get for search. */
public static Searcher getSearcher(File path) throws IOException
{
IndexSearcher is;
String sync = path.getAbsolutePath().intern();
synchronized (sync) // sync on specific index
{
CheckoutInfo info = (CheckoutInfo)SEARCHER_PATHS.get(path);
if (info == null || IndexReader.lastModified(path) >
info.creationTime)
{
// need new searcher
is = new IndexSearcher(IndexReader.open(path));
info = (CheckoutInfo)SEARCHER_PATHS.put(path, new
CheckoutInfo(is));
if (info != null)
{
if (info.checkoutCount > 1) // searcher has other
references
{
info.checkoutCount--;
OLD_SEARCHERS.put(info.searcher, info);
}
else // last reference to searcher
{
info.searcher.close();
}
}
}
else
{
// use existing searcher
is = info.searcher;
info.checkoutCount++;
}
}
return is;
}

public static void releaseSearcher(File path, Searcher searcher) throws
IOException
{
String sync = path.getAbsolutePath().intern();
synchronized (sync) // sync on specific index
{
CheckoutInfo info = (CheckoutInfo)SEARCHER_PATHS.get(path);
if (info == null || searcher != info.searcher) // this isn't the
info we're looking for
{
info = (CheckoutInfo)OLD_SEARCHERS.get(searcher);
}
if (info != null) // found a searcher
{
if (info.checkoutCount > 1) // searcher has other references
{
info.checkoutCount--;
}
else // last reference to searcher
{
info.searcher.close();
}
}
else // can't find searcher, just close it
{
searcher.close();
}
}
}

/** get for deleting documents.
* blocks: writers until released
*/
public static IndexReader getReader(File path) throws IOException
{
IndexReader reader = null;
String sync = path.getAbsolutePath().intern();
synchronized (sync) // sync on specific index
{
do
{
CheckoutInfo info = (CheckoutInfo)WRITER_PATHS.get(path);
if (info != null) // may already have a reader, use it
{
if (info.reader != null) // yup, have a reader
{
info.checkoutCount++;
reader = info.reader;
}
else // not a reader, it must be a writer, wait for it
to finish to try again
{
try
{
info.wait(); // wait for info to be released
}
catch (InterruptedException e)
{
// TODO: Will this ever happen?
e.printStackTrace();
return null;
}
}
}
else // no reader, create one
{
reader = IndexReader.open(path);
info = new CheckoutInfo(reader);
}
}
while (reader == null);
}
return reader;
}

public static void releaseReader(File path, IndexReader reader) throws
IOException
{
String sync = path.getAbsolutePath().intern();
synchronized (sync) // sync on specific index
{
CheckoutInfo info = (CheckoutInfo)WRITER_PATHS.get(path);
if (info != null && reader == info.reader) // reader was checked
out
{
if (info.checkoutCount > 1) // reader has other references
{
info.checkoutCount--;
reader = null; // avoid close()
}
else // last reference to reader
{
WRITER_PATHS.remove(path);
reader = info.reader;
info.notify(); // notify waiters to try again
}
}
}
// close the reader (unless it still has checkouts)
if (reader != null) reader.close();
}

/** used for updates to make sure nobody else grabs a writer or reader
between
* release and get operations.
*/
public static IndexWriter releaseReaderAndGetWriter(File path,
IndexReader reader) throws IOException
{
String sync = path.getAbsolutePath().intern();
synchronized (sync) // sync on specific index
{
releaseReader(path, reader);
return getWriter(path);
}
}

private static class CheckoutInfo
{
CheckoutInfo(IndexWriter writer)
{
this.writer = writer;
}
CheckoutInfo(IndexReader reader)
{
this.reader = reader;
}
CheckoutInfo(IndexSearcher searcher)
{
this.searcher = searcher;
this.creationTime = System.currentTimeMillis();
}

public IndexReader reader;
public IndexWriter writer;
public IndexSearcher searcher;
public int checkoutCount = 1;
public long creationTime;
}

/** no instances */
private IndexAccessControl() { }
}