001 package com.sptci.prevayler;
002
003 import com.sptci.ReflectionUtility;
004 import com.sptci.prevayler.annotations.Searchable;
005 import com.sptci.prevayler.annotations.Searchables;
006 import org.apache.lucene.analysis.standard.StandardAnalyzer;
007 import org.apache.lucene.document.Document;
008 import org.apache.lucene.index.IndexReader;
009 import org.apache.lucene.index.IndexWriter;
010 import org.apache.lucene.index.Term;
011 import org.apache.lucene.search.Filter;
012 import org.apache.lucene.search.IndexSearcher;
013 import org.apache.lucene.search.Query;
014 import org.apache.lucene.search.ScoreDoc;
015 import org.apache.lucene.search.Sort;
016 import org.apache.lucene.search.TopDocs;
017 import org.apache.lucene.store.NIOFSDirectory;
018
019 import java.io.IOException;
020 import java.lang.reflect.Field;
021 import java.util.Collection;
022 import java.util.logging.Level;
023
024 /**
025 * Abstracts all the full-text search indexing and de-indexing operations
026 * for the prevalent system. Indexing is performed using
027 * <a href='http://lucene.apache.org/java/docs/index.html' target='_top'>Lucene</a>.
028 *
029 * <p><b>Note:</b>The field values are converted to {@link String} using
030 * the {@link Object#toString()} method to retrieve the content to index.
031 * To ensure meaningful indices ensure that the fields annotated return
032 * meaningful values.</p>
033 *
034 * <p>© Copyright 2008 <a href='http://sptci.com/' target='_top'>Sans
035 * Pareil Technologies, Inc.</a></p>
036 * @author Rakesh Vidyadharan 2008-11-12
037 * @since Release 0.3.0
038 * @version $Id: SearchSystem.java 23 2008-11-24 19:49:55Z sptrakesh $
039 */
040 abstract class SearchSystem extends ConstraintSystem
041 {
042 private static final long serialVersionUID = 1l;
043
044 /** The index writer instance to use to create/delete documents. */
045 protected static transient final IndexWriter writer;
046
047 /** The index reader instance to use to read the index. */
048 protected static transient IndexReader reader;
049
050 /** The index searcher instance to use to search documents. */
051 protected static transient IndexSearcher searcher;
052
053 /**
054 * The name of the field in the indexed document that stores the unique
055 * document id.
056 */
057 static final String DOCUMENT_ID_FIELD = "documentId";
058
059 /**
060 * The name of the field in the indexed document that stores the class name.
061 *
062 * {@value}
063 */
064 public static final String CLASS_FIELD = "class";
065
066 /**
067 * The name of the field in the indexed document that stores the object id.
068 *
069 * {@value}
070 */
071 public static final String OBJECT_ID_FIELD = "objectId";
072
073 /**
074 * The separator character used to delimit composite index field names.
075 *
076 * {@value}
077 */
078 public static final char SEPARATOR_CHAR = '#';
079
080 /**
081 * The number or index updates before a {@link
082 * org.apache.lucene.index.IndexWriter#commit()} is invoked. Invoked only
083 * in batches to avoid expensive operation on each transaction.
084 */
085 public static final int SAVE_COUNT =
086 PrevalentSystemFactory.getSearchBatchSize();
087
088 /**
089 * A counter to ensure that the {@link org.apache.lucene.index.IndexWriter#commit()}
090 * method is called periodically.
091 */
092 private int saveCount;
093
094 /**
095 * Initialise {@link #writer}, {@link #reader}, and {@link #searcher}
096 * instances. Register {@link Closer} as a JVM shutdown hook to ensure
097 * that the indices are properly optimised and closed on exit.
098 */
099 static
100 {
101 try
102 {
103 writer = new IndexWriter(
104 PrevalentSystemFactory.getSearchDirectory( PrevalentObject.class ),
105 new StandardAnalyzer(), IndexWriter.MaxFieldLength.UNLIMITED );
106 reader = IndexReader.open( NIOFSDirectory.getDirectory(
107 PrevalentSystemFactory.getSearchDirectory( PrevalentObject.class ) ),
108 true );
109 searcher = new IndexSearcher( reader );
110 Runtime.getRuntime().addShutdownHook( new Closer() );
111 }
112 catch ( IOException e )
113 {
114 throw new RuntimeException( e );
115 }
116 }
117
118 /**
119 * Over-ridden to process search annotations for the object. If full=text
120 * search has been specified, create a {@link
121 * org.apache.lucene.document.Document} that holds all the indexed fields
122 * in the prevalent object.
123 *
124 * @param object The prevalent object to add query support for.
125 * @throws com.sptci.prevayler.PrevalentException If errors are encountered
126 * while fetching the values of the fields in prevalent object.
127 */
128 @Override
129 protected void index( final PrevalentObject object ) throws PrevalentException
130 {
131 if ( hasIndices( object ) )
132 {
133 final Document document = createDocument( object );
134
135 try
136 {
137 indexFields( object, document );
138 indexClass( object, document );
139 save( object, document );
140 }
141 catch ( Exception e )
142 {
143 throw new PrevalentException( e );
144 }
145 }
146
147 super.index( object );
148 }
149
150 /**
151 * Over-ridden to remove the search index for the specified prevalent object.
152 *
153 * {@inheritDoc}
154 */
155 @Override
156 protected void remove( final PrevalentObject object ) throws PrevalentException
157 {
158 super.remove( object );
159
160 try
161 {
162 writer.deleteDocuments(
163 new Term( DOCUMENT_ID_FIELD, getDocumentId( object ) ) );
164 ++saveCount;
165
166 if ( ( saveCount % SAVE_COUNT ) == 0 ) commit();
167 }
168 catch ( IOException e )
169 {
170 throw new PrevalentException( e );
171 }
172 }
173
174 /**
175 * Determine whether the prevalent object has any full-text search indices
176 * specified.
177 *
178 * @param object The prevalent object to check.
179 * @return Return <code>true</code> if any search indices have been specified.
180 * @throws com.sptci.prevayler.PrevalentException If errors are encountered
181 * while reflecting upon the annotations in the prevalent object.
182 */
183 private boolean hasIndices( final PrevalentObject object )
184 throws PrevalentException
185 {
186 try
187 {
188 Searchables indices = object.getClass().getAnnotation( Searchables.class );
189 if ( indices != null ) return true;
190 Searchable index = object.getClass().getAnnotation( Searchable.class );
191 if ( index != null ) return true;
192
193 for ( Field field : ReflectionUtility.fetchFields( object ).values() )
194 {
195 index = field.getAnnotation( Searchable.class );
196 if ( index != null ) return true;
197 }
198 }
199 catch ( Throwable t )
200 {
201 throw new PrevalentException( t );
202 }
203
204 return false;
205 }
206
207 /**
208 * Create a document to store the full-text indices for the specified
209 * object.
210 *
211 * @see #getDocumentId(PrevalentObject)
212 * @param object The prevalent object to index.
213 * @return The new document instance.
214 */
215 private Document createDocument( final PrevalentObject object )
216 {
217 final Document document = new Document();
218 document.add( new org.apache.lucene.document.Field( DOCUMENT_ID_FIELD,
219 getDocumentId( object ),
220 org.apache.lucene.document.Field.Store.NO,
221 org.apache.lucene.document.Field.Index.NOT_ANALYZED ) );
222 document.add( new org.apache.lucene.document.Field( CLASS_FIELD,
223 object.getClass().getName(), org.apache.lucene.document.Field.Store.YES,
224 org.apache.lucene.document.Field.Index.NOT_ANALYZED ) );
225 document.add( new org.apache.lucene.document.Field( OBJECT_ID_FIELD,
226 object.getObjectId().toString(),
227 org.apache.lucene.document.Field.Store.YES,
228 org.apache.lucene.document.Field.Index.NOT_ANALYZED ) );
229 return document;
230 }
231
232 /**
233 * Return the unique document id for the index document representing the
234 * specified prevalent object.
235 *
236 * @param object The prevalent object for which the document id is to be
237 * retrieved.
238 * @return The unique document identifier.
239 */
240 private String getDocumentId( final PrevalentObject object )
241 {
242 return object.getClass().getName() + SEPARATOR_CHAR +
243 object.getObjectId().toString();
244 }
245
246 /**
247 * Add full=text search indices for any fields annotated as searchable in
248 * the prevalent object.
249 *
250 * @param object The prevalent object whose fields are to be indexed.
251 * @param doc The document to which indices are to be added.
252 * @throws IllegalAccessException If errors are encountered
253 * while reflecting upon the fields in the prevalent object.
254 */
255 private void indexFields( final PrevalentObject object,
256 final Document doc ) throws IllegalAccessException
257 {
258 for ( Field field : ReflectionUtility.fetchFields( object ).values() )
259 {
260 final Searchable index = field.getAnnotation( Searchable.class );
261 if ( index != null )
262 {
263 final Object value = field.get( object );
264 if ( value != null )
265 {
266 doc.add( new org.apache.lucene.document.Field( field.getName(),
267 value.toString(), org.apache.lucene.document.Field.Store.NO,
268 org.apache.lucene.document.Field.Index.ANALYZED ) );
269 }
270 }
271 }
272 }
273
274 /**
275 * Add full-text search indices for composite indices defined at the
276 * class level. Composite index field names are assigned as a concatenation
277 * of the names of the fields that comprise the index if no value is
278 * specified for {@link com.sptci.prevayler.annotations.Searchable#name()}.
279 * The concatenated names are delimited by the {@link #SEPARATOR_CHAR}.
280 *
281 * @see #indexSearchables(PrevalentObject, org.apache.lucene.document.Document)
282 * @see #indexSearchable(PrevalentObject, org.apache.lucene.document.Document,
283 * com.sptci.prevayler.annotations.Searchable)
284 * @param object The prevalent object whose composite fields are to be indexed.
285 * @param doc The document to which indices are to be added.
286 * @throws IllegalAccessException If errors are encountered
287 * while reflecting upon the fields in the prevalent object.
288 */
289 private void indexClass( final PrevalentObject object,
290 final Document doc ) throws IllegalAccessException
291 {
292 indexSearchables( object, doc );
293 Searchable index = object.getClass().getAnnotation( Searchable.class );
294 if ( index != null ) indexSearchable( object, doc, index );
295 }
296
297 /**
298 * Process the {@link com.sptci.prevayler.annotations.Searchables} annotation
299 * for the specified prevalent object class. Create the composite
300 * indices for the specified array of composite indices.
301 *
302 * @param object The prevalent object whose composite fields are to be indexed.
303 * @param doc The document to which indices are to be added.
304 * @throws IllegalAccessException If errors are encountered while fetching
305 * the field values of the prevalent object.
306 */
307 private void indexSearchables( final PrevalentObject object,
308 final Document doc ) throws IllegalAccessException
309 {
310 Searchables indices = object.getClass().getAnnotation( Searchables.class );
311 if ( indices != null )
312 {
313 for ( Searchable index : indices.value() )
314 {
315 indexSearchable( object, doc, index );
316 }
317 }
318 }
319
320 /**
321 * Process the {@link com.sptci.prevayler.annotations.Searchable} annotation
322 * for the specified prevalent object class. Create the composite index
323 * for the array of fields.
324 *
325 * @param object The prevalent object whose composite fields are to be indexed.
326 * @param doc The document to which indices are to be added.
327 * @param index The searchable index annotation to process.
328 * @throws IllegalAccessException If errors are encountered while fetching
329 * the field values of the prevalent object.
330 */
331 private void indexSearchable( final PrevalentObject object,
332 final Document doc, final Searchable index ) throws IllegalAccessException
333 {
334 final StringBuilder name = new StringBuilder( 64 );
335 final StringBuilder value = new StringBuilder( 1024 );
336 boolean separator = false;
337
338 for ( String field : index.members() )
339 {
340 if ( separator ) name.append( SEPARATOR_CHAR );
341 name.append( field );
342
343 final Field f = ReflectionUtility.fetchField( field, object );
344 final Object v = f.get( object );
345 if ( v != null )
346 {
347 value.append( v.toString() );
348 value.append( " " );
349 }
350
351 separator = true;
352 }
353
354 String fieldName = index.name();
355 if ( Searchable.NULL.equals( fieldName ) ) fieldName = name.toString();
356 doc.add( new org.apache.lucene.document.Field( fieldName,
357 value.toString(), org.apache.lucene.document.Field.Store.NO,
358 org.apache.lucene.document.Field.Index.ANALYZED ) );
359 }
360
361 /**
362 * Save the specified document to the index writer. Delete an existing
363 * document instance if it exists.
364 *
365 * @see org.apache.lucene.index.IndexWriter#updateDocument(
366 * org.apache.lucene.index.Term, org.apache.lucene.document.Document)
367 * @see #commit
368 * @param object The prevalent object that is being indexed.
369 * @param document The new document to add to the index.
370 * @throws java.io.IOException If errors are encountered while saving the
371 * document.
372 */
373 private void save( final PrevalentObject object,
374 final Document document ) throws IOException
375 {
376 writer.updateDocument(
377 new Term( DOCUMENT_ID_FIELD, getDocumentId( object ) ), document );
378 ++saveCount;
379
380 if ( ( saveCount % SAVE_COUNT ) == 0 ) commit();
381 }
382
383 /**
384 * Commit pending writes to the index and refresh the reader.
385 *
386 * @throws java.io.IOException If errors are encountered while saving the
387 * document.
388 */
389 private void commit() throws IOException
390 {
391 writer.commit();
392 final IndexReader ir = reader.reopen();
393
394 if ( ir != reader )
395 {
396 new IndexCloser( reader, searcher ).start();
397 reader = ir;
398 searcher = new IndexSearcher( reader );
399 }
400
401 logger.fine( "Commited search index writes" );
402 }
403
404 /**
405 * Search the search indices and retrieve all the objects (regardless of type)
406 * that match the specified query and return up to the specified number of
407 * results.
408 *
409 * @see #fetch(Class, Object)
410 * @param query The query to execute.
411 * @param filter The filter to apply to the search.
412 * @param count The maximum number of results to return.
413 * @param sort The sort fields to apply to the results.
414 * @param collection The collection to which matching objects are added.
415 * @throws Exception If errors are encountered while reconstituting the
416 * prevalent objects or executing the search.
417 */
418 protected void search( final Query query, final Filter filter,
419 final int count, final Sort sort,
420 final Collection<PrevalentObject> collection ) throws Exception
421 {
422 try
423 {
424 reader.incRef();
425 final TopDocs docs = searcher.search( query, filter, count,
426 ( ( sort == null ) ? new Sort() : sort ) );
427 for ( ScoreDoc sd : docs.scoreDocs )
428 {
429 final Document doc = searcher.doc( sd.doc );
430 final Class prevalentClass = Class.forName( doc.get( CLASS_FIELD ) );
431 final PrevalentObject obj =
432 (PrevalentObject) prevalentClass.newInstance();
433 collection.add( fetch(
434 prevalentClass, obj.getObjectId( doc.get( OBJECT_ID_FIELD ) ) ) );
435 }
436 }
437 finally
438 {
439 reader.decRef();
440 }
441 }
442
443 /**
444 * A thread used as a shutdown hook to the JVM to close the {@link
445 * SearchSystem#writer} and other lucene resources.
446 */
447 private static class Closer extends Thread
448 {
449 /**
450 * Close the {@link SearchSystem#writer}, {@link SearchSystem#reader}
451 * and {@link SearchSystem#searcher} instances.
452 */
453 @Override
454 public void run()
455 {
456 try
457 {
458 logger.info( "Optimising search index" );
459 writer.optimize();
460 writer.close();
461 logger.info( "Closed index writer" );
462
463 reader.close();
464 searcher.close();
465 }
466 catch ( IOException e )
467 {
468 logger.log( Level.WARNING, "Error while closing index writer.", e );
469 }
470 }
471 }
472 }