001
014
015 package com.liferay.portal.search.lucene;
016
017 import com.liferay.portal.kernel.log.Log;
018 import com.liferay.portal.kernel.log.LogFactoryUtil;
019 import com.liferay.portal.kernel.search.SearchEngineUtil;
020 import com.liferay.portal.kernel.util.FileUtil;
021 import com.liferay.portal.kernel.util.InstanceFactory;
022 import com.liferay.portal.kernel.util.StringPool;
023 import com.liferay.portal.search.lucene.dump.DumpIndexDeletionPolicy;
024 import com.liferay.portal.search.lucene.dump.IndexCommitSerializationUtil;
025 import com.liferay.portal.util.ClassLoaderUtil;
026 import com.liferay.portal.util.PropsValues;
027
028 import java.io.File;
029 import java.io.IOException;
030 import java.io.InputStream;
031 import java.io.OutputStream;
032
033 import java.util.Map;
034 import java.util.concurrent.ConcurrentHashMap;
035 import java.util.concurrent.Executors;
036 import java.util.concurrent.ScheduledExecutorService;
037 import java.util.concurrent.TimeUnit;
038 import java.util.concurrent.locks.Lock;
039 import java.util.concurrent.locks.ReentrantLock;
040
041 import org.apache.lucene.analysis.Analyzer;
042 import org.apache.lucene.analysis.LimitTokenCountAnalyzer;
043 import org.apache.lucene.document.Document;
044 import org.apache.lucene.index.IndexReader;
045 import org.apache.lucene.index.IndexWriter;
046 import org.apache.lucene.index.IndexWriterConfig;
047 import org.apache.lucene.index.LogMergePolicy;
048 import org.apache.lucene.index.MergePolicy;
049 import org.apache.lucene.index.Term;
050 import org.apache.lucene.search.IndexSearcher;
051 import org.apache.lucene.search.MatchAllDocsQuery;
052 import org.apache.lucene.search.ScoreDoc;
053 import org.apache.lucene.search.TopDocs;
054 import org.apache.lucene.store.Directory;
055 import org.apache.lucene.store.FSDirectory;
056 import org.apache.lucene.store.MMapDirectory;
057 import org.apache.lucene.store.RAMDirectory;
058
059
065 public class IndexAccessorImpl implements IndexAccessor {
066
067 public IndexAccessorImpl(long companyId) {
068 _companyId = companyId;
069
070 _checkLuceneDir();
071 _initIndexWriter();
072 _initCommitScheduler();
073 }
074
075 @Override
076 public void addDocument(Document document) throws IOException {
077 if (SearchEngineUtil.isIndexReadOnly()) {
078 return;
079 }
080
081 _write(null, document);
082 }
083
084 @Override
085 public void close() {
086 try {
087 _indexWriter.close();
088 }
089 catch (Exception e) {
090 _log.error("Closing Lucene writer failed for " + _companyId, e);
091 }
092 }
093
094 @Override
095 public void delete() {
096 if (SearchEngineUtil.isIndexReadOnly()) {
097 return;
098 }
099
100 _deleteDirectory();
101 }
102
103 @Override
104 public void deleteDocuments(Term term) throws IOException {
105 if (SearchEngineUtil.isIndexReadOnly()) {
106 return;
107 }
108
109 try {
110 _indexWriter.deleteDocuments(term);
111
112 _batchCount++;
113 }
114 finally {
115 _commit();
116 }
117 }
118
119 @Override
120 public void dumpIndex(OutputStream outputStream) throws IOException {
121 _dumpIndexDeletionPolicy.dump(outputStream, _indexWriter, _commitLock);
122 }
123
124 @Override
125 public long getCompanyId() {
126 return _companyId;
127 }
128
129 @Override
130 public long getLastGeneration() {
131 return _dumpIndexDeletionPolicy.getLastGeneration();
132 }
133
134 @Override
135 public Directory getLuceneDir() {
136 if (_log.isDebugEnabled()) {
137 _log.debug("Lucene store type " + PropsValues.LUCENE_STORE_TYPE);
138 }
139
140 if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_FILE)) {
141 return _getLuceneDirFile();
142 }
143 else if (PropsValues.LUCENE_STORE_TYPE.equals(
144 _LUCENE_STORE_TYPE_JDBC)) {
145
146 throw new IllegalArgumentException(
147 "Store type JDBC is no longer supported in favor of SOLR");
148 }
149 else if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_RAM)) {
150 return _getLuceneDirRam();
151 }
152 else {
153 throw new RuntimeException(
154 "Invalid store type " + PropsValues.LUCENE_STORE_TYPE);
155 }
156 }
157
158 @Override
159 public void loadIndex(InputStream inputStream) throws IOException {
160 File tempFile = FileUtil.createTempFile();
161
162 Directory tempDirectory = FSDirectory.open(tempFile);
163
164 IndexCommitSerializationUtil.deserializeIndex(
165 inputStream, tempDirectory);
166
167 _deleteDirectory();
168
169 IndexReader indexReader = IndexReader.open(tempDirectory, false);
170
171 IndexSearcher indexSearcher = new IndexSearcher(indexReader);
172
173 try {
174 TopDocs topDocs = indexSearcher.search(
175 new MatchAllDocsQuery(), indexReader.numDocs());
176
177 ScoreDoc[] scoreDocs = topDocs.scoreDocs;
178
179 for (ScoreDoc scoreDoc : scoreDocs) {
180 Document document = indexSearcher.doc(scoreDoc.doc);
181
182 addDocument(document);
183 }
184 }
185 catch (IllegalArgumentException iae) {
186 if (_log.isDebugEnabled()) {
187 _log.debug(iae.getMessage());
188 }
189 }
190
191 indexSearcher.close();
192
193 indexReader.flush();
194 indexReader.close();
195
196 tempDirectory.close();
197
198 FileUtil.deltree(tempFile);
199 }
200
201 @Override
202 public void updateDocument(Term term, Document document)
203 throws IOException {
204
205 if (SearchEngineUtil.isIndexReadOnly()) {
206 return;
207 }
208
209 if (_log.isDebugEnabled()) {
210 _log.debug("Indexing " + document);
211 }
212
213 _write(term, document);
214 }
215
216 private void _checkLuceneDir() {
217 if (SearchEngineUtil.isIndexReadOnly()) {
218 return;
219 }
220
221 try {
222 Directory directory = getLuceneDir();
223
224 if (IndexWriter.isLocked(directory)) {
225 IndexWriter.unlock(directory);
226 }
227 }
228 catch (Exception e) {
229 _log.error("Check Lucene directory failed for " + _companyId, e);
230 }
231 }
232
233 private void _commit() throws IOException {
234 if ((PropsValues.LUCENE_COMMIT_BATCH_SIZE == 0) ||
235 (PropsValues.LUCENE_COMMIT_BATCH_SIZE <= _batchCount)) {
236
237 _doCommit();
238 }
239 }
240
241 private void _deleteDirectory() {
242 if (_log.isDebugEnabled()) {
243 _log.debug("Lucene store type " + PropsValues.LUCENE_STORE_TYPE);
244 }
245
246 if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_FILE)) {
247 _deleteFile();
248 }
249 else if (PropsValues.LUCENE_STORE_TYPE.equals(
250 _LUCENE_STORE_TYPE_JDBC)) {
251
252 throw new IllegalArgumentException(
253 "Store type JDBC is no longer supported in favor of SOLR");
254 }
255 else if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_RAM)) {
256 _deleteRam();
257 }
258 else {
259 throw new RuntimeException(
260 "Invalid store type " + PropsValues.LUCENE_STORE_TYPE);
261 }
262 }
263
264 private void _deleteFile() {
265 String path = _getPath();
266
267 try {
268 _indexWriter.deleteAll();
269
270
271
272 _indexWriter.commit();
273 }
274 catch (Exception e) {
275 if (_log.isWarnEnabled()) {
276 _log.warn("Could not delete index in directory " + path);
277 }
278 }
279 }
280
281 private void _deleteRam() {
282 }
283
284 private void _doCommit() throws IOException {
285 if (_indexWriter != null) {
286 _commitLock.lock();
287
288 try {
289 _indexWriter.commit();
290 }
291 finally {
292 _commitLock.unlock();
293 }
294 }
295
296 _batchCount = 0;
297 }
298
299 private FSDirectory _getDirectory(String path) throws IOException {
300 if (PropsValues.LUCENE_STORE_TYPE_FILE_FORCE_MMAP) {
301 return new MMapDirectory(new File(path));
302 }
303 else {
304 return FSDirectory.open(new File(path));
305 }
306 }
307
308 private Directory _getLuceneDirFile() {
309 Directory directory = null;
310
311 String path = _getPath();
312
313 try {
314 directory = _getDirectory(path);
315 }
316 catch (IOException ioe1) {
317 if (directory != null) {
318 try {
319 directory.close();
320 }
321 catch (Exception e) {
322 }
323 }
324 }
325
326 return directory;
327 }
328
329 private Directory _getLuceneDirRam() {
330 String path = _getPath();
331
332 Directory directory = _ramDirectories.get(path);
333
334 if (directory == null) {
335 directory = new RAMDirectory();
336
337 _ramDirectories.put(path, directory);
338 }
339
340 return directory;
341 }
342
343 private MergePolicy _getMergePolicy() throws Exception {
344 ClassLoader classLoader = ClassLoaderUtil.getPortalClassLoader();
345
346 MergePolicy mergePolicy = (MergePolicy)InstanceFactory.newInstance(
347 classLoader, PropsValues.LUCENE_MERGE_POLICY);
348
349 if (mergePolicy instanceof LogMergePolicy) {
350 LogMergePolicy logMergePolicy = (LogMergePolicy)mergePolicy;
351
352 logMergePolicy.setMergeFactor(PropsValues.LUCENE_MERGE_FACTOR);
353 }
354
355 return mergePolicy;
356 }
357
358 private String _getPath() {
359 return PropsValues.LUCENE_DIR.concat(String.valueOf(_companyId)).concat(
360 StringPool.SLASH);
361 }
362
363 private void _initCommitScheduler() {
364 if ((PropsValues.LUCENE_COMMIT_BATCH_SIZE <= 0) ||
365 (PropsValues.LUCENE_COMMIT_TIME_INTERVAL <= 0)) {
366
367 return;
368 }
369
370 ScheduledExecutorService scheduledExecutorService =
371 Executors.newSingleThreadScheduledExecutor();
372
373 Runnable runnable = new Runnable() {
374
375 @Override
376 public void run() {
377 try {
378 if (_batchCount > 0) {
379 _doCommit();
380 }
381 }
382 catch (IOException ioe) {
383 _log.error("Could not run scheduled commit", ioe);
384 }
385 }
386
387 };
388
389 scheduledExecutorService.scheduleWithFixedDelay(
390 runnable, 0, PropsValues.LUCENE_COMMIT_TIME_INTERVAL,
391 TimeUnit.MILLISECONDS);
392 }
393
394 private void _initIndexWriter() {
395 try {
396 Analyzer analyzer = new LimitTokenCountAnalyzer(
397 LuceneHelperUtil.getAnalyzer(),
398 PropsValues.LUCENE_ANALYZER_MAX_TOKENS);
399
400 IndexWriterConfig indexWriterConfig = new IndexWriterConfig(
401 LuceneHelperUtil.getVersion(), analyzer);
402
403 indexWriterConfig.setIndexDeletionPolicy(_dumpIndexDeletionPolicy);
404 indexWriterConfig.setMergePolicy(_getMergePolicy());
405 indexWriterConfig.setRAMBufferSizeMB(
406 PropsValues.LUCENE_BUFFER_SIZE);
407
408 _indexWriter = new IndexWriter(getLuceneDir(), indexWriterConfig);
409
410 if (!IndexReader.indexExists(getLuceneDir())) {
411
412
413
414 if (_log.isDebugEnabled()) {
415 _log.debug("Creating missing index");
416 }
417
418 _doCommit();
419 }
420 }
421 catch (Exception e) {
422 _log.error(
423 "Initializing Lucene writer failed for " + _companyId, e);
424 }
425 }
426
427 private void _write(Term term, Document document) throws IOException {
428 try {
429 if (term != null) {
430 _indexWriter.updateDocument(term, document);
431 }
432 else {
433 _indexWriter.addDocument(document);
434 }
435
436 _batchCount++;
437 }
438 finally {
439 _commit();
440 }
441 }
442
443 private static final String _LUCENE_STORE_TYPE_FILE = "file";
444
445 private static final String _LUCENE_STORE_TYPE_JDBC = "jdbc";
446
447 private static final String _LUCENE_STORE_TYPE_RAM = "ram";
448
449 private static Log _log = LogFactoryUtil.getLog(IndexAccessorImpl.class);
450
451 private volatile int _batchCount;
452 private Lock _commitLock = new ReentrantLock();
453 private long _companyId;
454 private DumpIndexDeletionPolicy _dumpIndexDeletionPolicy =
455 new DumpIndexDeletionPolicy();
456 private IndexWriter _indexWriter;
457 private Map<String, Directory> _ramDirectories =
458 new ConcurrentHashMap<String, Directory>();
459
460 }