001
014
015 package com.liferay.portal.search.lucene;
016
017 import com.liferay.portal.kernel.dao.jdbc.DataAccess;
018 import com.liferay.portal.kernel.log.Log;
019 import com.liferay.portal.kernel.log.LogFactoryUtil;
020 import com.liferay.portal.kernel.search.SearchEngineUtil;
021 import com.liferay.portal.kernel.util.FileUtil;
022 import com.liferay.portal.kernel.util.InfrastructureUtil;
023 import com.liferay.portal.kernel.util.PortalClassLoaderUtil;
024 import com.liferay.portal.kernel.util.PropsKeys;
025 import com.liferay.portal.kernel.util.StringPool;
026 import com.liferay.portal.util.PropsUtil;
027 import com.liferay.portal.util.PropsValues;
028
029 import java.io.File;
030 import java.io.IOException;
031
032 import java.sql.Connection;
033 import java.sql.DatabaseMetaData;
034 import java.sql.ResultSet;
035 import java.sql.Statement;
036
037 import java.util.Map;
038 import java.util.concurrent.ConcurrentHashMap;
039 import java.util.concurrent.Executors;
040 import java.util.concurrent.ScheduledExecutorService;
041 import java.util.concurrent.TimeUnit;
042
043 import javax.sql.DataSource;
044
045 import org.apache.lucene.document.Document;
046 import org.apache.lucene.index.IndexWriter;
047 import org.apache.lucene.index.Term;
048 import org.apache.lucene.store.Directory;
049 import org.apache.lucene.store.FSDirectory;
050 import org.apache.lucene.store.RAMDirectory;
051 import org.apache.lucene.store.jdbc.JdbcDirectory;
052 import org.apache.lucene.store.jdbc.JdbcStoreException;
053 import org.apache.lucene.store.jdbc.dialect.Dialect;
054 import org.apache.lucene.store.jdbc.lock.JdbcLock;
055 import org.apache.lucene.store.jdbc.support.JdbcTemplate;
056
057
062 public class IndexAccessorImpl implements IndexAccessor {
063
064 public IndexAccessorImpl(long companyId) {
065 _companyId = companyId;
066
067 _initDialect();
068 _checkLuceneDir();
069 _initIndexWriter();
070 _initCommitScheduler();
071 }
072
073 public void addDocument(Document document) throws IOException {
074 if (SearchEngineUtil.isIndexReadOnly()) {
075 return;
076 }
077
078 _write(null, document);
079 }
080
081 public void close() {
082 try {
083 _indexWriter.close();
084 }
085 catch(Exception e) {
086 _log.error(
087 "Closing Lucene writer failed for " + _companyId, e);
088 }
089 }
090
091 public void delete() {
092 if (SearchEngineUtil.isIndexReadOnly()) {
093 return;
094 }
095
096 close();
097
098 if (_log.isDebugEnabled()) {
099 _log.debug("Lucene store type " + PropsValues.LUCENE_STORE_TYPE);
100 }
101
102 if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_FILE)) {
103 _deleteFile();
104 }
105 else if (PropsValues.LUCENE_STORE_TYPE.equals(
106 _LUCENE_STORE_TYPE_JDBC)) {
107
108 _deleteJdbc();
109 }
110 else if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_RAM)) {
111 _deleteRam();
112 }
113 else {
114 throw new RuntimeException(
115 "Invalid store type " + PropsValues.LUCENE_STORE_TYPE);
116 }
117
118 _initIndexWriter();
119 }
120
121 public void deleteDocuments(Term term) throws IOException {
122 if (SearchEngineUtil.isIndexReadOnly()) {
123 return;
124 }
125
126 try {
127 _indexWriter.deleteDocuments(term);
128
129 _batchCount++;
130 }
131 finally {
132 _commit();
133 }
134 }
135
136 public long getCompanyId() {
137 return _companyId;
138 }
139
140 public Directory getLuceneDir() {
141 if (_log.isDebugEnabled()) {
142 _log.debug("Lucene store type " + PropsValues.LUCENE_STORE_TYPE);
143 }
144
145 if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_FILE)) {
146 return _getLuceneDirFile();
147 }
148 else if (PropsValues.LUCENE_STORE_TYPE.equals(
149 _LUCENE_STORE_TYPE_JDBC)) {
150
151 return _getLuceneDirJdbc();
152 }
153 else if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_RAM)) {
154 return _getLuceneDirRam();
155 }
156 else {
157 throw new RuntimeException(
158 "Invalid store type " + PropsValues.LUCENE_STORE_TYPE);
159 }
160 }
161
162 public void updateDocument(Term term, Document document)
163 throws IOException {
164
165 if (SearchEngineUtil.isIndexReadOnly()) {
166 return;
167 }
168
169 _write(term, document);
170 }
171
172 private void _checkLuceneDir() {
173 if (SearchEngineUtil.isIndexReadOnly()) {
174 return;
175 }
176
177 try {
178 Directory directory = getLuceneDir();
179
180 if (IndexWriter.isLocked(directory)) {
181 IndexWriter.unlock(directory);
182 }
183 }
184 catch (Exception e) {
185 _log.error("Check Lucene directory failed for " + _companyId, e);
186 }
187 }
188
189 private void _commit() throws IOException {
190 if ((PropsValues.LUCENE_COMMIT_BATCH_SIZE == 0) ||
191 (PropsValues.LUCENE_COMMIT_BATCH_SIZE <= _batchCount)) {
192
193 _doCommit();
194 }
195 }
196
197 private void _deleteFile() {
198 String path = _getPath();
199
200 try {
201 Directory directory = _getDirectory(path);
202
203 directory.close();
204 }
205 catch (Exception e) {
206 if (_log.isWarnEnabled()) {
207 _log.warn("Could not close directory " + path);
208 }
209 }
210
211 FileUtil.deltree(path);
212 }
213
214 private void _deleteJdbc() {
215 String tableName = _getTableName();
216
217 try {
218 Directory directory = _jdbcDirectories.remove(tableName);
219
220 if (directory != null) {
221 directory.close();
222 }
223 }
224 catch (Exception e) {
225 if (_log.isWarnEnabled()) {
226 _log.warn("Could not close directory " + tableName);
227 }
228 }
229
230 Connection con = null;
231 Statement s = null;
232
233 try {
234 con = DataAccess.getConnection();
235
236 s = con.createStatement();
237
238 s.executeUpdate("DELETE FROM " + tableName);
239 }
240 catch (Exception e) {
241 if (_log.isWarnEnabled()) {
242 _log.warn("Could not truncate " + tableName);
243 }
244 }
245 finally {
246 DataAccess.cleanUp(con, s);
247 }
248 }
249
250 private void _deleteRam() {
251 }
252
253 private void _doCommit() throws IOException {
254 if (_indexWriter != null) {
255 _indexWriter.commit();
256 }
257
258 _batchCount = 0;
259 }
260
261 private FSDirectory _getDirectory(String path) throws IOException {
262 return FSDirectory.open(new File(path));
263 }
264
265 private Directory _getLuceneDirFile() {
266 Directory directory = null;
267
268 String path = _getPath();
269
270 try {
271 directory = _getDirectory(path);
272 }
273 catch (IOException ioe1) {
274 if (directory != null) {
275 try {
276 directory.close();
277 }
278 catch (Exception e) {
279 }
280 }
281 }
282
283 return directory;
284 }
285
286 private Directory _getLuceneDirJdbc() {
287 JdbcDirectory jdbcDirectory = null;
288
289 Thread currentThread = Thread.currentThread();
290
291 ClassLoader contextClassLoader = currentThread.getContextClassLoader();
292
293 try {
294 currentThread.setContextClassLoader(
295 PortalClassLoaderUtil.getClassLoader());
296
297 String tableName = _getTableName();
298
299 jdbcDirectory = (JdbcDirectory)_jdbcDirectories.get(tableName);
300
301 if (jdbcDirectory != null) {
302 return jdbcDirectory;
303 }
304
305 try {
306 DataSource dataSource = InfrastructureUtil.getDataSource();
307
308 jdbcDirectory = new JdbcDirectory(
309 dataSource, _dialect, tableName);
310
311 _jdbcDirectories.put(tableName, jdbcDirectory);
312
313 if (!jdbcDirectory.tableExists()) {
314 jdbcDirectory.create();
315 }
316 }
317 catch (IOException ioe) {
318 throw new RuntimeException(ioe);
319 }
320 catch (UnsupportedOperationException uoe) {
321 if (_log.isWarnEnabled()) {
322 _log.warn(
323 "Database doesn't support the ability to check " +
324 "whether a table exists");
325 }
326
327 _manuallyCreateJdbcDirectory(jdbcDirectory, tableName);
328 }
329 }
330 finally {
331 currentThread.setContextClassLoader(contextClassLoader);
332 }
333
334 return jdbcDirectory;
335 }
336
337 private Directory _getLuceneDirRam() {
338 String path = _getPath();
339
340 Directory directory = _ramDirectories.get(path);
341
342 if (directory == null) {
343 directory = new RAMDirectory();
344
345 _ramDirectories.put(path, directory);
346 }
347
348 return directory;
349 }
350
351 private String _getPath() {
352 return PropsValues.LUCENE_DIR.concat(String.valueOf(_companyId)).concat(
353 StringPool.SLASH);
354 }
355
356 private String _getTableName() {
357 return _LUCENE_TABLE_PREFIX + _companyId;
358 }
359
360 private void _initCommitScheduler() {
361 if ((PropsValues.LUCENE_COMMIT_BATCH_SIZE <= 0) ||
362 (PropsValues.LUCENE_COMMIT_TIME_INTERVAL <= 0)) {
363
364 return;
365 }
366
367 ScheduledExecutorService scheduledExecutorService =
368 Executors.newSingleThreadScheduledExecutor();
369
370 Runnable runnable = new Runnable() {
371
372 public void run() {
373 try {
374 _doCommit();
375 }
376 catch (IOException ioe) {
377 _log.error("Could not run scheduled commit", ioe);
378 }
379 }
380
381 };
382
383 scheduledExecutorService.scheduleWithFixedDelay(
384 runnable, 0, PropsValues.LUCENE_COMMIT_TIME_INTERVAL,
385 TimeUnit.MILLISECONDS);
386 }
387
388 private void _initDialect() {
389 if (!PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_JDBC)) {
390 return;
391 }
392
393 Connection con = null;
394
395 try {
396 con = DataAccess.getConnection();
397
398 String url = con.getMetaData().getURL();
399
400 int x = url.indexOf(StringPool.COLON);
401 int y = url.indexOf(StringPool.COLON, x + 1);
402
403 String urlPrefix = url.substring(x + 1, y);
404
405 String dialectClass = PropsUtil.get(
406 PropsKeys.LUCENE_STORE_JDBC_DIALECT + urlPrefix);
407
408 if (dialectClass != null) {
409 if (_log.isDebugEnabled()) {
410 _log.debug("JDBC class implementation " + dialectClass);
411 }
412 }
413 else {
414 if (_log.isDebugEnabled()) {
415 _log.debug("JDBC class implementation is null");
416 }
417 }
418
419 if (dialectClass != null) {
420 _dialect = (Dialect)Class.forName(dialectClass).newInstance();
421 }
422 }
423 catch (Exception e) {
424 _log.error(e);
425 }
426 finally{
427 DataAccess.cleanUp(con);
428 }
429
430 if (_dialect == null) {
431 _log.error("No JDBC dialect found");
432 }
433 }
434
435 private void _initIndexWriter() {
436 try {
437 _indexWriter = new IndexWriter(
438 getLuceneDir(), LuceneHelperUtil.getAnalyzer(),
439 IndexWriter.MaxFieldLength.LIMITED);
440
441 _indexWriter.setMergeFactor(PropsValues.LUCENE_MERGE_FACTOR);
442 _indexWriter.setRAMBufferSizeMB(PropsValues.LUCENE_BUFFER_SIZE);
443 }
444 catch (Exception e) {
445 _log.error(
446 "Initializing Lucene writer failed for " + _companyId, e);
447 }
448 }
449
450 private void _manuallyCreateJdbcDirectory(
451 JdbcDirectory jdbcDirectory, String tableName) {
452
453
454
455 Connection con = null;
456 ResultSet rs = null;
457
458 try {
459 con = DataAccess.getConnection();
460
461
462
463 DatabaseMetaData metaData = con.getMetaData();
464
465 rs = metaData.getTables(null, null, tableName, null);
466
467 if (!rs.next()) {
468 JdbcTemplate jdbcTemplate = jdbcDirectory.getJdbcTemplate();
469
470 jdbcTemplate.executeUpdate(
471 jdbcDirectory.getTable().sqlCreate());
472
473 Class<?> lockClass = jdbcDirectory.getSettings().getLockClass();
474
475 JdbcLock jdbcLock = null;
476
477 try {
478 jdbcLock = (JdbcLock)lockClass.newInstance();
479 }
480 catch (Exception e) {
481 throw new JdbcStoreException(
482 "Could not create lock class " + lockClass);
483 }
484
485 jdbcLock.initializeDatabase(jdbcDirectory);
486 }
487 }
488 catch (Exception e) {
489 if (_log.isWarnEnabled()) {
490 _log.warn("Could not create " + tableName);
491 }
492 }
493 finally {
494 DataAccess.cleanUp(con, null, rs);
495 }
496 }
497
498 private void _write(Term term, Document document) throws IOException {
499 try {
500 if (term != null) {
501 _indexWriter.updateDocument(term, document);
502 }
503 else {
504 _indexWriter.addDocument(document);
505 }
506
507 _optimizeCount++;
508
509 if ((PropsValues.LUCENE_OPTIMIZE_INTERVAL == 0) ||
510 (_optimizeCount >= PropsValues.LUCENE_OPTIMIZE_INTERVAL)) {
511
512 _indexWriter.optimize();
513
514 _optimizeCount = 0;
515 }
516
517 _batchCount++;
518 }
519 finally {
520 _commit();
521 }
522 }
523
524 private static final String _LUCENE_STORE_TYPE_FILE = "file";
525
526 private static final String _LUCENE_STORE_TYPE_JDBC = "jdbc";
527
528 private static final String _LUCENE_STORE_TYPE_RAM = "ram";
529
530 private static final String _LUCENE_TABLE_PREFIX = "LUCENE_";
531
532 private static Log _log = LogFactoryUtil.getLog(IndexAccessorImpl.class);
533
534 private int _batchCount;
535 private long _companyId;
536 private Dialect _dialect;
537 private IndexWriter _indexWriter;
538 private Map<String, Directory> _jdbcDirectories =
539 new ConcurrentHashMap<String, Directory>();
540 private int _optimizeCount;
541 private Map<String, Directory> _ramDirectories =
542 new ConcurrentHashMap<String, Directory>();
543
544 }