1   /**
2    * Copyright (c) 2000-2009 Liferay, Inc. All rights reserved.
3    *
4    * Permission is hereby granted, free of charge, to any person obtaining a copy
5    * of this software and associated documentation files (the "Software"), to deal
6    * in the Software without restriction, including without limitation the rights
7    * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8    * copies of the Software, and to permit persons to whom the Software is
9    * furnished to do so, subject to the following conditions:
10   *
11   * The above copyright notice and this permission notice shall be included in
12   * all copies or substantial portions of the Software.
13   *
14   * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15   * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16   * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17   * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18   * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19   * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
20   * SOFTWARE.
21   */
22  
23  package com.liferay.portal.search.lucene;
24  
25  import com.liferay.portal.kernel.dao.jdbc.DataAccess;
26  import com.liferay.portal.kernel.log.Log;
27  import com.liferay.portal.kernel.log.LogFactoryUtil;
28  import com.liferay.portal.kernel.util.FileUtil;
29  import com.liferay.portal.kernel.util.InfrastructureUtil;
30  import com.liferay.portal.kernel.util.PortalClassLoaderUtil;
31  import com.liferay.portal.kernel.util.StringPool;
32  import com.liferay.portal.kernel.util.StringUtil;
33  import com.liferay.portal.kernel.util.Validator;
34  import com.liferay.portal.util.PropsKeys;
35  import com.liferay.portal.util.PropsUtil;
36  import com.liferay.portal.util.PropsValues;
37  import com.liferay.util.lucene.KeywordsUtil;
38  
39  import java.io.IOException;
40  
41  import java.sql.Connection;
42  import java.sql.DatabaseMetaData;
43  import java.sql.ResultSet;
44  import java.sql.Statement;
45  
46  import java.util.Date;
47  import java.util.Map;
48  import java.util.concurrent.ConcurrentHashMap;
49  
50  import javax.sql.DataSource;
51  
52  import org.apache.lucene.analysis.Analyzer;
53  import org.apache.lucene.analysis.WhitespaceAnalyzer;
54  import org.apache.lucene.document.Document;
55  import org.apache.lucene.index.IndexReader;
56  import org.apache.lucene.index.IndexWriter;
57  import org.apache.lucene.index.Term;
58  import org.apache.lucene.queryParser.ParseException;
59  import org.apache.lucene.queryParser.QueryParser;
60  import org.apache.lucene.search.BooleanClause;
61  import org.apache.lucene.search.BooleanQuery;
62  import org.apache.lucene.search.IndexSearcher;
63  import org.apache.lucene.search.Query;
64  import org.apache.lucene.search.TermQuery;
65  import org.apache.lucene.search.WildcardQuery;
66  import org.apache.lucene.store.Directory;
67  import org.apache.lucene.store.FSDirectory;
68  import org.apache.lucene.store.RAMDirectory;
69  import org.apache.lucene.store.jdbc.JdbcDirectory;
70  import org.apache.lucene.store.jdbc.JdbcStoreException;
71  import org.apache.lucene.store.jdbc.dialect.Dialect;
72  import org.apache.lucene.store.jdbc.lock.JdbcLock;
73  import org.apache.lucene.store.jdbc.support.JdbcTemplate;
74  
75  /**
76   * <a href="LuceneUtil.java.html"><b><i>View Source</i></b></a>
77   *
78   * @author Brian Wing Shun Chan
79   * @author Harry Mark
80   *
81   */
82  public class LuceneUtil {
83  
84      public static void acquireLock(long companyId) {
85          try {
86              _instance._sharedWriter.acquireLock(companyId, true);
87          }
88          catch (InterruptedException ie) {
89              _log.error(ie);
90          }
91      }
92  
93      public static void addDate(Document doc, String field, Date value) {
94          doc.add(LuceneFields.getDate(field, value));
95      }
96  
97      public static void addExactTerm(
98          BooleanQuery booleanQuery, String field, boolean value) {
99  
100         addExactTerm(booleanQuery, field, String.valueOf(value));
101     }
102 
103     public static void addExactTerm(
104         BooleanQuery booleanQuery, String field, Boolean value) {
105 
106         addExactTerm(booleanQuery, field, String.valueOf(value));
107     }
108 
109     public static void addExactTerm(
110         BooleanQuery booleanQuery, String field, double value) {
111 
112         addExactTerm(booleanQuery, field, String.valueOf(value));
113     }
114 
115     public static void addExactTerm(
116         BooleanQuery booleanQuery, String field, Double value) {
117 
118         addExactTerm(booleanQuery, field, String.valueOf(value));
119     }
120 
121     public static void addExactTerm(
122         BooleanQuery booleanQuery, String field, int value) {
123 
124         addExactTerm(booleanQuery, field, String.valueOf(value));
125     }
126 
127     public static void addExactTerm(
128         BooleanQuery booleanQuery, String field, Integer value) {
129 
130         addExactTerm(booleanQuery, field, String.valueOf(value));
131     }
132 
133     public static void addExactTerm(
134         BooleanQuery booleanQuery, String field, long value) {
135 
136         addExactTerm(booleanQuery, field, String.valueOf(value));
137     }
138 
139     public static void addExactTerm(
140         BooleanQuery booleanQuery, String field, Long value) {
141 
142         addExactTerm(booleanQuery, field, String.valueOf(value));
143     }
144 
145     public static void addExactTerm(
146         BooleanQuery booleanQuery, String field, short value) {
147 
148         addExactTerm(booleanQuery, field, String.valueOf(value));
149     }
150 
151     public static void addExactTerm(
152         BooleanQuery booleanQuery, String field, Short value) {
153 
154         addExactTerm(booleanQuery, field, String.valueOf(value));
155     }
156 
157     public static void addExactTerm(
158         BooleanQuery booleanQuery, String field, String value) {
159 
160         //text = KeywordsUtil.escape(value);
161 
162         Query query = new TermQuery(new Term(field, value));
163 
164         booleanQuery.add(query, BooleanClause.Occur.SHOULD);
165     }
166 
167     public static void addRequiredTerm(
168         BooleanQuery booleanQuery, String field, boolean value) {
169 
170         addRequiredTerm(booleanQuery, field, String.valueOf(value));
171     }
172 
173     public static void addRequiredTerm(
174         BooleanQuery booleanQuery, String field, Boolean value) {
175 
176         addRequiredTerm(booleanQuery, field, String.valueOf(value));
177     }
178 
179     public static void addRequiredTerm(
180         BooleanQuery booleanQuery, String field, double value) {
181 
182         addRequiredTerm(booleanQuery, field, String.valueOf(value));
183     }
184 
185     public static void addRequiredTerm(
186         BooleanQuery booleanQuery, String field, Double value) {
187 
188         addRequiredTerm(booleanQuery, field, String.valueOf(value));
189     }
190 
191     public static void addRequiredTerm(
192         BooleanQuery booleanQuery, String field, int value) {
193 
194         addRequiredTerm(booleanQuery, field, String.valueOf(value));
195     }
196 
197     public static void addRequiredTerm(
198         BooleanQuery booleanQuery, String field, Integer value) {
199 
200         addRequiredTerm(booleanQuery, field, String.valueOf(value));
201     }
202 
203     public static void addRequiredTerm(
204         BooleanQuery booleanQuery, String field, long value) {
205 
206         addRequiredTerm(booleanQuery, field, String.valueOf(value));
207     }
208 
209     public static void addRequiredTerm(
210         BooleanQuery booleanQuery, String field, Long value) {
211 
212         addRequiredTerm(booleanQuery, field, String.valueOf(value));
213     }
214 
215     public static void addRequiredTerm(
216         BooleanQuery booleanQuery, String field, short value) {
217 
218         addRequiredTerm(booleanQuery, field, String.valueOf(value));
219     }
220 
221     public static void addRequiredTerm(
222         BooleanQuery booleanQuery, String field, Short value) {
223 
224         addRequiredTerm(booleanQuery, field, String.valueOf(value));
225     }
226 
227     public static void addRequiredTerm(
228         BooleanQuery booleanQuery, String field, String value) {
229 
230         addRequiredTerm(booleanQuery, field, value, false);
231     }
232 
233     public static void addRequiredTerm(
234         BooleanQuery booleanQuery, String field, String value, boolean like) {
235 
236         if (like) {
237             value = StringUtil.replace(
238                 value, StringPool.PERCENT, StringPool.STAR);
239 
240             value = value.toLowerCase();
241 
242             WildcardQuery wildcardQuery = new WildcardQuery(
243                 new Term(field, value));
244 
245             booleanQuery.add(wildcardQuery, BooleanClause.Occur.MUST);
246         }
247         else {
248             //text = KeywordsUtil.escape(value);
249 
250             Term term = new Term(field, value);
251             TermQuery termQuery = new TermQuery(term);
252 
253             booleanQuery.add(termQuery, BooleanClause.Occur.MUST);
254         }
255     }
256 
257     public static void addTerm(
258             BooleanQuery booleanQuery, String field, long value)
259         throws ParseException {
260 
261         addTerm(booleanQuery, field, String.valueOf(value));
262     }
263 
264     public static void addTerm(
265             BooleanQuery booleanQuery, String field, String value)
266         throws ParseException {
267 
268         addTerm(booleanQuery, field, value, false);
269     }
270 
271     public static void addTerm(
272             BooleanQuery booleanQuery, String field, String value,
273             boolean like)
274         throws ParseException {
275 
276         if (Validator.isNull(value)) {
277             return;
278         }
279 
280         if (like) {
281             value = value.toLowerCase();
282 
283             StringBuilder sb = new StringBuilder();
284 
285             sb.append(StringPool.STAR);
286             sb.append(value);
287             sb.append(StringPool.STAR);
288 
289             WildcardQuery wildcardQuery = new WildcardQuery(
290                 new Term(field, sb.toString()));
291 
292             booleanQuery.add(wildcardQuery, BooleanClause.Occur.SHOULD);
293         }
294         else {
295             QueryParser queryParser = new QueryParser(
296                 field, LuceneUtil.getAnalyzer());
297 
298             try {
299                 Query query = queryParser.parse(value);
300 
301                 booleanQuery.add(query, BooleanClause.Occur.SHOULD);
302             }
303             catch (ParseException pe) {
304                 if (_log.isDebugEnabled()) {
305                     _log.debug(
306                         "ParseException thrown, reverting to literal search",
307                         pe);
308                 }
309 
310                 value = KeywordsUtil.escape(value);
311 
312                 Query query = queryParser.parse(value);
313 
314                 booleanQuery.add(query, BooleanClause.Occur.SHOULD);
315             }
316         }
317     }
318 
319     public static void checkLuceneDir(long companyId) {
320         if (PropsValues.INDEX_READ_ONLY) {
321             return;
322         }
323 
324         Directory luceneDir = LuceneUtil.getLuceneDir(companyId);
325 
326         try {
327 
328             // LEP-6078
329 
330             if (luceneDir.fileExists("write.lock")) {
331                 luceneDir.deleteFile("write.lock");
332             }
333         }
334         catch (IOException ioe) {
335             _log.error("Unable to clear write lock", ioe);
336         }
337 
338         IndexWriter writer = null;
339 
340         // Lucene does not properly release its lock on the index when
341         // IndexWriter throws an exception
342 
343         try {
344             if (luceneDir.fileExists("segments.gen")) {
345                 writer = new IndexWriter(
346                     luceneDir, LuceneUtil.getAnalyzer(), false);
347             }
348             else {
349                 writer = new IndexWriter(
350                     luceneDir, LuceneUtil.getAnalyzer(), true);
351             }
352         }
353         catch (IOException ioe) {
354             _log.error("Check Lucene directory failed for " + companyId, ioe);
355         }
356         finally {
357             if (writer != null) {
358                 try {
359                     writer.close();
360                 }
361                 catch (IOException ioe) {
362                     _log.error(ioe);
363                 }
364             }
365         }
366     }
367 
368     public static void delete(long companyId) {
369         _instance._delete(companyId);
370     }
371 
372     public static void deleteDocuments(long companyId, Term term)
373         throws IOException {
374 
375         try {
376             _instance._sharedWriter.deleteDocuments(companyId, term);
377         }
378         catch (InterruptedException ie) {
379             _log.error(ie);
380         }
381     }
382 
383     public static Analyzer getAnalyzer() {
384         return _instance._getAnalyzer();
385     }
386 
387     public static FSDirectory getDirectory(String path, boolean create)
388         throws IOException {
389 
390         return FSDirectory.getDirectory(path, false);
391     }
392 
393     public static Directory getLuceneDir(long companyId) {
394         return _instance._getLuceneDir(companyId);
395     }
396 
397     public static IndexReader getReader(long companyId) throws IOException {
398         return IndexReader.open(getLuceneDir(companyId));
399     }
400 
401     public static IndexSearcher getSearcher(long companyId)
402         throws IOException {
403 
404         return new IndexSearcher(getLuceneDir(companyId));
405     }
406 
407     public static IndexWriter getWriter(long companyId) throws IOException {
408         return getWriter(companyId, false);
409     }
410 
411     public static IndexWriter getWriter(long companyId, boolean create)
412         throws IOException {
413 
414         return _instance._sharedWriter.getWriter(companyId, create);
415     }
416 
417     public static void releaseLock(long companyId) {
418         _instance._sharedWriter.releaseLock(companyId);
419     }
420 
421     public static void write(long companyId) {
422         _instance._sharedWriter.write(companyId);
423     }
424 
425     public static void write(IndexWriter writer) throws IOException {
426         _instance._sharedWriter.write(writer);
427     }
428 
429     private LuceneUtil() {
430         String analyzerName = PropsUtil.get(PropsKeys.LUCENE_ANALYZER);
431 
432         if (Validator.isNotNull(analyzerName)) {
433             try {
434                 _analyzerClass = Class.forName(analyzerName);
435             }
436             catch (Exception e) {
437                 _log.error(e);
438             }
439         }
440 
441         // Dialect
442 
443         if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_JDBC)) {
444             Connection con = null;
445 
446             try {
447                 con = DataAccess.getConnection();
448 
449                 String url = con.getMetaData().getURL();
450 
451                 int x = url.indexOf(":");
452                 int y = url.indexOf(":", x + 1);
453 
454                 String urlPrefix = url.substring(x + 1, y);
455 
456                 String dialectClass = PropsUtil.get(
457                     PropsKeys.LUCENE_STORE_JDBC_DIALECT + urlPrefix);
458 
459                 if (dialectClass != null) {
460                     if (_log.isDebugEnabled()) {
461                         _log.debug("JDBC class implementation " + dialectClass);
462                     }
463                 }
464                 else {
465                     if (_log.isDebugEnabled()) {
466                         _log.debug("JDBC class implementation is null");
467                     }
468                 }
469 
470                 if (dialectClass != null) {
471                     _dialect =
472                         (Dialect)Class.forName(dialectClass).newInstance();
473                 }
474             }
475             catch (Exception e) {
476                 _log.error(e);
477             }
478             finally{
479                 DataAccess.cleanUp(con);
480             }
481 
482             if (_dialect == null) {
483                 _log.error("No JDBC dialect found");
484             }
485         }
486     }
487 
488     public void _delete(long companyId) {
489         if (PropsValues.INDEX_READ_ONLY) {
490             return;
491         }
492 
493         if (_log.isDebugEnabled()) {
494             _log.debug("Lucene store type " + PropsValues.LUCENE_STORE_TYPE);
495         }
496 
497         if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_FILE)) {
498             _deleteFile(companyId);
499         }
500         else if (PropsValues.LUCENE_STORE_TYPE.equals(
501                     _LUCENE_STORE_TYPE_JDBC)) {
502 
503             _deleteJdbc(companyId);
504         }
505         else if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_RAM)) {
506             _deleteRam(companyId);
507         }
508         else {
509             throw new RuntimeException(
510                 "Invalid store type " + PropsValues.LUCENE_STORE_TYPE);
511         }
512     }
513 
514     private void _deleteFile(long companyId) {
515         String path = _getPath(companyId);
516 
517         try {
518             Directory directory = getDirectory(path, false);
519 
520             directory.close();
521         }
522         catch (Exception e) {
523             if (_log.isWarnEnabled()) {
524                 _log.warn("Could not close directory " + path);
525             }
526         }
527 
528         FileUtil.deltree(path);
529     }
530 
531     private void _deleteJdbc(long companyId) {
532         String tableName = _getTableName(companyId);
533 
534         try {
535             Directory directory = _jdbcDirectories.remove(tableName);
536 
537             if (directory != null) {
538                 directory.close();
539             }
540         }
541         catch (Exception e) {
542             if (_log.isWarnEnabled()) {
543                 _log.warn("Could not close directory " + tableName);
544             }
545         }
546 
547         Connection con = null;
548         Statement s = null;
549 
550         try {
551             con = DataAccess.getConnection();
552 
553             s = con.createStatement();
554 
555             s.executeUpdate("DELETE FROM " + tableName);
556         }
557         catch (Exception e) {
558             if (_log.isWarnEnabled()) {
559                 _log.warn("Could not truncate " + tableName);
560             }
561         }
562         finally {
563             DataAccess.cleanUp(con, s);
564         }
565     }
566 
567     private void _deleteRam(long companyId) {
568     }
569 
570     private Analyzer _getAnalyzer() {
571         try {
572             return (Analyzer)_analyzerClass.newInstance();
573         }
574         catch (Exception e) {
575             throw new RuntimeException(e);
576         }
577     }
578 
579     private Directory _getLuceneDir(long companyId) {
580         if (_log.isDebugEnabled()) {
581             _log.debug("Lucene store type " + PropsValues.LUCENE_STORE_TYPE);
582         }
583 
584         if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_FILE)) {
585             return _getLuceneDirFile(companyId);
586         }
587         else if (PropsValues.LUCENE_STORE_TYPE.equals(
588                     _LUCENE_STORE_TYPE_JDBC)) {
589 
590             return _getLuceneDirJdbc(companyId);
591         }
592         else if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_RAM)) {
593             return _getLuceneDirRam(companyId);
594         }
595         else {
596             throw new RuntimeException(
597                 "Invalid store type " + PropsValues.LUCENE_STORE_TYPE);
598         }
599     }
600 
601     private Directory _getLuceneDirFile(long companyId) {
602         Directory directory = null;
603 
604         String path = _getPath(companyId);
605 
606         try {
607             directory = getDirectory(path, false);
608         }
609         catch (IOException ioe1) {
610             try {
611                 if (directory != null) {
612                     directory.close();
613                 }
614 
615                 directory = getDirectory(path, true);
616             }
617             catch (IOException ioe2) {
618                 throw new RuntimeException(ioe2);
619             }
620         }
621 
622         return directory;
623     }
624 
625     private Directory _getLuceneDirJdbc(long companyId) {
626         JdbcDirectory directory = null;
627 
628         Thread currentThread = Thread.currentThread();
629 
630         ClassLoader contextClassLoader = currentThread.getContextClassLoader();
631 
632         try {
633             currentThread.setContextClassLoader(
634                 PortalClassLoaderUtil.getClassLoader());
635 
636             String tableName = _getTableName(companyId);
637 
638             directory = (JdbcDirectory)_jdbcDirectories.get(tableName);
639 
640             if (directory != null) {
641                 return directory;
642             }
643 
644             try {
645                 DataSource ds = InfrastructureUtil.getDataSource();
646 
647                 directory = new JdbcDirectory(ds, _dialect, tableName);
648 
649                 _jdbcDirectories.put(tableName, directory);
650 
651                 if (!directory.tableExists()) {
652                     directory.create();
653                 }
654             }
655             catch (IOException ioe) {
656                 throw new RuntimeException(ioe);
657             }
658             catch (UnsupportedOperationException uoe) {
659                 if (_log.isWarnEnabled()) {
660                     _log.warn(
661                         "Database doesn't support the ability to check " +
662                             "whether a table exists");
663                 }
664 
665                 _manuallyCreateJdbcDirectory(directory, tableName);
666             }
667         }
668         finally {
669             currentThread.setContextClassLoader(contextClassLoader);
670         }
671 
672         return directory;
673     }
674 
675     private Directory _getLuceneDirRam(long companyId) {
676         String path = _getPath(companyId);
677 
678         Directory directory = _ramDirectories.get(path);
679 
680         if (directory == null) {
681             directory = new RAMDirectory();
682 
683             _ramDirectories.put(path, directory);
684         }
685 
686         return directory;
687     }
688 
689     private String _getPath(long companyId) {
690         StringBuilder sb = new StringBuilder();
691 
692         sb.append(PropsValues.LUCENE_DIR);
693         sb.append(companyId);
694         sb.append(StringPool.SLASH);
695 
696         return sb.toString();
697     }
698 
699     private String _getTableName(long companyId) {
700         return _LUCENE_TABLE_PREFIX + companyId;
701     }
702 
703     private void _manuallyCreateJdbcDirectory(
704         JdbcDirectory directory, String tableName) {
705 
706         // LEP-2181
707 
708         Connection con = null;
709         ResultSet rs = null;
710 
711         try {
712             con = DataAccess.getConnection();
713 
714             // Check if table exists
715 
716             DatabaseMetaData metaData = con.getMetaData();
717 
718             rs = metaData.getTables(null, null, tableName, null);
719 
720             if (!rs.next()) {
721                 JdbcTemplate jdbcTemplate = directory.getJdbcTemplate();
722 
723                 jdbcTemplate.executeUpdate(directory.getTable().sqlCreate());
724 
725                 Class<?> lockClass = directory.getSettings().getLockClass();
726 
727                 JdbcLock jdbcLock = null;
728 
729                 try {
730                     jdbcLock = (JdbcLock)lockClass.newInstance();
731                 }
732                 catch (Exception e) {
733                     throw new JdbcStoreException(
734                         "Failed to create lock class " + lockClass);
735                 }
736 
737                 jdbcLock.initializeDatabase(directory);
738             }
739         }
740         catch (Exception e) {
741             if (_log.isWarnEnabled()) {
742                 _log.warn("Could not create " + tableName);
743             }
744         }
745         finally {
746             DataAccess.cleanUp(con, null, rs);
747         }
748     }
749 
750     private static final String _LUCENE_STORE_TYPE_FILE = "file";
751 
752     private static final String _LUCENE_STORE_TYPE_JDBC = "jdbc";
753 
754     private static final String _LUCENE_STORE_TYPE_RAM = "ram";
755 
756     private static final String _LUCENE_TABLE_PREFIX = "LUCENE_";
757 
758     private static Log _log = LogFactoryUtil.getLog(LuceneUtil.class);
759 
760     private static LuceneUtil _instance = new LuceneUtil();
761 
762     private IndexWriterFactory _sharedWriter = new IndexWriterFactory();
763     private Class<?> _analyzerClass = WhitespaceAnalyzer.class;
764     private Dialect _dialect;
765     private Map<String, Directory> _jdbcDirectories =
766         new ConcurrentHashMap<String, Directory>();
767     private Map<String, Directory> _ramDirectories =
768         new ConcurrentHashMap<String, Directory>();
769 
770 }