001    /**
002     * Copyright (c) 2000-2013 Liferay, Inc. All rights reserved.
003     *
004     * This library is free software; you can redistribute it and/or modify it under
005     * the terms of the GNU Lesser General Public License as published by the Free
006     * Software Foundation; either version 2.1 of the License, or (at your option)
007     * any later version.
008     *
009     * This library is distributed in the hope that it will be useful, but WITHOUT
010     * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
011     * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
012     * details.
013     */
014    
015    package com.liferay.portlet.documentlibrary.store;
016    
017    import com.liferay.portal.kernel.exception.PortalException;
018    import com.liferay.portal.kernel.exception.SystemException;
019    import com.liferay.portal.kernel.log.Log;
020    import com.liferay.portal.kernel.log.LogFactoryUtil;
021    import com.liferay.portal.kernel.util.CharPool;
022    import com.liferay.portal.kernel.util.DateUtil;
023    import com.liferay.portal.kernel.util.FileUtil;
024    import com.liferay.portal.kernel.util.LocaleUtil;
025    import com.liferay.portal.kernel.util.PropsKeys;
026    import com.liferay.portal.kernel.util.StreamUtil;
027    import com.liferay.portal.kernel.util.StringBundler;
028    import com.liferay.portal.kernel.util.StringPool;
029    import com.liferay.portal.kernel.util.SystemProperties;
030    import com.liferay.portal.kernel.util.Time;
031    import com.liferay.portal.kernel.util.Validator;
032    import com.liferay.portal.kernel.uuid.PortalUUIDUtil;
033    import com.liferay.portal.util.PropsUtil;
034    import com.liferay.portal.util.PropsValues;
035    import com.liferay.portlet.documentlibrary.NoSuchFileException;
036    
037    import java.io.File;
038    import java.io.FileInputStream;
039    import java.io.FileOutputStream;
040    import java.io.IOException;
041    import java.io.InputStream;
042    import java.io.OutputStream;
043    
044    import java.util.ArrayList;
045    import java.util.Arrays;
046    import java.util.Date;
047    import java.util.List;
048    
049    import org.jets3t.service.S3Service;
050    import org.jets3t.service.S3ServiceException;
051    import org.jets3t.service.ServiceException;
052    import org.jets3t.service.impl.rest.httpclient.RestS3Service;
053    import org.jets3t.service.model.S3Bucket;
054    import org.jets3t.service.model.S3Object;
055    import org.jets3t.service.model.StorageObject;
056    import org.jets3t.service.security.AWSCredentials;
057    
058    /**
059     * @author Brian Wing Shun Chan
060     * @author Sten Martinez
061     * @author Edward Han
062     * @author Vilmos Papp
063     * @author Mate Thurzo
064     */
065    public class S3Store extends BaseStore {
066    
067            public S3Store() {
068                    try {
069                            _s3Service = getS3Service();
070                            _s3Bucket = getS3Bucket();
071                    }
072                    catch (S3ServiceException s3se) {
073                            _log.error(s3se.getMessage());
074                    }
075            }
076    
077            @Override
078            public void addDirectory(
079                    long companyId, long repositoryId, String dirName) {
080            }
081    
082            @Override
083            public void addFile(
084                            long companyId, long repositoryId, String fileName, InputStream is)
085                    throws SystemException {
086    
087                    try {
088                            S3Object s3Object = new S3Object(
089                                    _s3Bucket,
090                                    getKey(companyId, repositoryId, fileName, VERSION_DEFAULT));
091    
092                            s3Object.setDataInputStream(is);
093    
094                            _s3Service.putObject(_s3Bucket, s3Object);
095    
096                            is.close();
097                    }
098                    catch (IOException ioe) {
099                            throw new SystemException(ioe);
100                    }
101                    catch (S3ServiceException s3se) {
102                            throw new SystemException(s3se);
103                    }
104            }
105    
106            @Override
107            public void checkRoot(long companyId) {
108            }
109    
110            @Override
111            public void deleteDirectory(
112                            long companyId, long repositoryId, String dirName)
113                    throws SystemException {
114    
115                    try {
116                            S3Object[] s3Objects = _s3Service.listObjects(
117                                    _s3Bucket.getName(), getKey(companyId, repositoryId, dirName),
118                                    null);
119    
120                            for (S3Object s3Object : s3Objects) {
121                                    _s3Service.deleteObject(_s3Bucket, s3Object.getKey());
122                            }
123                    }
124                    catch (S3ServiceException s3se) {
125                            throw new SystemException(s3se);
126                    }
127            }
128    
129            @Override
130            public void deleteFile(long companyId, long repositoryId, String fileName)
131                    throws SystemException {
132    
133                    try {
134                            S3Object[] s3Objects = _s3Service.listObjects(
135                                    _s3Bucket.getName(), getKey(companyId, repositoryId, fileName),
136                                    null);
137    
138                            for (S3Object s3Object : s3Objects) {
139                                    _s3Service.deleteObject(_s3Bucket, s3Object.getKey());
140                            }
141                    }
142                    catch (S3ServiceException s3se) {
143                            throw new SystemException(s3se);
144                    }
145            }
146    
147            @Override
148            public void deleteFile(
149                            long companyId, long repositoryId, String fileName,
150                            String versionLabel)
151                    throws SystemException {
152    
153                    try {
154                            _s3Service.deleteObject(
155                                    _s3Bucket,
156                                    getKey(companyId, repositoryId, fileName, versionLabel));
157                    }
158                    catch (S3ServiceException s3se) {
159                            throw new SystemException(s3se);
160                    }
161            }
162    
163            @Override
164            public File getFile(
165                            long companyId, long repositoryId, String fileName,
166                            String versionLabel)
167                    throws PortalException, SystemException {
168    
169                    try {
170                            if (Validator.isNull(versionLabel)) {
171                                    versionLabel = getHeadVersionLabel(
172                                            companyId, repositoryId, fileName);
173                            }
174    
175                            S3Object s3Object = _s3Service.getObject(
176                                    _s3Bucket.getName(),
177                                    getKey(companyId, repositoryId, fileName, versionLabel));
178    
179                            File tempFile = getTempFile(s3Object, fileName);
180    
181                            cleanUpTempFiles();
182    
183                            return tempFile;
184                    }
185                    catch (IOException ioe) {
186                            throw new SystemException(ioe);
187                    }
188                    catch (ServiceException se) {
189                            throw new SystemException(se);
190                    }
191            }
192    
193            @Override
194            public InputStream getFileAsStream(
195                            long companyId, long repositoryId, String fileName,
196                            String versionLabel)
197                    throws PortalException, SystemException {
198    
199                    try {
200                            if (Validator.isNull(versionLabel)) {
201                                    versionLabel = getHeadVersionLabel(
202                                            companyId, repositoryId, fileName);
203                            }
204    
205                            S3Object s3Object = _s3Service.getObject(
206                                    _s3Bucket.getName(),
207                                    getKey(companyId, repositoryId, fileName, versionLabel));
208    
209                            return s3Object.getDataInputStream();
210                    }
211                    catch (ServiceException se) {
212                            throw new SystemException(se);
213                    }
214            }
215    
216            @Override
217            public String[] getFileNames(long companyId, long repositoryId)
218                    throws SystemException {
219    
220                    try {
221                            S3Object[] s3Objects = _s3Service.listObjects(
222                                    _s3Bucket.getName(), getKey(companyId, repositoryId), null);
223    
224                            return getFileNames(s3Objects);
225                    }
226                    catch (S3ServiceException s3se) {
227                            throw new SystemException(s3se);
228                    }
229            }
230    
231            @Override
232            public String[] getFileNames(
233                            long companyId, long repositoryId, String dirName)
234                    throws SystemException {
235    
236                    try {
237                            S3Object[] s3Objects = _s3Service.listObjects(
238                                    _s3Bucket.getName(), getKey(companyId, repositoryId, dirName),
239                                    null);
240    
241                            return getFileNames(s3Objects);
242                    }
243                    catch (S3ServiceException s3se) {
244                            throw new SystemException(s3se);
245                    }
246            }
247    
248            @Override
249            public long getFileSize(long companyId, long repositoryId, String fileName)
250                    throws PortalException, SystemException {
251    
252                    try {
253                            String versionLabel = getHeadVersionLabel(
254                                    companyId, repositoryId, fileName);
255    
256                            StorageObject storageObject = _s3Service.getObjectDetails(
257                                    _s3Bucket.getName(),
258                                    getKey(companyId, repositoryId, fileName, versionLabel));
259    
260                            return storageObject.getContentLength();
261                    }
262                    catch (ServiceException se) {
263                            throw new SystemException(se);
264                    }
265            }
266    
267            @Override
268            public boolean hasDirectory(
269                    long companyId, long repositoryId, String dirName) {
270    
271                    return true;
272            }
273    
274            @Override
275            public boolean hasFile(
276                            long companyId, long repositoryId, String fileName,
277                            String versionLabel)
278                    throws SystemException {
279    
280                    try {
281                            S3Object[] s3Objects = _s3Service.listObjects(
282                                    _s3Bucket.getName(),
283                                    getKey(companyId, repositoryId, fileName, versionLabel), null);
284    
285                            if (s3Objects.length == 0) {
286                                    return false;
287                            }
288                            else {
289                                    return true;
290                            }
291                    }
292                    catch (S3ServiceException s3se) {
293                            throw new SystemException(s3se);
294                    }
295            }
296    
297            @Override
298            public void move(String srcDir, String destDir) {
299            }
300    
301            @Override
302            public void updateFile(
303                            long companyId, long repositoryId, long newRepositoryId,
304                            String fileName)
305                    throws SystemException {
306    
307                    try {
308                            S3Object[] s3Objects = _s3Service.listObjects(
309                                    _s3Bucket.getName(), getKey(companyId, repositoryId, fileName),
310                                    null);
311    
312                            for (S3Object oldS3Object : s3Objects) {
313                                    String oldKey = oldS3Object.getKey();
314    
315                                    oldS3Object = _s3Service.getObject(_s3Bucket.getName(), oldKey);
316    
317                                    File tempFile = new File(
318                                            SystemProperties.get(SystemProperties.TMP_DIR) +
319                                                    File.separator + PortalUUIDUtil.generate());
320    
321                                    FileUtil.write(tempFile, oldS3Object.getDataInputStream());
322    
323                                    InputStream is = new FileInputStream(tempFile);
324    
325                                    String newPrefix = getKey(companyId, newRepositoryId);
326    
327                                    int x = oldKey.indexOf(CharPool.SLASH);
328    
329                                    x = oldKey.indexOf(CharPool.SLASH, x + 1);
330    
331                                    String newKey = newPrefix + oldKey.substring(x + 1);
332    
333                                    S3Object newS3Object = new S3Object(_s3Bucket, newKey);
334    
335                                    newS3Object.setDataInputStream(is);
336    
337                                    _s3Service.putObject(_s3Bucket, newS3Object);
338                                    _s3Service.deleteObject(_s3Bucket, oldKey);
339    
340                                    is.close();
341    
342                                    FileUtil.delete(tempFile);
343                            }
344                    }
345                    catch (IOException ioe) {
346                            throw new SystemException(ioe);
347                    }
348                    catch (ServiceException se) {
349                            throw new SystemException(se);
350                    }
351            }
352    
353            @Override
354            public void updateFile(
355                            long companyId, long repositoryId, String fileName,
356                            String newFileName)
357                    throws SystemException {
358    
359                    try {
360                            S3Object[] s3Objects = _s3Service.listObjects(
361                                    _s3Bucket.getName(), getKey(companyId, repositoryId, fileName),
362                                    null);
363    
364                            for (S3Object oldS3Object : s3Objects) {
365                                    String oldKey = oldS3Object.getKey();
366    
367                                    oldS3Object = _s3Service.getObject(_s3Bucket.getName(), oldKey);
368    
369                                    File tempFile = new File(
370                                            SystemProperties.get(SystemProperties.TMP_DIR) +
371                                                    File.separator + PortalUUIDUtil.generate());
372    
373                                    FileUtil.write(tempFile, oldS3Object.getDataInputStream());
374    
375                                    oldS3Object.closeDataInputStream();
376    
377                                    InputStream is = new FileInputStream(tempFile);
378    
379                                    String newPrefix = getKey(companyId, repositoryId, newFileName);
380    
381                                    int x = oldKey.indexOf(StringPool.SLASH);
382    
383                                    x = oldKey.indexOf(CharPool.SLASH, x + 1);
384                                    x = oldKey.indexOf(CharPool.SLASH, x + 1);
385    
386                                    String newKey = newPrefix + oldKey.substring(x + 1);
387    
388                                    S3Object newS3Object = new S3Object(_s3Bucket, newKey);
389    
390                                    newS3Object.setDataInputStream(is);
391    
392                                    _s3Service.putObject(_s3Bucket, newS3Object);
393                                    _s3Service.deleteObject(_s3Bucket, oldKey);
394    
395                                    newS3Object.closeDataInputStream();
396    
397                                    is.close();
398    
399                                    FileUtil.delete(tempFile);
400                            }
401                    }
402                    catch (IOException ioe) {
403                            throw new SystemException(ioe);
404                    }
405                    catch (ServiceException se) {
406                            throw new SystemException(se);
407                    }
408            }
409    
410            @Override
411            public void updateFile(
412                            long companyId, long repositoryId, String fileName,
413                            String versionLabel, InputStream is)
414                    throws SystemException {
415    
416                    try {
417                            S3Object s3Object = new S3Object(
418                                    _s3Bucket,
419                                    getKey(companyId, repositoryId, fileName, versionLabel));
420    
421                            s3Object.setDataInputStream(is);
422    
423                            _s3Service.putObject(_s3Bucket, s3Object);
424    
425                            is.close();
426                    }
427                    catch (IOException ioe) {
428                            throw new SystemException(ioe);
429                    }
430                    catch (S3ServiceException s3se) {
431                            throw new SystemException(s3se);
432                    }
433            }
434    
435            protected void cleanUpTempFiles() {
436                    _calledGetFileCount++;
437    
438                    if (_calledGetFileCount <
439                                    PropsValues.DL_STORE_S3_TEMP_DIR_CLEAN_UP_FREQUENCY) {
440    
441                            return;
442                    }
443    
444                    synchronized (this) {
445                            if (_calledGetFileCount == 0) {
446                                    return;
447                            }
448    
449                            _calledGetFileCount = 0;
450    
451                            String tempDirName =
452                                    SystemProperties.get(SystemProperties.TMP_DIR) + _TEMP_DIR_NAME;
453    
454                            File tempDir = new File(tempDirName);
455    
456                            long lastModified = System.currentTimeMillis();
457    
458                            lastModified -=
459                                    (PropsValues.DL_STORE_S3_TEMP_DIR_CLEAN_UP_EXPUNGE * Time.DAY);
460    
461                            cleanUpTempFiles(tempDir, lastModified);
462                    }
463            }
464    
465            protected void cleanUpTempFiles(File file, long lastModified) {
466                    if (!file.isDirectory()) {
467                            return;
468                    }
469    
470                    String[] fileNames = FileUtil.listDirs(file);
471    
472                    if (fileNames.length == 0) {
473                            if (file.lastModified() < lastModified) {
474                                    FileUtil.deltree(file);
475    
476                                    return;
477                            }
478                    }
479                    else {
480                            for (String fileName : fileNames) {
481                                    cleanUpTempFiles(new File(file, fileName), lastModified);
482                            }
483    
484                            String[] subfileNames = file.list();
485    
486                            if (subfileNames.length == 0) {
487                                    FileUtil.deltree(file);
488    
489                                    return;
490                            }
491                    }
492            }
493    
494            protected AWSCredentials getAWSCredentials() throws S3ServiceException {
495                    if (Validator.isNull(_ACCESS_KEY) || Validator.isNull(_SECRET_KEY)) {
496                            throw new S3ServiceException(
497                                    "S3 access and secret keys are not set");
498                    }
499                    else {
500                            return new AWSCredentials(_ACCESS_KEY, _SECRET_KEY);
501                    }
502            }
503    
504            protected String getFileName(String key) {
505    
506                    // Convert /${companyId}/${repositoryId}/${dirName}/${fileName}
507                    // /${versionLabel} to /${dirName}/${fileName}
508    
509                    int x = key.indexOf(CharPool.SLASH);
510    
511                    x = key.indexOf(CharPool.SLASH, x + 1);
512    
513                    int y = key.lastIndexOf(CharPool.SLASH);
514    
515                    return key.substring(x, y);
516            }
517    
518            protected String[] getFileNames(S3Object[] s3Objects) {
519                    List<String> fileNames = new ArrayList<String>();
520    
521                    for (S3Object s3Object : s3Objects) {
522                            String fileName = getFileName(s3Object.getKey());
523    
524                            fileNames.add(fileName);
525                    }
526    
527                    return fileNames.toArray(new String[fileNames.size()]);
528            }
529    
530            protected String getHeadVersionLabel(
531                            long companyId, long repositoryId, String fileName)
532                    throws PortalException, S3ServiceException {
533    
534                    S3Object[] s3Objects = _s3Service.listObjects(
535                            _s3Bucket.getName(), getKey(companyId, repositoryId, fileName),
536                            null);
537    
538                    String[] keys = new String[s3Objects.length];
539    
540                    for (int i = 0; i < s3Objects.length; i++) {
541                            S3Object s3Object = s3Objects[i];
542    
543                            keys[i] = s3Object.getKey();
544                    }
545    
546                    if (keys.length > 0) {
547                            Arrays.sort(keys);
548    
549                            String headKey = keys[keys.length - 1];
550    
551                            int x = headKey.lastIndexOf(CharPool.SLASH);
552    
553                            return headKey.substring(x + 1);
554                    }
555                    else {
556                            throw new NoSuchFileException(fileName);
557                    }
558            }
559    
560            protected String getKey(long companyId, long repositoryId) {
561                    StringBundler sb = new StringBundler(4);
562    
563                    sb.append(companyId);
564                    sb.append(StringPool.SLASH);
565                    sb.append(repositoryId);
566    
567                    return sb.toString();
568            }
569    
570            protected String getKey(
571                    long companyId, long repositoryId, String fileName) {
572    
573                    StringBundler sb = new StringBundler(4);
574    
575                    sb.append(companyId);
576                    sb.append(StringPool.SLASH);
577                    sb.append(repositoryId);
578                    sb.append(getNormalizedFileName(fileName));
579    
580                    return sb.toString();
581            }
582    
583            protected String getKey(
584                    long companyId, long repositoryId, String fileName,
585                    String versionLabel) {
586    
587                    StringBundler sb = new StringBundler(6);
588    
589                    sb.append(companyId);
590                    sb.append(StringPool.SLASH);
591                    sb.append(repositoryId);
592                    sb.append(getNormalizedFileName(fileName));
593                    sb.append(StringPool.SLASH);
594                    sb.append(versionLabel);
595    
596                    return sb.toString();
597            }
598    
599            protected String getNormalizedFileName(String fileName) {
600                    String normalizedFileName = fileName;
601    
602                    if (!fileName.startsWith(StringPool.SLASH)) {
603                            normalizedFileName = StringPool.SLASH + normalizedFileName;
604                    }
605    
606                    if (fileName.endsWith(StringPool.SLASH)) {
607                            normalizedFileName = normalizedFileName.substring(
608                                    0, normalizedFileName.length() - 1);
609                    }
610    
611                    return normalizedFileName;
612            }
613    
614            protected S3Bucket getS3Bucket() throws S3ServiceException {
615                    if (Validator.isNull(_BUCKET_NAME)) {
616                            throw new S3ServiceException("S3 bucket name is not set");
617                    }
618                    else {
619                            return getS3Service().getBucket(_BUCKET_NAME);
620                    }
621            }
622    
623            protected S3Service getS3Service() throws S3ServiceException {
624                    AWSCredentials credentials = getAWSCredentials();
625    
626                    return new RestS3Service(credentials);
627            }
628    
629            protected File getTempFile(S3Object s3Object, String fileName)
630                    throws IOException, ServiceException {
631    
632                    StringBundler sb = new StringBundler(5);
633    
634                    sb.append(SystemProperties.get(SystemProperties.TMP_DIR));
635                    sb.append(_TEMP_DIR_NAME);
636                    sb.append(
637                            DateUtil.getCurrentDate(
638                                    _TEMP_DIR_PATTERN, LocaleUtil.getDefault()));
639                    sb.append(getNormalizedFileName(fileName));
640    
641                    Date lastModifiedDate = s3Object.getLastModifiedDate();
642    
643                    sb.append(lastModifiedDate.getTime());
644    
645                    String tempFileName = sb.toString();
646    
647                    File tempFile = new File(tempFileName);
648    
649                    if (tempFile.exists() &&
650                            (tempFile.lastModified() >= lastModifiedDate.getTime())) {
651    
652                            return tempFile;
653                    }
654    
655                    InputStream inputStream = s3Object.getDataInputStream();
656    
657                    if (inputStream == null) {
658                            throw new IOException("S3 object input stream is null");
659                    }
660    
661                    OutputStream outputStream = null;
662    
663                    try {
664                            File parentFile = tempFile.getParentFile();
665    
666                            parentFile.mkdirs();
667    
668                            outputStream = new FileOutputStream(tempFile);
669    
670                            StreamUtil.transfer(inputStream, outputStream);
671                    }
672                    finally {
673                            StreamUtil.cleanUp(inputStream, outputStream);
674                    }
675    
676                    return tempFile;
677            }
678    
679            private static final String _ACCESS_KEY = PropsUtil.get(
680                    PropsKeys.DL_STORE_S3_ACCESS_KEY);
681    
682            private static final String _BUCKET_NAME = PropsUtil.get(
683                    PropsKeys.DL_STORE_S3_BUCKET_NAME);
684    
685            private static final String _SECRET_KEY = PropsUtil.get(
686                    PropsKeys.DL_STORE_S3_SECRET_KEY);
687    
688            private static final String _TEMP_DIR_NAME = "/liferay/s3";
689    
690            private static final String _TEMP_DIR_PATTERN = "/yyyy/MM/dd/HH/";
691    
692            private static Log _log = LogFactoryUtil.getLog(S3Store.class);
693    
694            private int _calledGetFileCount;
695            private S3Bucket _s3Bucket;
696            private S3Service _s3Service;
697    
698    }