001
014
015 package com.liferay.portlet.documentlibrary.store;
016
017 import com.liferay.portal.kernel.exception.PortalException;
018 import com.liferay.portal.kernel.exception.SystemException;
019 import com.liferay.portal.kernel.log.Log;
020 import com.liferay.portal.kernel.log.LogFactoryUtil;
021 import com.liferay.portal.kernel.util.CharPool;
022 import com.liferay.portal.kernel.util.DateUtil;
023 import com.liferay.portal.kernel.util.FileUtil;
024 import com.liferay.portal.kernel.util.LocaleUtil;
025 import com.liferay.portal.kernel.util.PropsKeys;
026 import com.liferay.portal.kernel.util.StreamUtil;
027 import com.liferay.portal.kernel.util.StringBundler;
028 import com.liferay.portal.kernel.util.StringPool;
029 import com.liferay.portal.kernel.util.SystemProperties;
030 import com.liferay.portal.kernel.util.Time;
031 import com.liferay.portal.kernel.util.Validator;
032 import com.liferay.portal.kernel.uuid.PortalUUIDUtil;
033 import com.liferay.portal.util.PropsUtil;
034 import com.liferay.portal.util.PropsValues;
035 import com.liferay.portlet.documentlibrary.NoSuchFileException;
036
037 import java.io.File;
038 import java.io.FileInputStream;
039 import java.io.FileOutputStream;
040 import java.io.IOException;
041 import java.io.InputStream;
042 import java.io.OutputStream;
043
044 import java.util.ArrayList;
045 import java.util.Arrays;
046 import java.util.Date;
047 import java.util.List;
048
049 import org.jets3t.service.S3Service;
050 import org.jets3t.service.S3ServiceException;
051 import org.jets3t.service.ServiceException;
052 import org.jets3t.service.impl.rest.httpclient.RestS3Service;
053 import org.jets3t.service.model.S3Bucket;
054 import org.jets3t.service.model.S3Object;
055 import org.jets3t.service.model.StorageObject;
056 import org.jets3t.service.security.AWSCredentials;
057
058
065 public class S3Store extends BaseStore {
066
067 public S3Store() {
068 try {
069 _s3Service = getS3Service();
070 _s3Bucket = getS3Bucket();
071 }
072 catch (S3ServiceException s3se) {
073 _log.error(s3se.getMessage());
074 }
075 }
076
077 @Override
078 public void addDirectory(
079 long companyId, long repositoryId, String dirName) {
080 }
081
082 @Override
083 public void addFile(
084 long companyId, long repositoryId, String fileName, InputStream is)
085 throws SystemException {
086
087 try {
088 S3Object s3Object = new S3Object(
089 _s3Bucket,
090 getKey(companyId, repositoryId, fileName, VERSION_DEFAULT));
091
092 s3Object.setDataInputStream(is);
093
094 _s3Service.putObject(_s3Bucket, s3Object);
095
096 is.close();
097 }
098 catch (IOException ioe) {
099 throw new SystemException(ioe);
100 }
101 catch (S3ServiceException s3se) {
102 throw new SystemException(s3se);
103 }
104 }
105
106 @Override
107 public void checkRoot(long companyId) {
108 }
109
110 @Override
111 public void deleteDirectory(
112 long companyId, long repositoryId, String dirName)
113 throws SystemException {
114
115 try {
116 S3Object[] s3Objects = _s3Service.listObjects(
117 _s3Bucket.getName(), getKey(companyId, repositoryId, dirName),
118 null);
119
120 for (S3Object s3Object : s3Objects) {
121 _s3Service.deleteObject(_s3Bucket, s3Object.getKey());
122 }
123 }
124 catch (S3ServiceException s3se) {
125 throw new SystemException(s3se);
126 }
127 }
128
129 @Override
130 public void deleteFile(long companyId, long repositoryId, String fileName)
131 throws SystemException {
132
133 try {
134 S3Object[] s3Objects = _s3Service.listObjects(
135 _s3Bucket.getName(), getKey(companyId, repositoryId, fileName),
136 null);
137
138 for (S3Object s3Object : s3Objects) {
139 _s3Service.deleteObject(_s3Bucket, s3Object.getKey());
140 }
141 }
142 catch (S3ServiceException s3se) {
143 throw new SystemException(s3se);
144 }
145 }
146
147 @Override
148 public void deleteFile(
149 long companyId, long repositoryId, String fileName,
150 String versionLabel)
151 throws SystemException {
152
153 try {
154 _s3Service.deleteObject(
155 _s3Bucket,
156 getKey(companyId, repositoryId, fileName, versionLabel));
157 }
158 catch (S3ServiceException s3se) {
159 throw new SystemException(s3se);
160 }
161 }
162
163 @Override
164 public File getFile(
165 long companyId, long repositoryId, String fileName,
166 String versionLabel)
167 throws PortalException, SystemException {
168
169 try {
170 if (Validator.isNull(versionLabel)) {
171 versionLabel = getHeadVersionLabel(
172 companyId, repositoryId, fileName);
173 }
174
175 S3Object s3Object = _s3Service.getObject(
176 _s3Bucket.getName(),
177 getKey(companyId, repositoryId, fileName, versionLabel));
178
179 File tempFile = getTempFile(s3Object, fileName);
180
181 cleanUpTempFiles();
182
183 return tempFile;
184 }
185 catch (IOException ioe) {
186 throw new SystemException(ioe);
187 }
188 catch (ServiceException se) {
189 throw new SystemException(se);
190 }
191 }
192
193 @Override
194 public InputStream getFileAsStream(
195 long companyId, long repositoryId, String fileName,
196 String versionLabel)
197 throws PortalException, SystemException {
198
199 try {
200 if (Validator.isNull(versionLabel)) {
201 versionLabel = getHeadVersionLabel(
202 companyId, repositoryId, fileName);
203 }
204
205 S3Object s3Object = _s3Service.getObject(
206 _s3Bucket.getName(),
207 getKey(companyId, repositoryId, fileName, versionLabel));
208
209 return s3Object.getDataInputStream();
210 }
211 catch (ServiceException se) {
212 throw new SystemException(se);
213 }
214 }
215
216 @Override
217 public String[] getFileNames(long companyId, long repositoryId)
218 throws SystemException {
219
220 try {
221 S3Object[] s3Objects = _s3Service.listObjects(
222 _s3Bucket.getName(), getKey(companyId, repositoryId), null);
223
224 return getFileNames(s3Objects);
225 }
226 catch (S3ServiceException s3se) {
227 throw new SystemException(s3se);
228 }
229 }
230
231 @Override
232 public String[] getFileNames(
233 long companyId, long repositoryId, String dirName)
234 throws SystemException {
235
236 try {
237 S3Object[] s3Objects = _s3Service.listObjects(
238 _s3Bucket.getName(), getKey(companyId, repositoryId, dirName),
239 null);
240
241 return getFileNames(s3Objects);
242 }
243 catch (S3ServiceException s3se) {
244 throw new SystemException(s3se);
245 }
246 }
247
248 @Override
249 public long getFileSize(long companyId, long repositoryId, String fileName)
250 throws PortalException, SystemException {
251
252 try {
253 String versionLabel = getHeadVersionLabel(
254 companyId, repositoryId, fileName);
255
256 StorageObject storageObject = _s3Service.getObjectDetails(
257 _s3Bucket.getName(),
258 getKey(companyId, repositoryId, fileName, versionLabel));
259
260 return storageObject.getContentLength();
261 }
262 catch (ServiceException se) {
263 throw new SystemException(se);
264 }
265 }
266
267 @Override
268 public boolean hasDirectory(
269 long companyId, long repositoryId, String dirName) {
270
271 return true;
272 }
273
274 @Override
275 public boolean hasFile(
276 long companyId, long repositoryId, String fileName,
277 String versionLabel)
278 throws SystemException {
279
280 try {
281 S3Object[] s3Objects = _s3Service.listObjects(
282 _s3Bucket.getName(),
283 getKey(companyId, repositoryId, fileName, versionLabel), null);
284
285 if (s3Objects.length == 0) {
286 return false;
287 }
288 else {
289 return true;
290 }
291 }
292 catch (S3ServiceException s3se) {
293 throw new SystemException(s3se);
294 }
295 }
296
297 @Override
298 public void move(String srcDir, String destDir) {
299 }
300
301 @Override
302 public void updateFile(
303 long companyId, long repositoryId, long newRepositoryId,
304 String fileName)
305 throws SystemException {
306
307 try {
308 S3Object[] s3Objects = _s3Service.listObjects(
309 _s3Bucket.getName(), getKey(companyId, repositoryId, fileName),
310 null);
311
312 for (S3Object oldS3Object : s3Objects) {
313 String oldKey = oldS3Object.getKey();
314
315 oldS3Object = _s3Service.getObject(_s3Bucket.getName(), oldKey);
316
317 File tempFile = new File(
318 SystemProperties.get(SystemProperties.TMP_DIR) +
319 File.separator + PortalUUIDUtil.generate());
320
321 FileUtil.write(tempFile, oldS3Object.getDataInputStream());
322
323 InputStream is = new FileInputStream(tempFile);
324
325 String newPrefix = getKey(companyId, newRepositoryId);
326
327 int x = oldKey.indexOf(CharPool.SLASH);
328
329 x = oldKey.indexOf(CharPool.SLASH, x + 1);
330
331 String newKey = newPrefix + oldKey.substring(x + 1);
332
333 S3Object newS3Object = new S3Object(_s3Bucket, newKey);
334
335 newS3Object.setDataInputStream(is);
336
337 _s3Service.putObject(_s3Bucket, newS3Object);
338 _s3Service.deleteObject(_s3Bucket, oldKey);
339
340 is.close();
341
342 FileUtil.delete(tempFile);
343 }
344 }
345 catch (IOException ioe) {
346 throw new SystemException(ioe);
347 }
348 catch (ServiceException se) {
349 throw new SystemException(se);
350 }
351 }
352
353 @Override
354 public void updateFile(
355 long companyId, long repositoryId, String fileName,
356 String newFileName)
357 throws SystemException {
358
359 try {
360 S3Object[] s3Objects = _s3Service.listObjects(
361 _s3Bucket.getName(), getKey(companyId, repositoryId, fileName),
362 null);
363
364 for (S3Object oldS3Object : s3Objects) {
365 String oldKey = oldS3Object.getKey();
366
367 oldS3Object = _s3Service.getObject(_s3Bucket.getName(), oldKey);
368
369 File tempFile = new File(
370 SystemProperties.get(SystemProperties.TMP_DIR) +
371 File.separator + PortalUUIDUtil.generate());
372
373 FileUtil.write(tempFile, oldS3Object.getDataInputStream());
374
375 oldS3Object.closeDataInputStream();
376
377 InputStream is = new FileInputStream(tempFile);
378
379 String newPrefix = getKey(companyId, repositoryId, newFileName);
380
381 int x = oldKey.indexOf(StringPool.SLASH);
382
383 x = oldKey.indexOf(CharPool.SLASH, x + 1);
384 x = oldKey.indexOf(CharPool.SLASH, x + 1);
385
386 String newKey = newPrefix + oldKey.substring(x + 1);
387
388 S3Object newS3Object = new S3Object(_s3Bucket, newKey);
389
390 newS3Object.setDataInputStream(is);
391
392 _s3Service.putObject(_s3Bucket, newS3Object);
393 _s3Service.deleteObject(_s3Bucket, oldKey);
394
395 newS3Object.closeDataInputStream();
396
397 is.close();
398
399 FileUtil.delete(tempFile);
400 }
401 }
402 catch (IOException ioe) {
403 throw new SystemException(ioe);
404 }
405 catch (ServiceException se) {
406 throw new SystemException(se);
407 }
408 }
409
410 @Override
411 public void updateFile(
412 long companyId, long repositoryId, String fileName,
413 String versionLabel, InputStream is)
414 throws SystemException {
415
416 try {
417 S3Object s3Object = new S3Object(
418 _s3Bucket,
419 getKey(companyId, repositoryId, fileName, versionLabel));
420
421 s3Object.setDataInputStream(is);
422
423 _s3Service.putObject(_s3Bucket, s3Object);
424
425 is.close();
426 }
427 catch (IOException ioe) {
428 throw new SystemException(ioe);
429 }
430 catch (S3ServiceException s3se) {
431 throw new SystemException(s3se);
432 }
433 }
434
435 protected void cleanUpTempFiles() {
436 _calledGetFileCount++;
437
438 if (_calledGetFileCount <
439 PropsValues.DL_STORE_S3_TEMP_DIR_CLEAN_UP_FREQUENCY) {
440
441 return;
442 }
443
444 synchronized (this) {
445 if (_calledGetFileCount == 0) {
446 return;
447 }
448
449 _calledGetFileCount = 0;
450
451 String tempDirName =
452 SystemProperties.get(SystemProperties.TMP_DIR) + _TEMP_DIR_NAME;
453
454 File tempDir = new File(tempDirName);
455
456 long lastModified = System.currentTimeMillis();
457
458 lastModified -=
459 (PropsValues.DL_STORE_S3_TEMP_DIR_CLEAN_UP_EXPUNGE * Time.DAY);
460
461 cleanUpTempFiles(tempDir, lastModified);
462 }
463 }
464
465 protected void cleanUpTempFiles(File file, long lastModified) {
466 if (!file.isDirectory()) {
467 return;
468 }
469
470 String[] fileNames = FileUtil.listDirs(file);
471
472 if (fileNames.length == 0) {
473 if (file.lastModified() < lastModified) {
474 FileUtil.deltree(file);
475
476 return;
477 }
478 }
479 else {
480 for (String fileName : fileNames) {
481 cleanUpTempFiles(new File(file, fileName), lastModified);
482 }
483
484 String[] subfileNames = file.list();
485
486 if (subfileNames.length == 0) {
487 FileUtil.deltree(file);
488
489 return;
490 }
491 }
492 }
493
494 protected AWSCredentials getAWSCredentials() throws S3ServiceException {
495 if (Validator.isNull(_ACCESS_KEY) || Validator.isNull(_SECRET_KEY)) {
496 throw new S3ServiceException(
497 "S3 access and secret keys are not set");
498 }
499 else {
500 return new AWSCredentials(_ACCESS_KEY, _SECRET_KEY);
501 }
502 }
503
504 protected String getFileName(String key) {
505
506
507
508
509 int x = key.indexOf(CharPool.SLASH);
510
511 x = key.indexOf(CharPool.SLASH, x + 1);
512
513 int y = key.lastIndexOf(CharPool.SLASH);
514
515 return key.substring(x, y);
516 }
517
518 protected String[] getFileNames(S3Object[] s3Objects) {
519 List<String> fileNames = new ArrayList<String>();
520
521 for (S3Object s3Object : s3Objects) {
522 String fileName = getFileName(s3Object.getKey());
523
524 fileNames.add(fileName);
525 }
526
527 return fileNames.toArray(new String[fileNames.size()]);
528 }
529
530 protected String getHeadVersionLabel(
531 long companyId, long repositoryId, String fileName)
532 throws PortalException, S3ServiceException {
533
534 S3Object[] s3Objects = _s3Service.listObjects(
535 _s3Bucket.getName(), getKey(companyId, repositoryId, fileName),
536 null);
537
538 String[] keys = new String[s3Objects.length];
539
540 for (int i = 0; i < s3Objects.length; i++) {
541 S3Object s3Object = s3Objects[i];
542
543 keys[i] = s3Object.getKey();
544 }
545
546 if (keys.length > 0) {
547 Arrays.sort(keys);
548
549 String headKey = keys[keys.length - 1];
550
551 int x = headKey.lastIndexOf(CharPool.SLASH);
552
553 return headKey.substring(x + 1);
554 }
555 else {
556 throw new NoSuchFileException(fileName);
557 }
558 }
559
560 protected String getKey(long companyId, long repositoryId) {
561 StringBundler sb = new StringBundler(4);
562
563 sb.append(companyId);
564 sb.append(StringPool.SLASH);
565 sb.append(repositoryId);
566
567 return sb.toString();
568 }
569
570 protected String getKey(
571 long companyId, long repositoryId, String fileName) {
572
573 StringBundler sb = new StringBundler(4);
574
575 sb.append(companyId);
576 sb.append(StringPool.SLASH);
577 sb.append(repositoryId);
578 sb.append(getNormalizedFileName(fileName));
579
580 return sb.toString();
581 }
582
583 protected String getKey(
584 long companyId, long repositoryId, String fileName,
585 String versionLabel) {
586
587 StringBundler sb = new StringBundler(6);
588
589 sb.append(companyId);
590 sb.append(StringPool.SLASH);
591 sb.append(repositoryId);
592 sb.append(getNormalizedFileName(fileName));
593 sb.append(StringPool.SLASH);
594 sb.append(versionLabel);
595
596 return sb.toString();
597 }
598
599 protected String getNormalizedFileName(String fileName) {
600 String normalizedFileName = fileName;
601
602 if (!fileName.startsWith(StringPool.SLASH)) {
603 normalizedFileName = StringPool.SLASH + normalizedFileName;
604 }
605
606 if (fileName.endsWith(StringPool.SLASH)) {
607 normalizedFileName = normalizedFileName.substring(
608 0, normalizedFileName.length() - 1);
609 }
610
611 return normalizedFileName;
612 }
613
614 protected S3Bucket getS3Bucket() throws S3ServiceException {
615 if (Validator.isNull(_BUCKET_NAME)) {
616 throw new S3ServiceException("S3 bucket name is not set");
617 }
618 else {
619 return getS3Service().getBucket(_BUCKET_NAME);
620 }
621 }
622
623 protected S3Service getS3Service() throws S3ServiceException {
624 AWSCredentials credentials = getAWSCredentials();
625
626 return new RestS3Service(credentials);
627 }
628
629 protected File getTempFile(S3Object s3Object, String fileName)
630 throws IOException, ServiceException {
631
632 StringBundler sb = new StringBundler(5);
633
634 sb.append(SystemProperties.get(SystemProperties.TMP_DIR));
635 sb.append(_TEMP_DIR_NAME);
636 sb.append(
637 DateUtil.getCurrentDate(
638 _TEMP_DIR_PATTERN, LocaleUtil.getDefault()));
639 sb.append(getNormalizedFileName(fileName));
640
641 Date lastModifiedDate = s3Object.getLastModifiedDate();
642
643 sb.append(lastModifiedDate.getTime());
644
645 String tempFileName = sb.toString();
646
647 File tempFile = new File(tempFileName);
648
649 if (tempFile.exists() &&
650 (tempFile.lastModified() >= lastModifiedDate.getTime())) {
651
652 return tempFile;
653 }
654
655 InputStream inputStream = s3Object.getDataInputStream();
656
657 if (inputStream == null) {
658 throw new IOException("S3 object input stream is null");
659 }
660
661 OutputStream outputStream = null;
662
663 try {
664 File parentFile = tempFile.getParentFile();
665
666 parentFile.mkdirs();
667
668 outputStream = new FileOutputStream(tempFile);
669
670 StreamUtil.transfer(inputStream, outputStream);
671 }
672 finally {
673 StreamUtil.cleanUp(inputStream, outputStream);
674 }
675
676 return tempFile;
677 }
678
679 private static final String _ACCESS_KEY = PropsUtil.get(
680 PropsKeys.DL_STORE_S3_ACCESS_KEY);
681
682 private static final String _BUCKET_NAME = PropsUtil.get(
683 PropsKeys.DL_STORE_S3_BUCKET_NAME);
684
685 private static final String _SECRET_KEY = PropsUtil.get(
686 PropsKeys.DL_STORE_S3_SECRET_KEY);
687
688 private static final String _TEMP_DIR_NAME = "/liferay/s3";
689
690 private static final String _TEMP_DIR_PATTERN = "/yyyy/MM/dd/HH/";
691
692 private static Log _log = LogFactoryUtil.getLog(S3Store.class);
693
694 private int _calledGetFileCount;
695 private S3Bucket _s3Bucket;
696 private S3Service _s3Service;
697
698 }