001
014
015 package com.liferay.portlet.documentlibrary.store;
016
017 import com.liferay.portal.kernel.exception.PortalException;
018 import com.liferay.portal.kernel.exception.SystemException;
019 import com.liferay.portal.kernel.log.Log;
020 import com.liferay.portal.kernel.log.LogFactoryUtil;
021 import com.liferay.portal.kernel.util.CharPool;
022 import com.liferay.portal.kernel.util.DateUtil;
023 import com.liferay.portal.kernel.util.FileUtil;
024 import com.liferay.portal.kernel.util.LocaleUtil;
025 import com.liferay.portal.kernel.util.PropsKeys;
026 import com.liferay.portal.kernel.util.StreamUtil;
027 import com.liferay.portal.kernel.util.StringBundler;
028 import com.liferay.portal.kernel.util.StringPool;
029 import com.liferay.portal.kernel.util.SystemProperties;
030 import com.liferay.portal.kernel.util.Time;
031 import com.liferay.portal.kernel.util.Validator;
032 import com.liferay.portal.kernel.uuid.PortalUUIDUtil;
033 import com.liferay.portal.util.PropsUtil;
034 import com.liferay.portal.util.PropsValues;
035 import com.liferay.portlet.documentlibrary.NoSuchFileException;
036
037 import java.io.File;
038 import java.io.FileInputStream;
039 import java.io.FileOutputStream;
040 import java.io.IOException;
041 import java.io.InputStream;
042 import java.io.OutputStream;
043
044 import java.util.ArrayList;
045 import java.util.Arrays;
046 import java.util.Date;
047 import java.util.List;
048
049 import org.jets3t.service.S3Service;
050 import org.jets3t.service.S3ServiceException;
051 import org.jets3t.service.ServiceException;
052 import org.jets3t.service.impl.rest.httpclient.RestS3Service;
053 import org.jets3t.service.model.S3Bucket;
054 import org.jets3t.service.model.S3Object;
055 import org.jets3t.service.model.StorageObject;
056 import org.jets3t.service.security.AWSCredentials;
057
058
065 public class S3Store extends BaseStore {
066
067 public S3Store() {
068 try {
069 _s3Service = getS3Service();
070 _s3Bucket = getS3Bucket();
071 }
072 catch (S3ServiceException s3se) {
073 _log.error(s3se.getMessage());
074 }
075 }
076
077 @Override
078 public void addDirectory(
079 long companyId, long repositoryId, String dirName) {
080 }
081
082 @Override
083 public void addFile(
084 long companyId, long repositoryId, String fileName, InputStream is)
085 throws SystemException {
086
087 try {
088 S3Object s3Object = new S3Object(
089 _s3Bucket,
090 getKey(companyId, repositoryId, fileName, VERSION_DEFAULT));
091
092 s3Object.setDataInputStream(is);
093
094 _s3Service.putObject(_s3Bucket, s3Object);
095 }
096 catch (S3ServiceException s3se) {
097 throw new SystemException(s3se);
098 }
099 finally {
100 StreamUtil.cleanUp(is);
101 }
102 }
103
104 @Override
105 public void checkRoot(long companyId) {
106 }
107
108 @Override
109 public void deleteDirectory(
110 long companyId, long repositoryId, String dirName)
111 throws SystemException {
112
113 try {
114 S3Object[] s3Objects = _s3Service.listObjects(
115 _s3Bucket.getName(), getKey(companyId, repositoryId, dirName),
116 null);
117
118 for (S3Object s3Object : s3Objects) {
119 _s3Service.deleteObject(_s3Bucket, s3Object.getKey());
120 }
121 }
122 catch (S3ServiceException s3se) {
123 throw new SystemException(s3se);
124 }
125 }
126
127 @Override
128 public void deleteFile(long companyId, long repositoryId, String fileName)
129 throws SystemException {
130
131 try {
132 S3Object[] s3Objects = _s3Service.listObjects(
133 _s3Bucket.getName(), getKey(companyId, repositoryId, fileName),
134 null);
135
136 for (S3Object s3Object : s3Objects) {
137 _s3Service.deleteObject(_s3Bucket, s3Object.getKey());
138 }
139 }
140 catch (S3ServiceException s3se) {
141 throw new SystemException(s3se);
142 }
143 }
144
145 @Override
146 public void deleteFile(
147 long companyId, long repositoryId, String fileName,
148 String versionLabel)
149 throws SystemException {
150
151 try {
152 _s3Service.deleteObject(
153 _s3Bucket,
154 getKey(companyId, repositoryId, fileName, versionLabel));
155 }
156 catch (S3ServiceException s3se) {
157 throw new SystemException(s3se);
158 }
159 }
160
161 @Override
162 public File getFile(
163 long companyId, long repositoryId, String fileName,
164 String versionLabel)
165 throws PortalException, SystemException {
166
167 try {
168 if (Validator.isNull(versionLabel)) {
169 versionLabel = getHeadVersionLabel(
170 companyId, repositoryId, fileName);
171 }
172
173 S3Object s3Object = _s3Service.getObject(
174 _s3Bucket.getName(),
175 getKey(companyId, repositoryId, fileName, versionLabel));
176
177 File tempFile = getTempFile(s3Object, fileName);
178
179 cleanUpTempFiles();
180
181 return tempFile;
182 }
183 catch (IOException ioe) {
184 throw new SystemException(ioe);
185 }
186 catch (ServiceException se) {
187 throw new SystemException(se);
188 }
189 }
190
191 @Override
192 public InputStream getFileAsStream(
193 long companyId, long repositoryId, String fileName,
194 String versionLabel)
195 throws PortalException, SystemException {
196
197 try {
198 if (Validator.isNull(versionLabel)) {
199 versionLabel = getHeadVersionLabel(
200 companyId, repositoryId, fileName);
201 }
202
203 S3Object s3Object = _s3Service.getObject(
204 _s3Bucket.getName(),
205 getKey(companyId, repositoryId, fileName, versionLabel));
206
207 return s3Object.getDataInputStream();
208 }
209 catch (ServiceException se) {
210 throw new SystemException(se);
211 }
212 }
213
214 @Override
215 public String[] getFileNames(long companyId, long repositoryId)
216 throws SystemException {
217
218 try {
219 S3Object[] s3Objects = _s3Service.listObjects(
220 _s3Bucket.getName(), getKey(companyId, repositoryId), null);
221
222 return getFileNames(s3Objects);
223 }
224 catch (S3ServiceException s3se) {
225 throw new SystemException(s3se);
226 }
227 }
228
229 @Override
230 public String[] getFileNames(
231 long companyId, long repositoryId, String dirName)
232 throws SystemException {
233
234 try {
235 S3Object[] s3Objects = _s3Service.listObjects(
236 _s3Bucket.getName(), getKey(companyId, repositoryId, dirName),
237 null);
238
239 return getFileNames(s3Objects);
240 }
241 catch (S3ServiceException s3se) {
242 throw new SystemException(s3se);
243 }
244 }
245
246 @Override
247 public long getFileSize(long companyId, long repositoryId, String fileName)
248 throws PortalException, SystemException {
249
250 try {
251 String versionLabel = getHeadVersionLabel(
252 companyId, repositoryId, fileName);
253
254 StorageObject storageObject = _s3Service.getObjectDetails(
255 _s3Bucket.getName(),
256 getKey(companyId, repositoryId, fileName, versionLabel));
257
258 return storageObject.getContentLength();
259 }
260 catch (ServiceException se) {
261 throw new SystemException(se);
262 }
263 }
264
265 @Override
266 public boolean hasDirectory(
267 long companyId, long repositoryId, String dirName) {
268
269 return true;
270 }
271
272 @Override
273 public boolean hasFile(
274 long companyId, long repositoryId, String fileName,
275 String versionLabel)
276 throws SystemException {
277
278 try {
279 S3Object[] s3Objects = _s3Service.listObjects(
280 _s3Bucket.getName(),
281 getKey(companyId, repositoryId, fileName, versionLabel), null);
282
283 if (s3Objects.length == 0) {
284 return false;
285 }
286 else {
287 return true;
288 }
289 }
290 catch (S3ServiceException s3se) {
291 throw new SystemException(s3se);
292 }
293 }
294
295 @Override
296 public void move(String srcDir, String destDir) {
297 }
298
299 @Override
300 public void updateFile(
301 long companyId, long repositoryId, long newRepositoryId,
302 String fileName)
303 throws SystemException {
304
305 File tempFile = null;
306 InputStream is = null;
307 S3Object newS3Object = null;
308
309 try {
310 S3Object[] s3Objects = _s3Service.listObjects(
311 _s3Bucket.getName(), getKey(companyId, repositoryId, fileName),
312 null);
313
314 for (S3Object oldS3Object : s3Objects) {
315 String oldKey = oldS3Object.getKey();
316
317 oldS3Object = _s3Service.getObject(_s3Bucket.getName(), oldKey);
318
319 tempFile = new File(
320 SystemProperties.get(SystemProperties.TMP_DIR) +
321 File.separator + PortalUUIDUtil.generate());
322
323 FileUtil.write(tempFile, oldS3Object.getDataInputStream());
324
325 is = new FileInputStream(tempFile);
326
327 String newPrefix = getKey(companyId, newRepositoryId);
328
329 int x = oldKey.indexOf(CharPool.SLASH);
330
331 x = oldKey.indexOf(CharPool.SLASH, x + 1);
332
333 String newKey = newPrefix + oldKey.substring(x);
334
335 newS3Object = new S3Object(_s3Bucket, newKey);
336
337 newS3Object.setDataInputStream(is);
338
339 _s3Service.putObject(_s3Bucket, newS3Object);
340 _s3Service.deleteObject(_s3Bucket, oldKey);
341 }
342 }
343 catch (IOException ioe) {
344 throw new SystemException(ioe);
345 }
346 catch (ServiceException se) {
347 throw new SystemException(se);
348 }
349 finally {
350 StreamUtil.cleanUp(is);
351
352 FileUtil.delete(tempFile);
353 }
354 }
355
356 @Override
357 public void updateFile(
358 long companyId, long repositoryId, String fileName,
359 String newFileName)
360 throws SystemException {
361
362 File tempFile = null;
363 InputStream is = null;
364 S3Object newS3Object = null;
365
366 try {
367 S3Object[] s3Objects = _s3Service.listObjects(
368 _s3Bucket.getName(), getKey(companyId, repositoryId, fileName),
369 null);
370
371 for (S3Object oldS3Object : s3Objects) {
372 String oldKey = oldS3Object.getKey();
373
374 oldS3Object = _s3Service.getObject(_s3Bucket.getName(), oldKey);
375
376 tempFile = new File(
377 SystemProperties.get(SystemProperties.TMP_DIR) +
378 File.separator + PortalUUIDUtil.generate());
379
380 FileUtil.write(tempFile, oldS3Object.getDataInputStream());
381
382 oldS3Object.closeDataInputStream();
383
384 is = new FileInputStream(tempFile);
385
386 String newPrefix = getKey(companyId, repositoryId, newFileName);
387
388 int x = oldKey.indexOf(StringPool.SLASH);
389
390 x = oldKey.indexOf(CharPool.SLASH, x + 1);
391 x = oldKey.indexOf(CharPool.SLASH, x + 1);
392
393 String newKey = newPrefix + oldKey.substring(x);
394
395 newS3Object = new S3Object(_s3Bucket, newKey);
396
397 newS3Object.setDataInputStream(is);
398
399 _s3Service.putObject(_s3Bucket, newS3Object);
400 _s3Service.deleteObject(_s3Bucket, oldKey);
401 }
402 }
403 catch (IOException ioe) {
404 throw new SystemException(ioe);
405 }
406 catch (ServiceException se) {
407 throw new SystemException(se);
408 }
409 finally {
410 StreamUtil.cleanUp(is);
411
412 FileUtil.delete(tempFile);
413 }
414 }
415
416 @Override
417 public void updateFile(
418 long companyId, long repositoryId, String fileName,
419 String versionLabel, InputStream is)
420 throws SystemException {
421
422 try {
423 S3Object s3Object = new S3Object(
424 _s3Bucket,
425 getKey(companyId, repositoryId, fileName, versionLabel));
426
427 s3Object.setDataInputStream(is);
428
429 _s3Service.putObject(_s3Bucket, s3Object);
430 }
431 catch (S3ServiceException s3se) {
432 throw new SystemException(s3se);
433 }
434 finally {
435 StreamUtil.cleanUp(is);
436 }
437 }
438
439 protected void cleanUpTempFiles() {
440 _calledGetFileCount++;
441
442 if (_calledGetFileCount <
443 PropsValues.DL_STORE_S3_TEMP_DIR_CLEAN_UP_FREQUENCY) {
444
445 return;
446 }
447
448 synchronized (this) {
449 if (_calledGetFileCount == 0) {
450 return;
451 }
452
453 _calledGetFileCount = 0;
454
455 String tempDirName =
456 SystemProperties.get(SystemProperties.TMP_DIR) + _TEMP_DIR_NAME;
457
458 File tempDir = new File(tempDirName);
459
460 long lastModified = System.currentTimeMillis();
461
462 lastModified -=
463 (PropsValues.DL_STORE_S3_TEMP_DIR_CLEAN_UP_EXPUNGE * Time.DAY);
464
465 cleanUpTempFiles(tempDir, lastModified);
466 }
467 }
468
469 protected void cleanUpTempFiles(File file, long lastModified) {
470 if (!file.isDirectory()) {
471 return;
472 }
473
474 String[] fileNames = FileUtil.listDirs(file);
475
476 if (fileNames.length == 0) {
477 if (file.lastModified() < lastModified) {
478 FileUtil.deltree(file);
479
480 return;
481 }
482 }
483 else {
484 for (String fileName : fileNames) {
485 cleanUpTempFiles(new File(file, fileName), lastModified);
486 }
487
488 String[] subfileNames = file.list();
489
490 if (subfileNames.length == 0) {
491 FileUtil.deltree(file);
492
493 return;
494 }
495 }
496 }
497
498 protected AWSCredentials getAWSCredentials() throws S3ServiceException {
499 if (Validator.isNull(_ACCESS_KEY) || Validator.isNull(_SECRET_KEY)) {
500 throw new S3ServiceException(
501 "S3 access and secret keys are not set");
502 }
503 else {
504 return new AWSCredentials(_ACCESS_KEY, _SECRET_KEY);
505 }
506 }
507
508 protected String getFileName(String key) {
509
510
511
512
513 int x = key.indexOf(CharPool.SLASH);
514
515 x = key.indexOf(CharPool.SLASH, x + 1);
516
517 int y = key.lastIndexOf(CharPool.SLASH);
518
519 return key.substring(x, y);
520 }
521
522 protected String[] getFileNames(S3Object[] s3Objects) {
523 List<String> fileNames = new ArrayList<String>();
524
525 for (S3Object s3Object : s3Objects) {
526 String fileName = getFileName(s3Object.getKey());
527
528 fileNames.add(fileName);
529 }
530
531 return fileNames.toArray(new String[fileNames.size()]);
532 }
533
534 protected String getHeadVersionLabel(
535 long companyId, long repositoryId, String fileName)
536 throws PortalException, S3ServiceException {
537
538 S3Object[] s3Objects = _s3Service.listObjects(
539 _s3Bucket.getName(), getKey(companyId, repositoryId, fileName),
540 null);
541
542 String[] keys = new String[s3Objects.length];
543
544 for (int i = 0; i < s3Objects.length; i++) {
545 S3Object s3Object = s3Objects[i];
546
547 keys[i] = s3Object.getKey();
548 }
549
550 if (keys.length > 0) {
551 Arrays.sort(keys);
552
553 String headKey = keys[keys.length - 1];
554
555 int x = headKey.lastIndexOf(CharPool.SLASH);
556
557 return headKey.substring(x + 1);
558 }
559 else {
560 throw new NoSuchFileException(fileName);
561 }
562 }
563
564 protected String getKey(long companyId, long repositoryId) {
565 StringBundler sb = new StringBundler(4);
566
567 sb.append(companyId);
568 sb.append(StringPool.SLASH);
569 sb.append(repositoryId);
570
571 return sb.toString();
572 }
573
574 protected String getKey(
575 long companyId, long repositoryId, String fileName) {
576
577 StringBundler sb = new StringBundler(4);
578
579 sb.append(companyId);
580 sb.append(StringPool.SLASH);
581 sb.append(repositoryId);
582 sb.append(getNormalizedFileName(fileName));
583
584 return sb.toString();
585 }
586
587 protected String getKey(
588 long companyId, long repositoryId, String fileName,
589 String versionLabel) {
590
591 StringBundler sb = new StringBundler(6);
592
593 sb.append(companyId);
594 sb.append(StringPool.SLASH);
595 sb.append(repositoryId);
596 sb.append(getNormalizedFileName(fileName));
597 sb.append(StringPool.SLASH);
598 sb.append(versionLabel);
599
600 return sb.toString();
601 }
602
603 protected String getNormalizedFileName(String fileName) {
604 String normalizedFileName = fileName;
605
606 if (!fileName.startsWith(StringPool.SLASH)) {
607 normalizedFileName = StringPool.SLASH + normalizedFileName;
608 }
609
610 if (fileName.endsWith(StringPool.SLASH)) {
611 normalizedFileName = normalizedFileName.substring(
612 0, normalizedFileName.length() - 1);
613 }
614
615 return normalizedFileName;
616 }
617
618 protected S3Bucket getS3Bucket() throws S3ServiceException {
619 if (Validator.isNull(_BUCKET_NAME)) {
620 throw new S3ServiceException("S3 bucket name is not set");
621 }
622 else {
623 return getS3Service().getBucket(_BUCKET_NAME);
624 }
625 }
626
627 protected S3Service getS3Service() throws S3ServiceException {
628 AWSCredentials credentials = getAWSCredentials();
629
630 return new RestS3Service(credentials);
631 }
632
633 protected File getTempFile(S3Object s3Object, String fileName)
634 throws IOException, ServiceException {
635
636 StringBundler sb = new StringBundler(5);
637
638 sb.append(SystemProperties.get(SystemProperties.TMP_DIR));
639 sb.append(_TEMP_DIR_NAME);
640 sb.append(
641 DateUtil.getCurrentDate(
642 _TEMP_DIR_PATTERN, LocaleUtil.getDefault()));
643 sb.append(getNormalizedFileName(fileName));
644
645 Date lastModifiedDate = s3Object.getLastModifiedDate();
646
647 sb.append(lastModifiedDate.getTime());
648
649 String tempFileName = sb.toString();
650
651 File tempFile = new File(tempFileName);
652
653 if (tempFile.exists() &&
654 (tempFile.lastModified() >= lastModifiedDate.getTime())) {
655
656 return tempFile;
657 }
658
659 InputStream inputStream = s3Object.getDataInputStream();
660
661 if (inputStream == null) {
662 throw new IOException("S3 object input stream is null");
663 }
664
665 OutputStream outputStream = null;
666
667 try {
668 File parentFile = tempFile.getParentFile();
669
670 FileUtil.mkdirs(parentFile);
671
672 outputStream = new FileOutputStream(tempFile);
673
674 StreamUtil.transfer(inputStream, outputStream);
675 }
676 finally {
677 StreamUtil.cleanUp(inputStream, outputStream);
678 }
679
680 return tempFile;
681 }
682
683 private static final String _ACCESS_KEY = PropsUtil.get(
684 PropsKeys.DL_STORE_S3_ACCESS_KEY);
685
686 private static final String _BUCKET_NAME = PropsUtil.get(
687 PropsKeys.DL_STORE_S3_BUCKET_NAME);
688
689 private static final String _SECRET_KEY = PropsUtil.get(
690 PropsKeys.DL_STORE_S3_SECRET_KEY);
691
692 private static final String _TEMP_DIR_NAME = "/liferay/s3";
693
694 private static final String _TEMP_DIR_PATTERN = "/yyyy/MM/dd/HH/";
695
696 private static Log _log = LogFactoryUtil.getLog(S3Store.class);
697
698 private int _calledGetFileCount;
699 private S3Bucket _s3Bucket;
700 private S3Service _s3Service;
701
702 }